wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

apply clang-format to *.c, *.cpp, *.h, *.hpp files

This commit is contained in:
FDB Formatster 2021-03-10 10:06:03 -08:00 committed by Vishesh Yadav
parent 2bb4f2e59f
commit df90cc89de
592 changed files with 118284 additions and 101547 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.clang-format
View File

@ -11,14 +11,14 @@ AllowAllParametersOfDeclarationOnNextLine: false
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: Inline
AllowShortIfStatementsOnASingleLine: true
AllowShortLoopsOnASingleLine: true
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: true
BinPackArguments: true
BinPackParameters: true
BinPackArguments: false
BinPackParameters: false
BreakBeforeBinaryOperators: None
BreakBeforeBraces: Attach
ColumnLimit: 120

7
FDBLibTLS/FDBLibTLSPlugin.cpp
View File

@ -31,10 +31,9 @@ FDBLibTLSPlugin::FDBLibTLSPlugin() {
rc = tls_init();
}
FDBLibTLSPlugin::~FDBLibTLSPlugin() {
}
FDBLibTLSPlugin::~FDBLibTLSPlugin() {}
ITLSPolicy *FDBLibTLSPlugin::create_policy() {
ITLSPolicy* FDBLibTLSPlugin::create_policy() {
if (rc < 0) {
// Log the failure from tls_init during our constructor.
TraceEvent(SevError, "FDBLibTLSInitError").detail("LibTLSErrorMessage", "failed to initialize libtls");
@ -43,7 +42,7 @@ ITLSPolicy *FDBLibTLSPlugin::create_policy() {
return new FDBLibTLSPolicy(Reference<FDBLibTLSPlugin>::addRef(this));
}
extern "C" BOOST_SYMBOL_EXPORT void *get_tls_plugin(const char *plugin_type_name_and_version) {
extern "C" BOOST_SYMBOL_EXPORT void* get_tls_plugin(const char* plugin_type_name_and_version) {
if (strcmp(plugin_type_name_and_version, FDBLibTLSPlugin::get_plugin_type_name_and_version()) == 0) {
return new FDBLibTLSPlugin;
}

2
FDBLibTLS/FDBLibTLSPlugin.h
View File

@ -35,7 +35,7 @@ struct FDBLibTLSPlugin : ITLSPlugin, ReferenceCounted<FDBLibTLSPlugin> {
virtual void addref() { ReferenceCounted<FDBLibTLSPlugin>::addref(); }
virtual void delref() { ReferenceCounted<FDBLibTLSPlugin>::delref(); }
virtual ITLSPolicy *create_policy();
virtual ITLSPolicy* create_policy();
int rc;
};

51
FDBLibTLS/FDBLibTLSPolicy.cpp
View File

@ -55,8 +55,13 @@ FDBLibTLSPolicy::~FDBLibTLSPolicy() {
tls_config_free(tls_cfg);
}
ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func,
void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid) {
ITLSSession* FDBLibTLSPolicy::create_session(bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid) {
if (is_client) {
// If verify peers has been set then there is no point specifying a
// servername, since this will be ignored - the servername should be
@ -76,20 +81,27 @@ ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servern
session_created = true;
try {
return new FDBLibTLSSession(Reference<FDBLibTLSPolicy>::addRef(this), is_client, servername, send_func,
send_ctx, recv_func, recv_ctx, uid);
} catch ( ... ) {
return new FDBLibTLSSession(Reference<FDBLibTLSPolicy>::addRef(this),
is_client,
servername,
send_func,
send_ctx,
recv_func,
recv_ctx,
uid);
} catch (...) {
return nullptr;
}
}
static int password_cb(char *buf, int size, int rwflag, void *u) {
const char *password = (const char *)u;
static int password_cb(char* buf, int size, int rwflag, void* u) {
const char* password = (const char*)u;
int plen;
if (size < 0)
return 0;
if (u == nullptr) return 0;
if (u == nullptr)
return 0;
plen = strlen(password);
if (plen > size)
@ -146,7 +158,7 @@ struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, s
return certs;
err:
err:
sk_X509_pop_free(certs, X509_free);
X509_free(cert);
BIO_free(bio);
@ -167,7 +179,8 @@ bool FDBLibTLSPolicy::set_ca_data(const uint8_t* ca_data, int ca_len) {
if (ca_len < 0)
return false;
sk_X509_pop_free(roots, X509_free);
if ((roots = parse_cert_pem(ca_data, ca_len)) == nullptr) return false;
if ((roots = parse_cert_pem(ca_data, ca_len)) == nullptr)
return false;
if (tls_config_set_ca_mem(tls_cfg, ca_data, ca_len) == -1) {
TraceEvent(SevError, "FDBLibTLSCAError").detail("LibTLSErrorMessage", tls_config_error(tls_cfg));
@ -214,7 +227,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
}
if (password != nullptr) {
char *data;
char* data;
long len;
if ((bio = BIO_new_mem_buf((void*)key_data, key_len)) == nullptr) {
@ -227,7 +240,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
char errbuf[256];
if ((ERR_GET_LIB(errnum) == ERR_LIB_PEM && ERR_GET_REASON(errnum) == PEM_R_BAD_DECRYPT) ||
(ERR_GET_LIB(errnum) == ERR_LIB_EVP && ERR_GET_REASON(errnum) == EVP_R_BAD_DECRYPT)) {
(ERR_GET_LIB(errnum) == ERR_LIB_EVP && ERR_GET_REASON(errnum) == EVP_R_BAD_DECRYPT)) {
TraceEvent(SevError, "FDBLibTLSIncorrectPassword");
} else {
ERR_error_string_n(errnum, errbuf, sizeof(errbuf));
@ -248,7 +261,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
TraceEvent(SevError, "FDBLibTLSOutOfMemory");
goto err;
}
if (tls_config_set_key_mem(tls_cfg, (const uint8_t *)data, len) == -1) {
if (tls_config_set_key_mem(tls_cfg, (const uint8_t*)data, len) == -1) {
TraceEvent(SevError, "FDBLibTLSKeyError").detail("LibTLSErrorMessage", tls_config_error(tls_cfg));
goto err;
}
@ -262,7 +275,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
key_data_set = true;
rc = true;
err:
err:
BIO_free(bio);
EVP_PKEY_free(key);
return rc;
@ -287,20 +300,20 @@ bool FDBLibTLSPolicy::set_verify_peers(int count, const uint8_t* verify_peers[],
try {
std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]);
int start = 0;
while(start < verifyString.size()) {
while (start < verifyString.size()) {
int split = verifyString.find('|', start);
if(split == std::string::npos) {
if (split == std::string::npos) {
break;
}
if(split == start || verifyString[split-1] != '\\') {
if (split == start || verifyString[split - 1] != '\\') {
auto verify = makeReference<FDBLibTLSVerify>(verifyString.substr(start, split - start));
verify_rules.push_back(verify);
start = split+1;
start = split + 1;
}
}
auto verify = makeReference<FDBLibTLSVerify>(verifyString.substr(start));
verify_rules.push_back(verify);
} catch ( const std::runtime_error& ) {
} catch (const std::runtime_error&) {
verify_rules.clear();
std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]);
TraceEvent(SevError, "FDBLibTLSVerifyPeersParseError").detail("Config", verifyString);

10
FDBLibTLS/FDBLibTLSPolicy.h
View File

@ -32,7 +32,7 @@
#include <string>
#include <vector>
struct FDBLibTLSPolicy: ITLSPolicy, ReferenceCounted<FDBLibTLSPolicy> {
struct FDBLibTLSPolicy : ITLSPolicy, ReferenceCounted<FDBLibTLSPolicy> {
FDBLibTLSPolicy(Reference<FDBLibTLSPlugin> plugin);
virtual ~FDBLibTLSPolicy();
@ -41,7 +41,13 @@ struct FDBLibTLSPolicy: ITLSPolicy, ReferenceCounted<FDBLibTLSPolicy> {
Reference<FDBLibTLSPlugin> plugin;
virtual ITLSSession* create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid);
virtual ITLSSession* create_session(bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid);
struct stack_st_X509* parse_cert_pem(const uint8_t* cert_pem, size_t cert_pem_len);
void parse_verify(std::string input);

109
FDBLibTLS/FDBLibTLSSession.cpp
View File

@ -36,11 +36,10 @@
#include <string.h>
#include <limits.h>
static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb_arg)
{
FDBLibTLSSession *session = (FDBLibTLSSession *)cb_arg;
static ssize_t tls_read_func(struct tls* ctx, void* buf, size_t buflen, void* cb_arg) {
FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg;
int rv = session->recv_func(session->recv_ctx, (uint8_t *)buf, buflen);
int rv = session->recv_func(session->recv_ctx, (uint8_t*)buf, buflen);
if (rv < 0)
return 0;
if (rv == 0)
@ -48,11 +47,10 @@ static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb
return (ssize_t)rv;
}
static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, void *cb_arg)
{
FDBLibTLSSession *session = (FDBLibTLSSession *)cb_arg;
static ssize_t tls_write_func(struct tls* ctx, const void* buf, size_t buflen, void* cb_arg) {
FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg;
int rv = session->send_func(session->send_ctx, (const uint8_t *)buf, buflen);
int rv = session->send_func(session->send_ctx, (const uint8_t*)buf, buflen);
if (rv < 0)
return 0;
if (rv == 0)
@ -60,13 +58,18 @@ static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, v
return (ssize_t)rv;
}
FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_client, const char* servername,
TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func,
void* recv_ctx, void* uidptr)
FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy,
bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uidptr)
: tls_ctx(nullptr), tls_sctx(nullptr), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx),
recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) {
if (uidptr)
uid = * (UID*) uidptr;
uid = *(UID*)uidptr;
if (is_client) {
if ((tls_ctx = tls_client()) == nullptr) {
@ -116,7 +119,8 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match
unsigned char* entry_utf8 = nullptr;
int entry_utf8_len = 0;
if ((asn_criteria = ASN1_IA5STRING_new()) == nullptr) goto err;
if ((asn_criteria = ASN1_IA5STRING_new()) == nullptr)
goto err;
if (ASN1_STRING_set(asn_criteria, criteria.c_str(), criteria.size()) != 1)
goto err;
if ((criteria_utf8_len = ASN1_STRING_to_UTF8(&criteria_utf8, asn_criteria)) < 1)
@ -124,12 +128,10 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match
if ((entry_utf8_len = ASN1_STRING_to_UTF8(&entry_utf8, entry)) < 1)
goto err;
if (mt == MatchType::EXACT) {
if (criteria_utf8_len == entry_utf8_len &&
memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
if (criteria_utf8_len == entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
rc = true;
} else if (mt == MatchType::PREFIX) {
if (criteria_utf8_len <= entry_utf8_len &&
memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
if (criteria_utf8_len <= entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
rc = true;
} else if (mt == MatchType::SUFFIX) {
if (criteria_utf8_len <= entry_utf8_len &&
@ -137,15 +139,15 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match
rc = true;
}
err:
err:
ASN1_STRING_free(asn_criteria);
free(criteria_utf8);
free(entry_utf8);
return rc;
}
bool match_name_criteria(X509_NAME *name, NID nid, const std::string& criteria, MatchType mt) {
X509_NAME_ENTRY *name_entry;
bool match_name_criteria(X509_NAME* name, NID nid, const std::string& criteria, MatchType mt) {
X509_NAME_ENTRY* name_entry;
int idx;
// If name does not exist, or has multiple of this RDN, refuse to proceed.
@ -153,12 +155,13 @@ bool match_name_criteria(X509_NAME *name, NID nid, const std::string& criteria,
return false;
if (X509_NAME_get_index_by_NID(name, nid, idx) != -1)
return false;
if ((name_entry = X509_NAME_get_entry(name, idx)) == nullptr) return false;
if ((name_entry = X509_NAME_get_entry(name, idx)) == nullptr)
return false;
return match_criteria_entry(criteria, name_entry->value, mt);
}
bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, MatchType mt) {
bool match_extension_criteria(X509* cert, NID nid, const std::string& value, MatchType mt) {
if (nid != NID_subject_alt_name && nid != NID_issuer_alt_name) {
// I have no idea how other extensions work.
return false;
@ -168,29 +171,27 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
return false;
}
std::string value_gen = value.substr(0, pos);
std::string value_val = value.substr(pos+1, value.npos);
std::string value_val = value.substr(pos + 1, value.npos);
STACK_OF(GENERAL_NAME)* sans =
reinterpret_cast<STACK_OF(GENERAL_NAME)*>(X509_get_ext_d2i(cert, nid, nullptr, nullptr));
if (sans == nullptr) {
return false;
}
int num_sans = sk_GENERAL_NAME_num( sans );
int num_sans = sk_GENERAL_NAME_num(sans);
bool rc = false;
for( int i = 0; i < num_sans && !rc; ++i ) {
GENERAL_NAME* altname = sk_GENERAL_NAME_value( sans, i );
for (int i = 0; i < num_sans && !rc; ++i) {
GENERAL_NAME* altname = sk_GENERAL_NAME_value(sans, i);
std::string matchable;
switch (altname->type) {
case GEN_OTHERNAME:
break;
case GEN_EMAIL:
if (value_gen == "EMAIL" &&
match_criteria_entry( value_val, altname->d.rfc822Name, mt)) {
if (value_gen == "EMAIL" && match_criteria_entry(value_val, altname->d.rfc822Name, mt)) {
rc = true;
break;
}
case GEN_DNS:
if (value_gen == "DNS" &&
match_criteria_entry( value_val, altname->d.dNSName, mt )) {
if (value_gen == "DNS" && match_criteria_entry(value_val, altname->d.dNSName, mt)) {
rc = true;
break;
}
@ -199,14 +200,12 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
case GEN_EDIPARTY:
break;
case GEN_URI:
if (value_gen == "URI" &&
match_criteria_entry( value_val, altname->d.uniformResourceIdentifier, mt )) {
if (value_gen == "URI" && match_criteria_entry(value_val, altname->d.uniformResourceIdentifier, mt)) {
rc = true;
break;
}
case GEN_IPADD:
if (value_gen == "IP" &&
match_criteria_entry( value_val, altname->d.iPAddress, mt )) {
if (value_gen == "IP" && match_criteria_entry(value_val, altname->d.iPAddress, mt)) {
rc = true;
break;
}
@ -218,8 +217,13 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
return rc;
}
bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string& criteria, MatchType mt, X509Location loc) {
switch(loc) {
bool match_criteria(X509* cert,
X509_NAME* subject,
NID nid,
const std::string& criteria,
MatchType mt,
X509Location loc) {
switch (loc) {
case X509Location::NAME: {
return match_name_criteria(subject, nid, criteria, mt);
}
@ -231,7 +235,8 @@ bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string&
return false;
}
std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs) {
std::tuple<bool, std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify,
struct stack_st_X509* certs) {
X509_STORE_CTX* store_ctx = nullptr;
X509_NAME *subject, *issuer;
bool rc = false;
@ -258,7 +263,7 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
if (!verify->verify_time)
X509_VERIFY_PARAM_set_flags(X509_STORE_CTX_get0_param(store_ctx), X509_V_FLAG_NO_CHECK_TIME);
if (X509_verify_cert(store_ctx) <= 0) {
const char *errstr = X509_verify_cert_error_string(X509_STORE_CTX_get_error(store_ctx));
const char* errstr = X509_verify_cert_error_string(X509_STORE_CTX_get_error(store_ctx));
reason = "Verify cert error: " + std::string(errstr);
goto err;
}
@ -269,8 +274,9 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
reason = "Cert subject error";
goto err;
}
for (auto &pair: verify->subject_criteria) {
if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
for (auto& pair : verify->subject_criteria) {
if (!match_criteria(
cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Cert subject match failure";
goto err;
}
@ -281,8 +287,9 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
reason = "Cert issuer error";
goto err;
}
for (auto &pair: verify->issuer_criteria) {
if (!match_criteria(cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
for (auto& pair : verify->issuer_criteria) {
if (!match_criteria(
cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Cert issuer match failure";
goto err;
}
@ -294,8 +301,9 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
reason = "Root subject error";
goto err;
}
for (auto &pair: verify->root_criteria) {
if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
for (auto& pair : verify->root_criteria) {
if (!match_criteria(
cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Root subject match failure";
goto err;
}
@ -304,7 +312,7 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
// If we got this far, everything checked out...
rc = true;
err:
err:
X509_STORE_CTX_free(store_ctx);
return std::make_tuple(rc, reason);
@ -312,7 +320,7 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
bool FDBLibTLSSession::verify_peer() {
struct stack_st_X509* certs = nullptr;
const uint8_t *cert_pem;
const uint8_t* cert_pem;
size_t cert_pem_len;
bool rc = false;
std::set<std::string> verify_failure_reasons;
@ -328,10 +336,11 @@ bool FDBLibTLSSession::verify_peer() {
TraceEvent(SevError, "FDBLibTLSNoCertError", uid);
goto err;
}
if ((certs = policy->parse_cert_pem(cert_pem, cert_pem_len)) == nullptr) goto err;
if ((certs = policy->parse_cert_pem(cert_pem, cert_pem_len)) == nullptr)
goto err;
// Any matching rule is sufficient.
for (auto &verify_rule: policy->verify_rules) {
for (auto& verify_rule : policy->verify_rules) {
std::tie(verify_success, verify_failure_reason) = check_verify(verify_rule, certs);
if (verify_success) {
rc = true;
@ -344,7 +353,7 @@ bool FDBLibTLSSession::verify_peer() {
if (!rc) {
// log the various failure reasons
if(now() - lastVerifyFailureLogged > 1.0) {
if (now() - lastVerifyFailureLogged > 1.0) {
for (std::string reason : verify_failure_reasons) {
lastVerifyFailureLogged = now();
TraceEvent("FDBLibTLSVerifyFailure", uid).suppressFor(1.0).detail("Reason", reason);
@ -352,7 +361,7 @@ bool FDBLibTLSSession::verify_peer() {
}
}
err:
err:
sk_X509_pop_free(certs, X509_free);
return rc;

15
FDBLibTLS/FDBLibTLSSession.h
View File

@ -33,14 +33,21 @@
#include <tls.h>
struct FDBLibTLSSession : ITLSSession, ReferenceCounted<FDBLibTLSSession> {
FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid);
FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy,
bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid);
virtual ~FDBLibTLSSession();
virtual void addref() { ReferenceCounted<FDBLibTLSSession>::addref(); }
virtual void delref() { ReferenceCounted<FDBLibTLSSession>::delref(); }
bool verify_peer();
std::tuple<bool,std::string> check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs);
std::tuple<bool, std::string> check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509* certs);
virtual int handshake();
virtual int read(uint8_t* data, int length);
@ -50,8 +57,8 @@ struct FDBLibTLSSession : ITLSSession, ReferenceCounted<FDBLibTLSSession> {
bool is_client;
struct tls *tls_ctx;
struct tls *tls_sctx;
struct tls* tls_ctx;
struct tls* tls_sctx;
TLSSendCallbackFunc send_func;
void* send_ctx;

43
FDBLibTLS/FDBLibTLSVerify.cpp
View File

@ -43,24 +43,24 @@ static int hexValue(char c) {
static std::string de4514(std::string const& input, int start, int& out_end) {
std::string output;
if(input[start] == '#' || input[start] == ' ') {
if (input[start] == '#' || input[start] == ' ') {
out_end = start;
return output;
}
int space_count = 0;
for(int p = start; p < input.size();) {
switch(input[p]) {
for (int p = start; p < input.size();) {
switch (input[p]) {
case '\\': // Handle escaped sequence
// Backslash escaping nothing!
if(p == input.size() - 1) {
if (p == input.size() - 1) {
out_end = p;
goto FIN;
}
switch(input[p+1]) {
switch (input[p + 1]) {
case ' ':
case '"':
case '#':
@ -72,24 +72,24 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
case '>':
case '|':
case '\\':
output += input[p+1];
output += input[p + 1];
p += 2;
space_count = 0;
continue;
default:
// Backslash escaping pair of hex digits requires two characters
if(p == input.size() - 2) {
if (p == input.size() - 2) {
out_end = p;
goto FIN;
}
try {
output += hexValue(input[p+1]) * 16 + hexValue(input[p+2]);
output += hexValue(input[p + 1]) * 16 + hexValue(input[p + 2]);
p += 3;
space_count = 0;
continue;
} catch( ... ) {
} catch (...) {
out_end = p;
goto FIN;
}
@ -109,7 +109,7 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
default:
// Character is what it is
output += input[p];
if(input[p] == ' ')
if (input[p] == ' ')
space_count++;
else
space_count = 0;
@ -119,7 +119,7 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
out_end = input.size();
FIN:
FIN:
out_end -= space_count;
output.resize(output.size() - space_count);
@ -128,16 +128,17 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
static std::pair<std::string, std::string> splitPair(std::string const& input, char c) {
int p = input.find_first_of(c);
if(p == input.npos) {
if (p == input.npos) {
throw std::runtime_error("splitPair");
}
return std::make_pair(input.substr(0, p), input.substr(p+1, input.size()));
return std::make_pair(input.substr(0, p), input.substr(p + 1, input.size()));
}
static NID abbrevToNID(std::string const& sn) {
NID nid = NID_undef;
if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" || sn == "subjectAltName")
if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" ||
sn == "subjectAltName")
nid = OBJ_sn2nid(sn.c_str());
if (nid == NID_undef)
throw std::runtime_error("abbrevToNID");
@ -158,13 +159,11 @@ static X509Location locationForNID(NID nid) {
}
}
FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config):
verify_cert(true), verify_time(true) {
FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config) : verify_cert(true), verify_time(true) {
parse_verify(verify_config);
}
FDBLibTLSVerify::~FDBLibTLSVerify() {
}
FDBLibTLSVerify::~FDBLibTLSVerify() {}
void FDBLibTLSVerify::parse_verify(std::string input) {
int s = 0;
@ -176,8 +175,10 @@ void FDBLibTLSVerify::parse_verify(std::string input) {
throw std::runtime_error("parse_verify");
MatchType mt = MatchType::EXACT;
if (input[eq-1] == '>') mt = MatchType::PREFIX;
if (input[eq-1] == '<') mt = MatchType::SUFFIX;
if (input[eq - 1] == '>')
mt = MatchType::PREFIX;
if (input[eq - 1] == '<')
mt = MatchType::SUFFIX;
std::string term = input.substr(s, eq - s - (mt == MatchType::EXACT ? 0 : 1));
if (term.find("Check.") == 0) {
@ -206,7 +207,7 @@ void FDBLibTLSVerify::parse_verify(std::string input) {
s = eq + 3;
} else {
std::map< int, Criteria >* criteria = &subject_criteria;
std::map<int, Criteria>* criteria = &subject_criteria;
if (term.find('.') != term.npos) {
auto scoped = splitPair(term, '.');

20
FDBLibTLS/FDBLibTLSVerify.h
View File

@ -47,14 +47,10 @@ enum class X509Location {
};
struct Criteria {
Criteria( const std::string& s )
: criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {}
Criteria( const std::string& s, MatchType mt )
: criteria(s), match_type(mt), location(X509Location::NAME) {}
Criteria( const std::string& s, X509Location loc)
: criteria(s), match_type(MatchType::EXACT), location(loc) {}
Criteria( const std::string& s, MatchType mt, X509Location loc)
: criteria(s), match_type(mt), location(loc) {}
Criteria(const std::string& s) : criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {}
Criteria(const std::string& s, MatchType mt) : criteria(s), match_type(mt), location(X509Location::NAME) {}
Criteria(const std::string& s, X509Location loc) : criteria(s), match_type(MatchType::EXACT), location(loc) {}
Criteria(const std::string& s, MatchType mt, X509Location loc) : criteria(s), match_type(mt), location(loc) {}
std::string criteria;
MatchType match_type;
@ -65,7 +61,7 @@ struct Criteria {
}
};
struct FDBLibTLSVerify: ReferenceCounted<FDBLibTLSVerify> {
struct FDBLibTLSVerify : ReferenceCounted<FDBLibTLSVerify> {
FDBLibTLSVerify(std::string verify);
virtual ~FDBLibTLSVerify();
@ -77,9 +73,9 @@ struct FDBLibTLSVerify: ReferenceCounted<FDBLibTLSVerify> {
bool verify_cert;
bool verify_time;
std::map< NID, Criteria > subject_criteria;
std::map< NID, Criteria > issuer_criteria;
std::map< NID, Criteria > root_criteria;
std::map<NID, Criteria> subject_criteria;
std::map<NID, Criteria> issuer_criteria;
std::map<NID, Criteria> root_criteria;
};
#endif /* FDB_LIBTLS_VERIFY_H */

1138
FDBLibTLS/plugin-test.cpp
View File

File diff suppressed because it is too large Load Diff

140
FDBLibTLS/verify-test.cpp
View File

@ -33,11 +33,18 @@
#include "FDBLibTLS/FDBLibTLSPolicy.h"
struct FDBLibTLSVerifyTest {
FDBLibTLSVerifyTest(std::string input):
input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}), root_criteria({}) {};
FDBLibTLSVerifyTest(std::string input, bool verify_cert, bool verify_time, std::map<int, Criteria> subject, std::map<int, Criteria> issuer, std::map<int, Criteria> root):
input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject), issuer_criteria(issuer), root_criteria(root) {};
~FDBLibTLSVerifyTest() {};
FDBLibTLSVerifyTest(std::string input)
: input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}),
root_criteria({}){};
FDBLibTLSVerifyTest(std::string input,
bool verify_cert,
bool verify_time,
std::map<int, Criteria> subject,
std::map<int, Criteria> issuer,
std::map<int, Criteria> root)
: input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject),
issuer_criteria(issuer), root_criteria(root){};
~FDBLibTLSVerifyTest(){};
int run();
@ -54,20 +61,21 @@ struct FDBLibTLSVerifyTest {
static std::string criteriaToString(std::map<int, Criteria> const& criteria) {
std::string s;
for (auto &pair: criteria) {
s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " + boost::lexical_cast<std::string>((int)pair.second.match_type) + ", " + boost::lexical_cast<std::string>((int)pair.second.location) + ")}";
for (auto& pair : criteria) {
s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " +
boost::lexical_cast<std::string>((int)pair.second.match_type) + ", " +
boost::lexical_cast<std::string>((int)pair.second.location) + ")}";
}
return "{" + s + "}";
}
static void logf(const char* event, void* uid, bool is_error, ...) {
}
static void logf(const char* event, void* uid, bool is_error, ...) {}
int FDBLibTLSVerifyTest::run() {
Reference<FDBLibTLSVerify> verify;
try {
verify = makeReference<FDBLibTLSVerify>(input);
} catch ( const std::runtime_error& e ) {
} catch (const std::runtime_error& e) {
if (valid) {
std::cerr << "FAIL: Verify test failed, but should have succeeded - '" << input << "'\n";
return 1;
@ -87,15 +95,18 @@ int FDBLibTLSVerifyTest::run() {
return 1;
}
if (verify->subject_criteria != subject_criteria) {
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want " << criteriaToString(subject_criteria) << "\n";
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want "
<< criteriaToString(subject_criteria) << "\n";
return 1;
}
if (verify->issuer_criteria != issuer_criteria) {
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want " << criteriaToString(issuer_criteria) << "\n";
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want "
<< criteriaToString(issuer_criteria) << "\n";
return 1;
}
if (verify->root_criteria != root_criteria) {
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want " << criteriaToString(root_criteria) << "\n";
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want "
<< criteriaToString(root_criteria) << "\n";
return 1;
}
return 0;
@ -105,7 +116,7 @@ static int policy_verify_test() {
auto plugin = makeReference<FDBLibTLSPlugin>();
auto policy = makeReference<FDBLibTLSPolicy>(plugin, (ITLSLogFunc)logf);
const char *verify_peers[] = {
const char* verify_peers[] = {
"S.CN=abc",
"I.CN=def",
"R.CN=xyz,Check.Unexpired=0",
@ -121,7 +132,7 @@ static int policy_verify_test() {
makeReference<FDBLibTLSVerify>(std::string(verify_peers[2], verify_peers_len[2])),
};
if (!policy->set_verify_peers(3, (const uint8_t **)verify_peers, verify_peers_len)) {
if (!policy->set_verify_peers(3, (const uint8_t**)verify_peers, verify_peers_len)) {
std::cerr << "FAIL: Policy verify test failed, but should have succeeded\n";
return 1;
}
@ -131,25 +142,30 @@ static int policy_verify_test() {
}
int i = 0;
for (auto &verify_rule: policy->verify_rules) {
for (auto& verify_rule : policy->verify_rules) {
if (verify_rule->verify_cert != verify_rules[i]->verify_cert) {
std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want " << verify_rules[i]->verify_cert << "\n";
std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want "
<< verify_rules[i]->verify_cert << "\n";
return 1;
}
if (verify_rule->verify_time != verify_rules[i]->verify_time) {
std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want " << verify_rules[i]->verify_time << "\n";
std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want "
<< verify_rules[i]->verify_time << "\n";
return 1;
}
if (verify_rule->subject_criteria != verify_rules[i]->subject_criteria) {
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want " << criteriaToString(verify_rules[i]->subject_criteria) << "\n";
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want "
<< criteriaToString(verify_rules[i]->subject_criteria) << "\n";
return 1;
}
if (verify_rule->issuer_criteria != verify_rules[i]->issuer_criteria) {
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want " << criteriaToString(verify_rules[i]->issuer_criteria) << "\n";
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want "
<< criteriaToString(verify_rules[i]->issuer_criteria) << "\n";
return 1;
}
if (verify_rule->root_criteria != verify_rules[i]->root_criteria) {
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want " << criteriaToString(verify_rules[i]->root_criteria) << "\n";
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want "
<< criteriaToString(verify_rules[i]->root_criteria) << "\n";
return 1;
}
i++;
@ -157,8 +173,7 @@ static int policy_verify_test() {
return 0;
}
int main(int argc, char **argv)
{
int main(int argc, char** argv) {
int failed = 0;
#define EXACT(x) Criteria(x, MatchType::EXACT, X509Location::NAME)
@ -173,29 +188,60 @@ int main(int argc, char **argv)
FDBLibTLSVerifyTest("Check.Unexpired=0", true, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Valid=1,Check.Unexpired=0", true, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,Check.Valid=0", false, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {}, {{NID_countryName, EXACT("US")}}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC", true, true,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc", true, true,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}},
{{NID_countryName, EXACT("US")}},
{{NID_commonName, EXACT("abc")}}),
FDBLibTLSVerifyTest("C=\\,S=abc", true, true, {{NID_countryName, EXACT(",S=abc")}}, {}, {}),
FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, {{NID_commonName, EXACT("abc")}}, {}, {}),
FDBLibTLSVerifyTest("CN=a\\62c", true, true, {{NID_commonName, EXACT("abc")}}, {}, {}),
FDBLibTLSVerifyTest("CN=a\\01c", true, true, {{NID_commonName, EXACT("a\001c")}}, {}, {}),
FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp", true, true, {{NID_subject_alt_name, {"XYZCorp", MatchType::EXACT, X509Location::EXTENSION}}}, {}, {}),
FDBLibTLSVerifyTest("S.O>=XYZ", true, true, {{NID_organizationName, PREFIX("XYZ")}}, {}, {}),
FDBLibTLSVerifyTest("S.O<=LLC", true, true, {{NID_organizationName, SUFFIX("LLC")}}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } },
{},
{ { NID_countryName, EXACT("US") } }),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC",
true,
true,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc",
true,
true,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{ { NID_commonName, EXACT("abc") } }),
FDBLibTLSVerifyTest("C=\\,S=abc", true, true, { { NID_countryName, EXACT(",S=abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=a\\62c", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=a\\01c", true, true, { { NID_commonName, EXACT("a\001c") } }, {}, {}),
FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp",
true,
true,
{ { NID_subject_alt_name, { "XYZCorp", MatchType::EXACT, X509Location::EXTENSION } } },
{},
{}),
FDBLibTLSVerifyTest("S.O>=XYZ", true, true, { { NID_organizationName, PREFIX("XYZ") } }, {}, {}),
FDBLibTLSVerifyTest("S.O<=LLC", true, true, { { NID_organizationName, SUFFIX("LLC") } }, {}, {}),
// Invalid cases.
FDBLibTLSVerifyTest("Check.Invalid=0"),
@ -212,7 +258,7 @@ int main(int argc, char **argv)
#undef PREFIX
#undef SUFFIX
for (auto &test: tests)
for (auto& test : tests)
failed |= test.run();
failed |= policy_verify_test();

10
bindings/c/ThreadCleanup.cpp
View File

@ -25,17 +25,17 @@
#include <Windows.h>
BOOL WINAPI DllMain( HINSTANCE dll, DWORD reason, LPVOID reserved ) {
BOOL WINAPI DllMain(HINSTANCE dll, DWORD reason, LPVOID reserved) {
if (reason == DLL_THREAD_DETACH)
releaseAllThreadMagazines();
return TRUE;
}
#elif defined( __unixish__ )
#elif defined(__unixish__)
#ifdef __INTEL_COMPILER
#pragma warning ( disable:2415 )
#pragma warning(disable : 2415)
#endif
static pthread_key_t threadDestructorKey;
@ -45,13 +45,13 @@ static void threadDestructor(void*) {
}
void registerThread() {
pthread_setspecific( threadDestructorKey, (const void*)1 );
pthread_setspecific(threadDestructorKey, (const void*)1);
}
static int initThreadDestructorKey() {
if (!pthread_key_create(&threadDestructorKey, &threadDestructor)) {
registerThread();
setFastAllocatorThreadInitFunction( &registerThread );
setFastAllocatorThreadInitFunction(&registerThread);
}
return 0;

786
bindings/c/fdb_c.cpp
View File

File diff suppressed because it is too large Load Diff

3
bindings/c/foundationdb/ClientWorkload.h
View File

@ -44,7 +44,8 @@ enum class FDBSeverity { Debug, Info, Warn, WarnAlways, Error };
class FDBLogger {
public:
virtual void trace(FDBSeverity sev, const std::string& name,
virtual void trace(FDBSeverity sev,
const std::string& name,
const std::vector<std::pair<std::string, std::string>>& details) = 0;
};

354
bindings/c/test/mako/mako.c
View File

@ -101,8 +101,13 @@ int commit_transaction(FDBTransaction* transaction) {
return FDB_SUCCESS;
}
void update_op_lat_stats(struct timespec* start, struct timespec* end, int op, mako_stats_t* stats,
lat_block_t* block[], int* elem_size, bool* is_memory_allocated) {
void update_op_lat_stats(struct timespec* start,
struct timespec* end,
int op,
mako_stats_t* stats,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
uint64_t latencyus;
latencyus = (((uint64_t)end->tv_sec * 1000000000 + end->tv_nsec) -
@ -116,7 +121,8 @@ void update_op_lat_stats(struct timespec* start, struct timespec* end, int op, m
if (latencyus > stats->latency_us_max[op]) {
stats->latency_us_max[op] = latencyus;
}
if (!is_memory_allocated[op]) return;
if (!is_memory_allocated[op])
return;
if (elem_size[op] < stats->latency_samples[op]) {
elem_size[op] = elem_size[op] + LAT_BLOCK_SIZE;
lat_block_t* temp_block = (lat_block_t*)malloc(sizeof(lat_block_t));
@ -157,11 +163,13 @@ int cleanup(FDBTransaction* transaction, mako_args_t* args) {
endstr[4] = 0xff;
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start);
fdb_transaction_clear_range(transaction, (uint8_t*)beginstr, 5, (uint8_t*)endstr, 5);
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit;
if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
fdb_transaction_reset(transaction);
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_end);
fprintf(printme, "INFO: Clear range: %6.3f sec\n",
fprintf(printme,
"INFO: Clear range: %6.3f sec\n",
((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) /
1000000000);
return 0;
@ -172,8 +180,15 @@ failExit:
}
/* populate database */
int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int thread_id, int thread_tps,
mako_stats_t* stats, lat_block_t* block[], int* elem_size, bool* is_memory_allocated) {
int populate(FDBTransaction* transaction,
mako_args_t* args,
int worker_id,
int thread_id,
int thread_tps,
mako_stats_t* stats,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
int i;
struct timespec timer_start, timer_end;
struct timespec timer_prev, timer_now; /* for throttling */
@ -188,7 +203,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
int tracetimer = 0;
keystr = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr) return -1;
if (!keystr)
return -1;
valstr = (char*)malloc(sizeof(char) * args->value_length + 1);
if (!valstr) {
free(keystr);
@ -226,8 +242,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
fdb_error_t err;
tracetimer = 0;
fprintf(debugme, "DEBUG: txn tracing %s\n", keystr);
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
(uint8_t*)keystr, strlen(keystr));
err = fdb_transaction_set_option(
transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, (uint8_t*)keystr, strlen(keystr));
if (err) {
fprintf(
stderr,
@ -236,7 +252,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
}
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0);
if (err) {
fprintf(stderr, "ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n",
fprintf(stderr,
"ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n",
fdb_get_error(err));
}
}
@ -259,14 +276,20 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_start_commit);
}
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit;
if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
/* xact latency stats */
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size,
is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size,
update_op_lat_stats(
&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start,
&timer_per_xact_end,
OP_TRANSACTION,
stats,
block,
elem_size,
is_memory_allocated);
}
@ -283,21 +306,26 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_start_commit);
}
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit;
if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
/* xact latency stats */
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size,
is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size,
is_memory_allocated);
update_op_lat_stats(
&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, is_memory_allocated);
update_op_lat_stats(
&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, is_memory_allocated);
}
clock_gettime(CLOCK_MONOTONIC, &timer_end);
stats->xacts++;
fprintf(debugme, "DEBUG: Populated %d rows (%d-%d): %6.3f sec\n", end - begin, begin, end,
fprintf(debugme,
"DEBUG: Populated %d rows (%d-%d): %6.3f sec\n",
end - begin,
begin,
end,
((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) /
1000000000);
@ -306,8 +334,10 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
return 0;
failExit:
if (keystr) free(keystr);
if (valstr) free(valstr);
if (keystr)
free(keystr);
if (valstr)
free(valstr);
fprintf(stderr, "ERROR: FDB failure in populate()\n");
return -1;
}
@ -365,17 +395,28 @@ int run_op_get(FDBTransaction* transaction, char* keystr, char* valstr, int snap
return FDB_SUCCESS;
}
int run_op_getrange(FDBTransaction* transaction, char* keystr, char* keystr2, char* valstr, int snapshot, int reverse, FDBStreamingMode streaming_mode) {
int run_op_getrange(FDBTransaction* transaction,
char* keystr,
char* keystr2,
char* valstr,
int snapshot,
int reverse,
FDBStreamingMode streaming_mode) {
FDBFuture* f;
fdb_error_t err;
FDBKeyValue const* out_kv;
int out_count;
int out_more;
f = fdb_transaction_get_range(transaction, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)),
FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1, 0 /* limit */,
0 /* target_bytes */, streaming_mode /* FDBStreamingMode */,
0 /* iteration */, snapshot, reverse /* reverse */);
f = fdb_transaction_get_range(transaction,
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)),
FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1,
0 /* limit */,
0 /* target_bytes */,
streaming_mode /* FDBStreamingMode */,
0 /* iteration */,
snapshot,
reverse /* reverse */);
fdb_wait_and_handle_error(fdb_transaction_get_range, f, transaction);
err = fdb_future_get_keyvalue_array(f, &out_kv, &out_count, &out_more);
@ -428,8 +469,15 @@ int run_op_clearrange(FDBTransaction* transaction, char* keystr, char* keystr2)
}
/* run one transaction */
int run_one_transaction(FDBTransaction* transaction, mako_args_t* args, mako_stats_t* stats, char* keystr,
char* keystr2, char* valstr, lat_block_t* block[], int* elem_size, bool* is_memory_allocated) {
int run_one_transaction(FDBTransaction* transaction,
mako_args_t* args,
mako_stats_t* stats,
char* keystr,
char* keystr2,
char* valstr,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
int i;
int count;
int rc;
@ -488,13 +536,25 @@ retryTxn:
rc = run_op_get(transaction, keystr, valstr, 0);
break;
case OP_GETRANGE:
rc = run_op_getrange(transaction, keystr, keystr2, valstr, 0, args->txnspec.ops[i][OP_REVERSE], args->streaming_mode);
rc = run_op_getrange(transaction,
keystr,
keystr2,
valstr,
0,
args->txnspec.ops[i][OP_REVERSE],
args->streaming_mode);
break;
case OP_SGET:
rc = run_op_get(transaction, keystr, valstr, 1);
break;
case OP_SGETRANGE:
rc = run_op_getrange(transaction, keystr, keystr2, valstr, 1, args->txnspec.ops[i][OP_REVERSE], args->streaming_mode);
rc = run_op_getrange(transaction,
keystr,
keystr2,
valstr,
1,
args->txnspec.ops[i][OP_REVERSE],
args->streaming_mode);
break;
case OP_UPDATE:
randstr(valstr, args->value_length + 1);
@ -512,10 +572,13 @@ retryTxn:
randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */
randstr(valstr, args->value_length + 1);
for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) {
sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]),
sprintf(keystr + KEYPREFIXLEN + randstrlen,
"%0.*d",
digits(args->txnspec.ops[i][OP_RANGE]),
rangei);
rc = run_op_insert(transaction, keystr, valstr);
if (rc != FDB_SUCCESS) break;
if (rc != FDB_SUCCESS)
break;
}
docommit = 1;
break;
@ -539,10 +602,20 @@ retryTxn:
stats->ops[OP_TRANSACTION]++;
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block,
elem_size, is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats,
block, elem_size, is_memory_allocated);
update_op_lat_stats(&timer_start_commit,
&timer_per_xact_end,
OP_COMMIT,
stats,
block,
elem_size,
is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start,
&timer_per_xact_end,
OP_TRANSACTION,
stats,
block,
elem_size,
is_memory_allocated);
}
} else {
/* error */
@ -572,7 +645,9 @@ retryTxn:
randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */
randstr(valstr, args->value_length + 1);
for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) {
sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]),
sprintf(keystr + KEYPREFIXLEN + randstrlen,
"%0.*d",
digits(args->txnspec.ops[i][OP_RANGE]),
rangei);
if (rangei == 0) {
strcpy(keystr2, keystr);
@ -598,10 +673,20 @@ retryTxn:
stats->ops[OP_TRANSACTION]++;
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block,
elem_size, is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats,
block, elem_size, is_memory_allocated);
update_op_lat_stats(&timer_start_commit,
&timer_per_xact_end,
OP_COMMIT,
stats,
block,
elem_size,
is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start,
&timer_per_xact_end,
OP_TRANSACTION,
stats,
block,
elem_size,
is_memory_allocated);
}
} else {
/* error */
@ -666,8 +751,8 @@ retryTxn:
stats->ops[OP_COMMIT]++;
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size,
is_memory_allocated);
update_op_lat_stats(
&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, is_memory_allocated);
}
} else {
/* error */
@ -688,8 +773,8 @@ retryTxn:
stats->ops[OP_TRANSACTION]++;
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size,
is_memory_allocated);
update_op_lat_stats(
&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, is_memory_allocated);
}
stats->xacts++;
@ -699,9 +784,18 @@ retryTxn:
return 0;
}
int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps, volatile double* throttle_factor,
int thread_iters, volatile int* signal, mako_stats_t* stats, int dotrace, int dotagging, lat_block_t* block[],
int* elem_size, bool* is_memory_allocated) {
int run_workload(FDBTransaction* transaction,
mako_args_t* args,
int thread_tps,
volatile double* throttle_factor,
int thread_iters,
volatile int* signal,
mako_stats_t* stats,
int dotrace,
int dotagging,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
int xacts = 0;
int64_t total_xacts = 0;
int rc = 0;
@ -714,13 +808,14 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
int tracetimer = 0;
char* tagstr;
if (thread_tps < 0) return 0;
if (thread_tps < 0)
return 0;
if (dotrace) {
traceid = (char*)malloc(32);
}
if(dotagging) {
if (dotagging) {
tagstr = (char*)calloc(16, 1);
memcpy(tagstr, KEYPREFIX, KEYPREFIXLEN);
memcpy(tagstr + KEYPREFIXLEN, args->txntagging_prefix, TAGPREFIXLENGTH_MAX);
@ -729,7 +824,8 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
current_tps = (int)((double)thread_tps * *throttle_factor);
keystr = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr) return -1;
if (!keystr)
return -1;
keystr2 = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr2) {
free(keystr);
@ -770,11 +866,13 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
tracetimer = 0;
snprintf(traceid, 32, "makotrace%019lld", total_xacts);
fprintf(debugme, "DEBUG: txn tracing %s\n", traceid);
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
(uint8_t*)traceid, strlen(traceid));
err = fdb_transaction_set_option(transaction,
FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
(uint8_t*)traceid,
strlen(traceid));
if (err) {
fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n",
fdb_get_error(err));
fprintf(
stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fdb_get_error(err));
}
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0);
if (err) {
@ -783,7 +881,6 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
}
}
} else {
if (thread_tps > 0) {
/* 1 second not passed, throttle */
@ -796,16 +893,15 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
/* enable transaction tagging */
if (dotagging > 0) {
sprintf(tagstr + KEYPREFIXLEN + TAGPREFIXLENGTH_MAX, "%03d", urand(0, args->txntagging - 1));
fdb_error_t err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_AUTO_THROTTLE_TAG,
(uint8_t*)tagstr, 16);
fdb_error_t err =
fdb_transaction_set_option(transaction, FDB_TR_OPTION_AUTO_THROTTLE_TAG, (uint8_t*)tagstr, 16);
if (err) {
fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n",
fdb_get_error(err));
fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fdb_get_error(err));
}
}
rc = run_one_transaction(transaction, args, stats, keystr, keystr2, valstr, block, elem_size,
is_memory_allocated);
rc = run_one_transaction(
transaction, args, stats, keystr, keystr2, valstr, block, elem_size, is_memory_allocated);
if (rc) {
/* FIXME: run_one_transaction should return something meaningful */
fprintf(annoyme, "ERROR: run_one_transaction failed (%d)\n", rc);
@ -829,7 +925,7 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
if (dotrace) {
free(traceid);
}
if(dotagging) {
if (dotagging) {
free(tagstr);
}
@ -924,8 +1020,13 @@ void* worker_thread(void* thread_args) {
stats->latency_samples[op] = 0;
}
fprintf(debugme, "DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n", worker_id, args->num_processes, thread_id,
args->num_threads, (unsigned int)pthread_self());
fprintf(debugme,
"DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n",
worker_id,
args->num_processes,
thread_id,
args->num_threads,
(unsigned int)pthread_self());
if (args->tpsmax) {
thread_tps = compute_thread_tps(args->tpsmax, worker_id, thread_id, args->num_processes, args->num_threads);
@ -965,8 +1066,18 @@ void* worker_thread(void* thread_args) {
/* run the workload */
else if (args->mode == MODE_RUN) {
rc = run_workload(transaction, args, thread_tps, throttle_factor, thread_iters,
signal, stats, dotrace, dotagging, block, elem_size, is_memory_allocated);
rc = run_workload(transaction,
args,
thread_tps,
throttle_factor,
thread_iters,
signal,
stats,
dotrace,
dotagging,
block,
elem_size,
is_memory_allocated);
if (rc < 0) {
fprintf(stderr, "ERROR: run_workload failed\n");
}
@ -991,7 +1102,8 @@ void* worker_thread(void* thread_args) {
temp_block = temp_block->next_block;
}
size = stats->latency_samples[op] % LAT_BLOCK_SIZE;
if (size != 0) fwrite(&temp_block->data, sizeof(uint64_t) * size, 1, fp);
if (size != 0)
fwrite(&temp_block->data, sizeof(uint64_t) * size, 1, fp);
} else {
while (temp_block) {
fwrite(&temp_block->data, sizeof(uint64_t) * LAT_BLOCK_SIZE, 1, fp);
@ -1047,7 +1159,6 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
return -1;
}
/* enable flatbuffers if specified */
if (args->flatbuffers) {
#ifdef FDB_NET_OPTION_USE_FLATBUFFERS
@ -1063,7 +1174,9 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
/* enable tracing if specified */
if (args->trace) {
fprintf(debugme, "DEBUG: Enable Tracing in %s (%s)\n", (args->traceformat == 0) ? "XML" : "JSON",
fprintf(debugme,
"DEBUG: Enable Tracing in %s (%s)\n",
(args->traceformat == 0) ? "XML" : "JSON",
(args->tracepath[0] == '\0') ? "current directory" : args->tracepath);
err = fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE, (uint8_t*)args->tracepath, strlen(args->tracepath));
if (err) {
@ -1095,11 +1208,10 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
fprintf(debugme, "DEBUG: fdb_setup_network\n");
err = fdb_setup_network();
if (err) {
fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err));
return -1;
fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err));
return -1;
}
/* Each worker process will have its own network thread */
fprintf(debugme, "DEBUG: creating network thread\n");
rc = pthread_create(&network_thread, NULL, fdb_network_thread, (void*)args);
@ -1181,8 +1293,10 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
}
failExit:
if (worker_threads) free(worker_threads);
if (thread_args) free(thread_args);
if (worker_threads)
free(worker_threads);
if (thread_args)
free(thread_args);
/* clean up database and cluster */
fdb_database_destroy(process.database);
@ -1208,7 +1322,8 @@ failExit:
/* initialize the parameters with default values */
int init_args(mako_args_t* args) {
int i;
if (!args) return -1;
if (!args)
return -1;
memset(args, 0, sizeof(mako_args_t)); /* zero-out everything */
args->api_version = fdb_get_max_api_version();
args->json = 0;
@ -1394,8 +1509,11 @@ void usage() {
printf("%-24s %s\n", " --tracepath=PATH", "Set trace file path");
printf("%-24s %s\n", " --trace_format <xml|json>", "Set trace format (Default: json)");
printf("%-24s %s\n", " --txntrace=sec", "Specify transaction tracing interval (Default: 0)");
printf("%-24s %s\n", " --txntagging", "Specify the number of different transaction tag (Default: 0, max = 1000)");
printf("%-24s %s\n", " --txntagging_prefix", "Specify the prefix of transaction tag - mako${txntagging_prefix} (Default: '')");
printf(
"%-24s %s\n", " --txntagging", "Specify the number of different transaction tag (Default: 0, max = 1000)");
printf("%-24s %s\n",
" --txntagging_prefix",
"Specify the prefix of transaction tag - mako${txntagging_prefix} (Default: '')");
printf("%-24s %s\n", " --knobs=KNOBS", "Set client knobs");
printf("%-24s %s\n", " --flatbuffers", "Use flatbuffers");
printf("%-24s %s\n", " --streaming", "Streaming mode: all (default), iterator, small, medium, large, serial");
@ -1430,7 +1548,7 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
{ "knobs", required_argument, NULL, ARG_KNOBS },
{ "tracepath", required_argument, NULL, ARG_TRACEPATH },
{ "trace_format", required_argument, NULL, ARG_TRACEFORMAT },
{ "streaming", required_argument, NULL, ARG_STREAMING_MODE },
{ "streaming", required_argument, NULL, ARG_STREAMING_MODE },
{ "txntrace", required_argument, NULL, ARG_TXNTRACE },
/* no args */
{ "help", no_argument, NULL, 'h' },
@ -1440,13 +1558,14 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
{ "flatbuffers", no_argument, NULL, ARG_FLATBUFFERS },
{ "trace", no_argument, NULL, ARG_TRACE },
{ "txntagging", required_argument, NULL, ARG_TXNTAGGING },
{ "txntagging_prefix", required_argument, NULL, ARG_TXNTAGGINGPREFIX},
{ "txntagging_prefix", required_argument, NULL, ARG_TXNTAGGINGPREFIX },
{ "version", no_argument, NULL, ARG_VERSION },
{ NULL, 0, NULL, 0 }
};
idx = 0;
c = getopt_long(argc, argv, short_options, long_options, &idx);
if (c < 0) break;
if (c < 0)
break;
switch (c) {
case '?':
case 'h':
@ -1475,7 +1594,8 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
break;
case 'x':
rc = parse_transaction(args, optarg);
if (rc < 0) return -1;
if (rc < 0)
return -1;
break;
case 'v':
args->verbose = atoi(optarg);
@ -1577,19 +1697,18 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
case ARG_TXNTAGGING:
args->txntagging = atoi(optarg);
if(args->txntagging > 1000) {
if (args->txntagging > 1000) {
args->txntagging = 1000;
}
break;
case ARG_TXNTAGGINGPREFIX: {
if(strlen(optarg) > TAGPREFIXLENGTH_MAX) {
if (strlen(optarg) > TAGPREFIXLENGTH_MAX) {
fprintf(stderr, "Error: the length of txntagging_prefix is larger than %d\n", TAGPREFIXLENGTH_MAX);
exit(0);
}
memcpy(args->txntagging_prefix, optarg, strlen(optarg));
break;
}
}
}
@ -1649,9 +1768,9 @@ int validate_args(mako_args_t* args) {
fprintf(stderr, "ERROR: Must specify either seconds or iteration\n");
return -1;
}
if(args->txntagging < 0) {
fprintf(stderr, "ERROR: --txntagging must be a non-negative integer\n");
return -1;
if (args->txntagging < 0) {
fprintf(stderr, "ERROR: --txntagging must be a non-negative integer\n");
return -1;
}
}
return 0;
@ -1724,7 +1843,8 @@ void print_stats_header(mako_args_t* args, bool show_commit, bool is_first_heade
/* header */
if (is_first_header_empty)
for (i = 0; i <= STATS_TITLE_WIDTH; i++) printf(" ");
for (i = 0; i <= STATS_TITLE_WIDTH; i++)
printf(" ");
for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0) {
switch (op) {
@ -1768,7 +1888,8 @@ void print_stats_header(mako_args_t* args, bool show_commit, bool is_first_heade
}
}
if (show_commit) printf("%" STR(STATS_FIELD_WIDTH) "s ", "COMMIT");
if (show_commit)
printf("%" STR(STATS_FIELD_WIDTH) "s ", "COMMIT");
if (show_op_stats) {
printf("%" STR(STATS_FIELD_WIDTH) "s\n", "TRANSACTION");
} else {
@ -1776,37 +1897,46 @@ void print_stats_header(mako_args_t* args, bool show_commit, bool is_first_heade
printf("%" STR(STATS_FIELD_WIDTH) "s\n", "Conflicts/s");
}
for (i = 0; i < STATS_TITLE_WIDTH; i++) printf("=");
for (i = 0; i < STATS_TITLE_WIDTH; i++)
printf("=");
printf(" ");
for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0) {
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" ");
}
}
/* COMMIT */
if (show_commit) {
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" ");
}
if (show_op_stats) {
/* TRANSACTION */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" ");
} else {
/* TPS */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" ");
/* Conflicts */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
}
printf("\n");
}
void print_report(mako_args_t* args, mako_stats_t* stats, struct timespec* timer_now, struct timespec* timer_start,
void print_report(mako_args_t* args,
mako_stats_t* stats,
struct timespec* timer_now,
struct timespec* timer_start,
pid_t* pid_main) {
int i, j, k, op, index;
uint64_t totalxacts = 0;
@ -2069,13 +2199,18 @@ void print_report(mako_args_t* args, mako_stats_t* stats, struct timespec* timer
for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION) {
if (lat_total[op]) free(dataPoints[op]);
if (lat_total[op])
free(dataPoints[op]);
}
}
}
int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double* throttle_factor, volatile int* signal,
volatile int* stopcount, pid_t* pid_main) {
int stats_process_main(mako_args_t* args,
mako_stats_t* stats,
volatile double* throttle_factor,
volatile int* signal,
volatile int* stopcount,
pid_t* pid_main) {
struct timespec timer_start, timer_prev, timer_now;
double sin_factor;
@ -2084,7 +2219,8 @@ int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double*
usleep(10000); /* 10ms */
}
if (args->verbose >= VERBOSE_DEFAULT) print_stats_header(args, false, true, false);
if (args->verbose >= VERBOSE_DEFAULT)
print_stats_header(args, false, true, false);
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start);
timer_prev.tv_sec = timer_start.tv_sec;
@ -2126,7 +2262,8 @@ int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double*
}
}
if (args->verbose >= VERBOSE_DEFAULT) print_stats(args, stats, &timer_now, &timer_prev);
if (args->verbose >= VERBOSE_DEFAULT)
print_stats(args, stats, &timer_now, &timer_prev);
timer_prev.tv_sec = timer_now.tv_sec;
timer_prev.tv_nsec = timer_now.tv_nsec;
}
@ -2172,7 +2309,8 @@ int main(int argc, char* argv[]) {
}
rc = validate_args(&args);
if (rc < 0) return -1;
if (rc < 0)
return -1;
if (args.mode == MODE_CLEAN) {
/* cleanup will be done from a single thread */
@ -2338,9 +2476,11 @@ int main(int argc, char* argv[]) {
failExit:
if (worker_pids) free(worker_pids);
if (worker_pids)
free(worker_pids);
if (shm != MAP_FAILED) munmap(shm, shmsize);
if (shm != MAP_FAILED)
munmap(shm, shmsize);
if (shmfd) {
close(shmfd);

336
bindings/c/test/performance_test.c
View File

@ -31,38 +31,40 @@ int numKeys = 1000000;
int keySize = 16;
uint8_t** keys = NULL;
int valueSize = 100;
uint8_t *valueStr = NULL;
uint8_t* valueStr = NULL;
fdb_error_t waitError(FDBFuture *f) {
fdb_error_t waitError(FDBFuture* f) {
fdb_error_t blockError = fdb_future_block_until_ready(f);
if(!blockError) {
if (!blockError) {
return fdb_future_get_error(f);
} else {
return blockError;
}
}
struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) {
FDBTransaction *tr = NULL;
struct RunResult run(struct ResultSet* rs,
FDBDatabase* db,
struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) {
FDBTransaction* tr = NULL;
fdb_error_t e = fdb_database_create_transaction(db, &tr);
checkError(e, "create transaction", rs);
while(1) {
while (1) {
struct RunResult r = func(rs, tr);
e = r.e;
if(!e) {
FDBFuture *f = fdb_transaction_commit(tr);
if (!e) {
FDBFuture* f = fdb_transaction_commit(tr);
e = waitError(f);
fdb_future_destroy(f);
}
if(e) {
FDBFuture *f = fdb_transaction_on_error(tr, e);
if (e) {
FDBFuture* f = fdb_transaction_on_error(tr, e);
fdb_error_t retryE = waitError(f);
fdb_future_destroy(f);
if (retryE) {
fdb_transaction_destroy(tr);
return (struct RunResult) {0, retryE};
return (struct RunResult){ 0, retryE };
}
} else {
fdb_transaction_destroy(tr);
@ -73,19 +75,22 @@ struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*f
return RES(0, 4100); // internal_error ; we should never get here
}
int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) {
int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*),
FDBDatabase* db,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25;
int *results = malloc(sizeof(int)*numRuns);
int* results = malloc(sizeof(int) * numRuns);
int i = 0;
for(; i < numRuns; ++i) {
for (; i < numRuns; ++i) {
struct RunResult res = run(rs, db, testFxn);
if(res.e) {
if (res.e) {
logError(res.e, kpiName, rs);
free(results);
return 0;
}
results[i] = res.res;
if(results[i] < 0) {
if (results[i] < 0) {
free(results);
return -1;
}
@ -99,19 +104,22 @@ int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDB
return result;
}
int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) {
int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*),
FDBDatabase* db,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25;
int *results = malloc(sizeof(int)*numRuns);
int* results = malloc(sizeof(int) * numRuns);
int i = 0;
for(; i < numRuns; ++i) {
for (; i < numRuns; ++i) {
struct RunResult res = testFxn(rs, db);
if(res.e) {
if (res.e) {
logError(res.e, kpiName, rs);
free(results);
return 0;
}
results[i] = res.res;
if(results[i] < 0) {
if (results[i] < 0) {
free(results);
return -1;
}
@ -125,139 +133,144 @@ int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBD
return result;
}
struct RunResult clearAll(struct ResultSet *rs, FDBTransaction *tr) {
struct RunResult clearAll(struct ResultSet* rs, FDBTransaction* tr) {
fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1);
return RES(0, 0);
}
uint32_t start = 0;
uint32_t stop = 0;
struct RunResult insertRange(struct ResultSet *rs, FDBTransaction *tr) {
struct RunResult insertRange(struct ResultSet* rs, FDBTransaction* tr) {
int i;
for(i = start; i < stop; i++) {
for (i = start; i < stop; i++) {
fdb_transaction_set(tr, keys[i], keySize, valueStr, valueSize);
}
return RES(0, 0);
}
void insertData(struct ResultSet *rs, FDBDatabase *db) {
void insertData(struct ResultSet* rs, FDBDatabase* db) {
checkError(run(rs, db, &clearAll).e, "clearing database", rs);
// TODO: Do this asynchronously.
start = 0;
while(start < numKeys) {
while (start < numKeys) {
stop = start + 1000;
if(stop > numKeys) stop = numKeys;
if (stop > numKeys)
stop = numKeys;
checkError(run(rs, db, &insertRange).e, "inserting data range", rs);
start = stop;
}
}
fdb_error_t setRetryLimit(struct ResultSet *rs, FDBTransaction *tr, uint64_t limit) {
fdb_error_t setRetryLimit(struct ResultSet* rs, FDBTransaction* tr, uint64_t limit) {
return fdb_transaction_set_option(tr, FDB_TR_OPTION_RETRY_LIMIT, (const uint8_t*)&limit, sizeof(uint64_t));
}
uint32_t FUTURE_LATENCY_COUNT = 100000;
const char *FUTURE_LATENCY_KPI = "C future throughput (local client)";
struct RunResult futureLatency(struct ResultSet *rs, FDBTransaction *tr) {
const char* FUTURE_LATENCY_KPI = "C future throughput (local client)";
struct RunResult futureLatency(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
FDBFuture *f = fdb_transaction_get_read_version(tr);
FDBFuture* f = fdb_transaction_get_read_version(tr);
e = waitError(f);
fdb_future_destroy(f);
maybeLogError(e, "getting initial read version", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
double start = getTime();
int i;
for(i = 0; i < FUTURE_LATENCY_COUNT; i++) {
FDBFuture *f = fdb_transaction_get_read_version(tr);
for (i = 0; i < FUTURE_LATENCY_COUNT; i++) {
FDBFuture* f = fdb_transaction_get_read_version(tr);
e = waitError(f);
fdb_future_destroy(f);
maybeLogError(e, "getting read version", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
}
double end = getTime();
return RES(FUTURE_LATENCY_COUNT/(end - start), 0);
return RES(FUTURE_LATENCY_COUNT / (end - start), 0);
}
uint32_t CLEAR_COUNT = 100000;
const char *CLEAR_KPI = "C clear throughput (local client)";
struct RunResult clear(struct ResultSet *rs, FDBTransaction *tr) {
const char* CLEAR_KPI = "C clear throughput (local client)";
struct RunResult clear(struct ResultSet* rs, FDBTransaction* tr) {
double start = getTime();
int i;
for(i = 0; i < CLEAR_COUNT; i++) {
for (i = 0; i < CLEAR_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_clear(tr, keys[k], keySize);
}
double end = getTime();
fdb_transaction_reset(tr); // Don't actually clear things.
return RES(CLEAR_COUNT/(end - start), 0);
return RES(CLEAR_COUNT / (end - start), 0);
}
uint32_t CLEAR_RANGE_COUNT = 100000;
const char *CLEAR_RANGE_KPI = "C clear range throughput (local client)";
struct RunResult clearRange(struct ResultSet *rs, FDBTransaction *tr) {
const char* CLEAR_RANGE_KPI = "C clear range throughput (local client)";
struct RunResult clearRange(struct ResultSet* rs, FDBTransaction* tr) {
double start = getTime();
int i;
for(i = 0; i < CLEAR_RANGE_COUNT; i++) {
for (i = 0; i < CLEAR_RANGE_COUNT; i++) {
int k = ((uint64_t)rand()) % (numKeys - 1);
fdb_transaction_clear_range(tr, keys[k], keySize, keys[k+1], keySize);
fdb_transaction_clear_range(tr, keys[k], keySize, keys[k + 1], keySize);
}
double end = getTime();
fdb_transaction_reset(tr); // Don't actually clear things.
return RES(CLEAR_RANGE_COUNT/(end - start), 0);
return RES(CLEAR_RANGE_COUNT / (end - start), 0);
}
uint32_t SET_COUNT = 100000;
const char *SET_KPI = "C set throughput (local client)";
struct RunResult set(struct ResultSet *rs, FDBTransaction *tr) {
const char* SET_KPI = "C set throughput (local client)";
struct RunResult set(struct ResultSet* rs, FDBTransaction* tr) {
double start = getTime();
int i;
for(i = 0; i < SET_COUNT; i++) {
for (i = 0; i < SET_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
}
double end = getTime();
fdb_transaction_reset(tr); // Don't actually set things.
return RES(SET_COUNT/(end - start), 0);
return RES(SET_COUNT / (end - start), 0);
}
uint32_t PARALLEL_GET_COUNT = 10000;
const char *PARALLEL_GET_KPI = "C parallel get throughput (local client)";
struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) {
const char* PARALLEL_GET_KPI = "C parallel get throughput (local client)";
struct RunResult parallelGet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT);
FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT);
double start = getTime();
int i;
for(i = 0; i < PARALLEL_GET_COUNT; i++) {
for (i = 0; i < PARALLEL_GET_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys;
futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0);
}
fdb_bool_t present;
uint8_t const *outValue;
uint8_t const* outValue;
int outValueLength;
for(i = 0; i < PARALLEL_GET_COUNT; i++) {
for (i = 0; i < PARALLEL_GET_COUNT; i++) {
e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs);
if(e) {
if (e) {
fdb_future_destroy(futures[i]);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if(e) {
e = maybeLogError(
fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if (e) {
fdb_future_destroy(futures[i]);
return RES(0, e);
}
@ -268,39 +281,41 @@ struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
free(futures);
return RES(PARALLEL_GET_COUNT/(end - start), 0);
return RES(PARALLEL_GET_COUNT / (end - start), 0);
}
uint32_t ALTERNATING_GET_SET_COUNT = 2000;
const char *ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)";
struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) {
const char* ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)";
struct RunResult alternatingGetSet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT);
FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT);
double start = getTime();
int i;
for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
for (i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0);
}
fdb_bool_t present;
uint8_t const *outValue;
uint8_t const* outValue;
int outValueLength;
for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
for (i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs);
if(e) {
if (e) {
fdb_future_destroy(futures[i]);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if(e) {
e = maybeLogError(
fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if (e) {
fdb_future_destroy(futures[i]);
return RES(0, e);
}
@ -311,38 +326,39 @@ struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
free(futures);
return RES(ALTERNATING_GET_SET_COUNT/(end - start), 0);
return RES(ALTERNATING_GET_SET_COUNT / (end - start), 0);
}
uint32_t SERIAL_GET_COUNT = 2000;
const char *SERIAL_GET_KPI = "C serial get throughput (local client)";
struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
const char* SERIAL_GET_KPI = "C serial get throughput (local client)";
struct RunResult serialGet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
int i;
uint32_t *keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT);
uint32_t* keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT);
if(SERIAL_GET_COUNT > numKeys/2) {
for(i = 0; i < SERIAL_GET_COUNT; i++) {
if (SERIAL_GET_COUNT > numKeys / 2) {
for (i = 0; i < SERIAL_GET_COUNT; i++) {
keyIndices[i] = ((uint64_t)rand()) % numKeys;
}
} else {
for(i = 0; i < SERIAL_GET_COUNT; i++) {
while(1) {
for (i = 0; i < SERIAL_GET_COUNT; i++) {
while (1) {
// Yes, this is a linear scan. This happens outside
// the part we are measuring.
uint32_t index = ((uint64_t)rand()) % numKeys;
int j;
fdb_bool_t found = 0;
for(j = 0; j < i; j++) {
if(keyIndices[j] == index) {
for (j = 0; j < i; j++) {
if (keyIndices[j] == index) {
found = 1;
break;
}
}
if(!found) {
if (!found) {
keyIndices[i] = index;
break;
}
@ -353,13 +369,13 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
double start = getTime();
fdb_bool_t present;
uint8_t const *outValue;
uint8_t const* outValue;
int outValueLength;
for(i = 0; i < SERIAL_GET_COUNT; i++) {
FDBFuture *f = fdb_transaction_get(tr, keys[keyIndices[i]], keySize, 0);
for (i = 0; i < SERIAL_GET_COUNT; i++) {
FDBFuture* f = fdb_transaction_get(tr, keys[keyIndices[i]], keySize, 0);
fdb_error_t e = maybeLogError(fdb_future_block_until_ready(f), "getting key in serial", rs);
if(e) {
if (e) {
free(keyIndices);
fdb_future_destroy(f);
return RES(0, e);
@ -367,7 +383,7 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs);
fdb_future_destroy(f);
if(e) {
if (e) {
free(keyIndices);
return RES(0, e);
}
@ -376,66 +392,87 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
free(keyIndices);
return RES(SERIAL_GET_COUNT/(end - start), 0);
return RES(SERIAL_GET_COUNT / (end - start), 0);
}
uint32_t GET_RANGE_COUNT = 100000;
const char *GET_RANGE_KPI = "C get range throughput (local client)";
struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) {
const char* GET_RANGE_KPI = "C get range throughput (local client)";
struct RunResult getRange(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
uint32_t startKey = ((uint64_t)rand()) % (numKeys - GET_RANGE_COUNT - 1);
double start = getTime();
const FDBKeyValue *outKv;
const FDBKeyValue* outKv;
int outCount;
fdb_bool_t outMore = 1;
int totalOut = 0;
int iteration = 0;
FDBFuture *f = fdb_transaction_get_range(tr,
keys[startKey], keySize, 1, 0,
keys[startKey + GET_RANGE_COUNT], keySize, 1, 0,
0, 0,
FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0);
FDBFuture* f = fdb_transaction_get_range(tr,
keys[startKey],
keySize,
1,
0,
keys[startKey + GET_RANGE_COUNT],
keySize,
1,
0,
0,
0,
FDB_STREAMING_MODE_WANT_ALL,
++iteration,
0,
0);
while(outMore) {
while (outMore) {
e = maybeLogError(fdb_future_block_until_ready(f), "getting range", rs);
if(e) {
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading range array", rs);
if(e) {
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
totalOut += outCount;
if(outMore) {
FDBFuture *f2 = fdb_transaction_get_range(tr,
outKv[outCount - 1].key, outKv[outCount - 1].key_length, 1, 1,
keys[startKey + GET_RANGE_COUNT], keySize, 1, 0,
0, 0,
FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0);
if (outMore) {
FDBFuture* f2 = fdb_transaction_get_range(tr,
outKv[outCount - 1].key,
outKv[outCount - 1].key_length,
1,
1,
keys[startKey + GET_RANGE_COUNT],
keySize,
1,
0,
0,
0,
FDB_STREAMING_MODE_WANT_ALL,
++iteration,
0,
0);
fdb_future_destroy(f);
f = f2;
}
}
if(totalOut != GET_RANGE_COUNT) {
char *msg = (char*)malloc((sizeof(char)) * 200);
if (totalOut != GET_RANGE_COUNT) {
char* msg = (char*)malloc((sizeof(char)) * 200);
sprintf(msg, "verifying out count (%d != %d)", totalOut, GET_RANGE_COUNT);
logError(4100, msg, rs);
free(msg);
fdb_future_destroy(f);
return RES(0, 4100);
}
if(outMore) {
if (outMore) {
logError(4100, "verifying no more in range", rs);
fdb_future_destroy(f);
return RES(0, 4100);
@ -444,84 +481,84 @@ struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
return RES(GET_RANGE_COUNT/(end - start), 0);
return RES(GET_RANGE_COUNT / (end - start), 0);
}
uint32_t GET_KEY_COUNT = 2000;
const char *GET_KEY_KPI = "C get key throughput (local client)";
struct RunResult getKey(struct ResultSet *rs, FDBTransaction *tr) {
const char* GET_KEY_KPI = "C get key throughput (local client)";
struct RunResult getKey(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
double start = getTime();
fdb_bool_t present;
uint8_t const *outValue;
uint8_t const* outValue;
int outValueLength;
int i;
for(i = 0; i < GET_KEY_COUNT; i++) {
for (i = 0; i < GET_KEY_COUNT; i++) {
int key = ((uint64_t)rand()) % numKeys;
int offset = (((uint64_t)rand()) % 21) - 10;
FDBFuture *f = fdb_transaction_get_key(tr, keys[key], keySize, 1, offset, 0);
FDBFuture* f = fdb_transaction_get_key(tr, keys[key], keySize, 1, offset, 0);
e = maybeLogError(fdb_future_block_until_ready(f), "waiting for get key", rs);
if(e) {
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs);
fdb_future_destroy(f);
if(e) {
if (e) {
return RES(0, e);
}
}
double end = getTime();
return RES(GET_KEY_COUNT/(end - start), 0);
return RES(GET_KEY_COUNT / (end - start), 0);
}
uint32_t GET_SINGLE_KEY_RANGE_COUNT = 2000;
const char *GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)";
struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) {
const char* GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)";
struct RunResult getSingleKeyRange(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
double start = getTime();
const FDBKeyValue *outKv;
const FDBKeyValue* outKv;
int outCount;
fdb_bool_t outMore;
int i;
for(i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) {
for (i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) {
int key = ((uint64_t)rand()) % (numKeys - 1);
FDBFuture *f = fdb_transaction_get_range(tr,
keys[key], keySize, 1, 0,
keys[key + 1], keySize, 1, 0,
2, 0,
FDB_STREAMING_MODE_EXACT, 1, 0, 0);
FDBFuture* f = fdb_transaction_get_range(
tr, keys[key], keySize, 1, 0, keys[key + 1], keySize, 1, 0, 2, 0, FDB_STREAMING_MODE_EXACT, 1, 0, 0);
e = maybeLogError(fdb_future_block_until_ready(f), "waiting for single key range", rs);
if(e) {
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs);
if(e) {
e = maybeLogError(
fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs);
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
if(outCount != 1) {
if (outCount != 1) {
logError(4100, "more than one key returned in single key range read", rs);
fdb_future_destroy(f);
return RES(0, 4100);
}
if(outMore) {
if (outMore) {
logError(4100, "more keys to read in single key range read", rs);
fdb_future_destroy(f);
return RES(0, 4100);
@ -532,33 +569,34 @@ struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
return RES(GET_SINGLE_KEY_RANGE_COUNT/(end - start), 0);
return RES(GET_SINGLE_KEY_RANGE_COUNT / (end - start), 0);
}
struct RunResult singleKey(struct ResultSet *rs, FDBTransaction *tr) {
struct RunResult singleKey(struct ResultSet* rs, FDBTransaction* tr) {
int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
return RES(0, 0);
}
uint32_t WRITE_TRANSACTION_COUNT = 1000;
const char *WRITE_TRANSACTION_KPI = "C write_transaction throughput (local client)";
struct RunResult writeTransaction(struct ResultSet *rs, FDBDatabase *db) {
const char* WRITE_TRANSACTION_KPI = "C write_transaction throughput (local client)";
struct RunResult writeTransaction(struct ResultSet* rs, FDBDatabase* db) {
double start = getTime();
int i;
for(i = 0; i < WRITE_TRANSACTION_COUNT; i++) {
for (i = 0; i < WRITE_TRANSACTION_COUNT; i++) {
struct RunResult res = run(rs, db, &singleKey);
if(res.e) return res;
if (res.e)
return res;
}
double end = getTime();
return RES(WRITE_TRANSACTION_COUNT/(end - start), 0);
return RES(WRITE_TRANSACTION_COUNT / (end - start), 0);
}
void runTests(struct ResultSet *rs) {
FDBDatabase *db = openDatabase(rs, &netThread);
void runTests(struct ResultSet* rs) {
FDBDatabase* db = openDatabase(rs, &netThread);
printf("Loading database...\n");
insertData(rs, db);
@ -600,15 +638,15 @@ void runTests(struct ResultSet *rs) {
fdb_stop_network();
}
int main(int argc, char **argv) {
int main(int argc, char** argv) {
srand(time(NULL));
struct ResultSet *rs = newResultSet();
struct ResultSet* rs = newResultSet();
checkError(fdb_select_api_version(700), "select API version", rs);
printf("Running performance test at client version: %s\n", fdb_get_client_version());
valueStr = (uint8_t*)malloc((sizeof(uint8_t))*valueSize);
valueStr = (uint8_t*)malloc((sizeof(uint8_t)) * valueSize);
int i;
for(i = 0; i < valueSize; i++) {
for (i = 0; i < valueSize; i++) {
valueStr[i] = (uint8_t)'x';
}

156
bindings/c/test/ryw_benchmark.c
View File

@ -34,23 +34,26 @@ int numKeys = 10000;
int keySize = 16;
uint8_t** keys;
void insertData(FDBTransaction *tr) {
void insertData(FDBTransaction* tr) {
fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1);
uint8_t *v = (uint8_t*)"foo";
uint8_t* v = (uint8_t*)"foo";
uint32_t i;
for(i = 0; i <= numKeys; ++i) {
for (i = 0; i <= numKeys; ++i) {
fdb_transaction_set(tr, keys[i], keySize, v, 3);
}
}
int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction *tr, struct ResultSet *rs, const char *kpiName) {
int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*),
FDBTransaction* tr,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25;
int *results = malloc(sizeof(int)*numRuns);
int* results = malloc(sizeof(int) * numRuns);
int i = 0;
for(; i < numRuns; ++i) {
for (; i < numRuns; ++i) {
results[i] = testFxn(tr, rs);
if(results[i] < 0) {
if (results[i] < 0) {
free(results);
return -1;
}
@ -64,17 +67,19 @@ int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction *
return result;
}
int getSingle(FDBTransaction *tr, struct ResultSet *rs) {
int getSingle(FDBTransaction* tr, struct ResultSet* rs) {
int present;
uint8_t const *value;
uint8_t const* value;
int length;
int i;
double start = getTime();
for(i = 0; i < numKeys; ++i) {
FDBFuture *f = fdb_transaction_get(tr, keys[5001], keySize, 0);
if(getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs)) return -1;
if(getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs)) return -1;
for (i = 0; i < numKeys; ++i) {
FDBFuture* f = fdb_transaction_get(tr, keys[5001], keySize, 0);
if (getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs))
return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs))
return -1;
fdb_future_destroy(f);
}
double end = getTime();
@ -82,17 +87,19 @@ int getSingle(FDBTransaction *tr, struct ResultSet *rs) {
return numKeys / (end - start);
}
int getManySequential(FDBTransaction *tr, struct ResultSet *rs) {
int getManySequential(FDBTransaction* tr, struct ResultSet* rs) {
int present;
uint8_t const *value;
uint8_t const* value;
int length;
int i;
double start = getTime();
for(i = 0; i < numKeys; ++i) {
FDBFuture *f = fdb_transaction_get(tr, keys[i], keySize, 0);
if(getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs)) return -1;
if(getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs)) return -1;
for (i = 0; i < numKeys; ++i) {
FDBFuture* f = fdb_transaction_get(tr, keys[i], keySize, 0);
if (getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs))
return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs))
return -1;
fdb_future_destroy(f);
}
double end = getTime();
@ -100,20 +107,30 @@ int getManySequential(FDBTransaction *tr, struct ResultSet *rs) {
return numKeys / (end - start);
}
int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) {
int getRangeBasic(FDBTransaction* tr, struct ResultSet* rs) {
int count;
const FDBKeyValue *kvs;
const FDBKeyValue* kvs;
int more;
int i;
double start = getTime();
for(i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
for (i = 0; i < 100; ++i) {
FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs)) return -1;
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs))
return -1;
if (getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs))
return -1;
if(count != numKeys) {
if (count != numKeys) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys);
addError(rs, "GetRangeBasic bad count");
return -1;
@ -124,26 +141,37 @@ int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) {
return 100 * numKeys / (end - start);
}
int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) {
int singleClearGetRange(FDBTransaction* tr, struct ResultSet* rs) {
int count;
const FDBKeyValue *kvs;
const FDBKeyValue* kvs;
int more;
int i;
for(i = 0; i < numKeys; i+=2) {
for (i = 0; i < numKeys; i += 2) {
fdb_transaction_clear(tr, keys[i], keySize);
}
double start = getTime();
for(i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
for (i = 0; i < 100; ++i) {
FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs)) return -1;
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs))
return -1;
if (getError(
fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs))
return -1;
fdb_future_destroy(f);
if(count != numKeys/2) {
if (count != numKeys / 2) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys);
addError(rs, "SingleClearGetRange bad count");
return -1;
@ -155,27 +183,38 @@ int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) {
return 100 * numKeys / 2 / (end - start);
}
int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) {
int clearRangeGetRange(FDBTransaction* tr, struct ResultSet* rs) {
int count;
const FDBKeyValue *kvs;
const FDBKeyValue* kvs;
int more;
int i;
for(i = 0; i < numKeys; i+=4) {
fdb_transaction_clear_range(tr, keys[i], keySize, keys[i+1], keySize);
for (i = 0; i < numKeys; i += 4) {
fdb_transaction_clear_range(tr, keys[i], keySize, keys[i + 1], keySize);
}
double start = getTime();
for(i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
for (i = 0; i < 100; ++i) {
FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs)) return -1;
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs))
return -1;
if (getError(
fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs))
return -1;
fdb_future_destroy(f);
if(count != numKeys*3/4) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys*3/4);
if (count != numKeys * 3 / 4) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys * 3 / 4);
addError(rs, "ClearRangeGetRange bad count");
return -1;
}
@ -186,13 +225,13 @@ int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) {
return 100 * numKeys * 3 / 4 / (end - start);
}
int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
int interleavedSetsGets(FDBTransaction* tr, struct ResultSet* rs) {
int present;
uint8_t const *value;
uint8_t const* value;
int length;
int i;
uint8_t *k = (uint8_t*)"foo";
uint8_t* k = (uint8_t*)"foo";
uint8_t v[10];
int num = 1;
@ -200,10 +239,12 @@ int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
sprintf((char*)v, "%d", num);
fdb_transaction_set(tr, k, 3, v, strlen((char*)v));
for(i = 0; i < 10000; ++i) {
FDBFuture *f = fdb_transaction_get(tr, k, 3, 0);
if(getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs)) return -1;
if(getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs)) return -1;
for (i = 0; i < 10000; ++i) {
FDBFuture* f = fdb_transaction_get(tr, k, 3, 0);
if (getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs))
return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs))
return -1;
fdb_future_destroy(f);
sprintf((char*)v, "%d", ++num);
@ -214,13 +255,13 @@ int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
return 10000 / (end - start);
}
void runTests(struct ResultSet *rs) {
FDBDatabase *db = openDatabase(rs, &netThread);
void runTests(struct ResultSet* rs) {
FDBDatabase* db = openDatabase(rs, &netThread);
FDBTransaction *tr;
FDBTransaction* tr;
checkError(fdb_database_create_transaction(db, &tr), "create transaction", rs);
FDBFuture *f = fdb_transaction_get_read_version(tr);
FDBFuture* f = fdb_transaction_get_read_version(tr);
checkError(fdb_future_block_until_ready(f), "block for read version", rs);
int64_t version;
@ -241,9 +282,9 @@ void runTests(struct ResultSet *rs) {
fdb_stop_network();
}
int main(int argc, char **argv) {
int main(int argc, char** argv) {
srand(time(NULL));
struct ResultSet *rs = newResultSet();
struct ResultSet* rs = newResultSet();
checkError(fdb_select_api_version(700), "select API version", rs);
printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version());
@ -255,4 +296,3 @@ int main(int argc, char **argv) {
return 0;
}

109
bindings/c/test/test.h
View File

@ -38,27 +38,27 @@
double getTime() {
static struct timeval tv;
gettimeofday(&tv, NULL);
return tv.tv_usec/1000000.0 + tv.tv_sec;
return tv.tv_usec / 1000000.0 + tv.tv_sec;
}
void writeKey(uint8_t **dest, int key, int keySize) {
*dest = (uint8_t*)malloc((sizeof(uint8_t))*keySize);
void writeKey(uint8_t** dest, int key, int keySize) {
*dest = (uint8_t*)malloc((sizeof(uint8_t)) * keySize);
sprintf((char*)*dest, "%0*d", keySize, key);
}
uint8_t **generateKeys(int numKeys, int keySize) {
uint8_t **keys = (uint8_t**)malloc(sizeof(uint8_t*)*(numKeys+1));
uint8_t** generateKeys(int numKeys, int keySize) {
uint8_t** keys = (uint8_t**)malloc(sizeof(uint8_t*) * (numKeys + 1));
uint32_t i;
for(i = 0; i <= numKeys; ++i) {
for (i = 0; i <= numKeys; ++i) {
writeKey(keys + i, i, keySize);
}
return keys;
}
void freeKeys(uint8_t **keys, int numKeys) {
void freeKeys(uint8_t** keys, int numKeys) {
uint32_t i;
for(i = 0; i < numKeys; i++) {
for (i = 0; i < numKeys; i++) {
free(keys[i]);
}
free(keys);
@ -68,38 +68,39 @@ int cmpfunc(const void* a, const void* b) {
return (*(int*)a - *(int*)b);
}
int median(int *values, int length) {
int median(int* values, int length) {
qsort(values, length, sizeof(int), cmpfunc);
return values[length/2];
return values[length / 2];
}
struct RunResult {
int res;
fdb_error_t e;
};
#define RES(x, y) (struct RunResult) { x, y }
#define RES(x, y) \
(struct RunResult) { x, y }
struct Kpi {
const char *name;
const char* name;
int value;
const char *units;
const char* units;
struct Kpi *next;
struct Kpi* next;
};
struct Error {
char *message;
char* message;
struct Error *next;
struct Error* next;
};
struct ResultSet {
struct Kpi *kpis;
struct Error *errors;
struct Kpi* kpis;
struct Error* errors;
};
struct ResultSet* newResultSet() {
struct ResultSet *rs = malloc(sizeof(struct ResultSet));
struct ResultSet* rs = malloc(sizeof(struct ResultSet));
rs->kpis = NULL;
rs->errors = NULL;
@ -107,8 +108,8 @@ struct ResultSet* newResultSet() {
return rs;
}
void addKpi(struct ResultSet *rs, const char *name, int value, const char *units) {
struct Kpi *k = malloc(sizeof(struct Kpi));
void addKpi(struct ResultSet* rs, const char* name, int value, const char* units) {
struct Kpi* k = malloc(sizeof(struct Kpi));
k->name = name;
k->value = value;
k->units = units;
@ -116,20 +117,20 @@ void addKpi(struct ResultSet *rs, const char *name, int value, const char *units
rs->kpis = k;
}
void addError(struct ResultSet *rs, const char *message) {
struct Error *e = malloc(sizeof(struct Error));
e->message = (char*)malloc(strlen(message)+1);
void addError(struct ResultSet* rs, const char* message) {
struct Error* e = malloc(sizeof(struct Error));
e->message = (char*)malloc(strlen(message) + 1);
strcpy(e->message, message);
e->next = rs->errors;
rs->errors = e;
}
void writeResultSet(struct ResultSet *rs) {
void writeResultSet(struct ResultSet* rs) {
uint64_t id = ((uint64_t)rand() << 32) + rand();
char name[100];
sprintf(name, "fdb-c_result-%" SCNu64 ".json", id);
FILE *fp = fopen(name, "w");
if(!fp) {
FILE* fp = fopen(name, "w");
if (!fp) {
fprintf(stderr, "Could not open results file %s\n", name);
exit(1);
}
@ -137,10 +138,10 @@ void writeResultSet(struct ResultSet *rs) {
fprintf(fp, "{\n");
fprintf(fp, "\t\"kpis\": {\n");
struct Kpi *k = rs->kpis;
while(k != NULL) {
struct Kpi* k = rs->kpis;
while (k != NULL) {
fprintf(fp, "\t\t\"%s\": { \"units\": \"%s\", \"value\": %d }", k->name, k->units, k->value);
if(k->next != NULL) {
if (k->next != NULL) {
fprintf(fp, ",");
}
fprintf(fp, "\n");
@ -150,10 +151,10 @@ void writeResultSet(struct ResultSet *rs) {
fprintf(fp, "\t},\n");
fprintf(fp, "\t\"errors\": [\n");
struct Error *e = rs->errors;
while(e != NULL) {
struct Error* e = rs->errors;
while (e != NULL) {
fprintf(fp, "\t\t\"%s\"", e->message);
if(e->next != NULL) {
if (e->next != NULL) {
fprintf(fp, ",");
}
fprintf(fp, "\n");
@ -166,17 +167,17 @@ void writeResultSet(struct ResultSet *rs) {
fclose(fp);
}
void freeResultSet(struct ResultSet *rs) {
struct Kpi *k = rs->kpis;
while(k != NULL) {
struct Kpi *next = k->next;
void freeResultSet(struct ResultSet* rs) {
struct Kpi* k = rs->kpis;
while (k != NULL) {
struct Kpi* next = k->next;
free(k);
k = next;
}
struct Error *e = rs->errors;
while(e != NULL) {
struct Error *next = e->next;
struct Error* e = rs->errors;
while (e != NULL) {
struct Error* next = e->next;
free(e->message);
free(e);
e = next;
@ -185,12 +186,12 @@ void freeResultSet(struct ResultSet *rs) {
free(rs);
}
fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet *rs) {
if(err) {
char *msg = (char*)malloc(strlen(context) + 100);
fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet* rs) {
if (err) {
char* msg = (char*)malloc(strlen(context) + 100);
sprintf(msg, "Error in %s: %s", context, fdb_get_error(err));
fprintf(stderr, "%s\n", msg);
if(rs != NULL) {
if (rs != NULL) {
addError(rs, msg);
}
@ -200,9 +201,9 @@ fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet *rs)
return err;
}
void checkError(fdb_error_t err, const char* context, struct ResultSet *rs) {
if(getError(err, context, rs)) {
if(rs != NULL) {
void checkError(fdb_error_t err, const char* context, struct ResultSet* rs) {
if (getError(err, context, rs)) {
if (rs != NULL) {
writeResultSet(rs);
freeResultSet(rs);
}
@ -210,11 +211,11 @@ void checkError(fdb_error_t err, const char* context, struct ResultSet *rs) {
}
}
fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet *rs) {
char *msg = (char*)malloc(strlen(context) + 100);
fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet* rs) {
char* msg = (char*)malloc(strlen(context) + 100);
sprintf(msg, "Error in %s: %s", context, fdb_get_error(err));
fprintf(stderr, "%s\n", msg);
if(rs != NULL) {
if (rs != NULL) {
addError(rs, msg);
}
@ -222,8 +223,8 @@ fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet *rs)
return err;
}
fdb_error_t maybeLogError(fdb_error_t err, const char* context, struct ResultSet *rs) {
if(err && !fdb_error_predicate( FDB_ERROR_PREDICATE_RETRYABLE, err ) ) {
fdb_error_t maybeLogError(fdb_error_t err, const char* context, struct ResultSet* rs) {
if (err && !fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, err)) {
return logError(err, context, rs);
}
return err;
@ -234,11 +235,11 @@ void* runNetwork() {
return NULL;
}
FDBDatabase* openDatabase(struct ResultSet *rs, pthread_t *netThread) {
FDBDatabase* openDatabase(struct ResultSet* rs, pthread_t* netThread) {
checkError(fdb_setup_network(), "setup network", rs);
pthread_create(netThread, NULL, (void*)(&runNetwork), NULL);
FDBDatabase *db;
FDBDatabase* db;
checkError(fdb_create_database(NULL, &db), "create database", rs);
return db;

16
bindings/c/test/txn_size_test.c
View File

@ -31,14 +31,14 @@ pthread_t netThread;
const int numKeys = 100;
uint8_t** keys = NULL;
#define KEY_SIZE 16
#define KEY_SIZE 16
#define VALUE_SIZE 100
uint8_t valueStr[VALUE_SIZE];
fdb_error_t getSize(struct ResultSet* rs, FDBTransaction* tr, int64_t* out_size) {
fdb_error_t e;
FDBFuture* future = fdb_transaction_get_approximate_size(tr);
e = maybeLogError(fdb_future_block_until_ready(future), "waiting for get future", rs);
if (e) {
fdb_future_destroy(future);
@ -55,11 +55,11 @@ fdb_error_t getSize(struct ResultSet* rs, FDBTransaction* tr, int64_t* out_size)
return 0;
}
void runTests(struct ResultSet *rs) {
void runTests(struct ResultSet* rs) {
int64_t sizes[numKeys];
int i = 0, j = 0;
FDBDatabase *db = openDatabase(rs, &netThread);
FDBTransaction *tr = NULL;
FDBDatabase* db = openDatabase(rs, &netThread);
FDBTransaction* tr = NULL;
fdb_error_t e = fdb_database_create_transaction(db, &tr);
checkError(e, "create transaction", rs);
memset(sizes, 0, numKeys * sizeof(uint32_t));
@ -82,7 +82,7 @@ void runTests(struct ResultSet *rs) {
printf("size %d: %u\n", i, sizes[i]);
i++;
fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i+1], KEY_SIZE);
fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i + 1], KEY_SIZE);
e = getSize(rs, tr, sizes + i);
checkError(e, "transaction get size", rs);
printf("size %d: %u\n", i, sizes[i]);
@ -94,9 +94,9 @@ void runTests(struct ResultSet *rs) {
printf("Test passed!\n");
}
int main(int argc, char **argv) {
int main(int argc, char** argv) {
srand(time(NULL));
struct ResultSet *rs = newResultSet();
struct ResultSet* rs = newResultSet();
checkError(fdb_select_api_version(700), "select API version", rs);
printf("Running performance test at client version: %s\n", fdb_get_client_version());

173
bindings/c/test/unit/fdb_api.cpp
View File

@ -27,82 +27,80 @@ namespace fdb {
// Future
Future::~Future() {
fdb_future_destroy(future_);
fdb_future_destroy(future_);
}
bool Future::is_ready() {
return fdb_future_is_ready(future_);
return fdb_future_is_ready(future_);
}
[[nodiscard]] fdb_error_t Future::block_until_ready() {
return fdb_future_block_until_ready(future_);
return fdb_future_block_until_ready(future_);
}
[[nodiscard]] fdb_error_t Future::set_callback(FDBCallback callback,
void* callback_parameter) {
return fdb_future_set_callback(future_, callback, callback_parameter);
[[nodiscard]] fdb_error_t Future::set_callback(FDBCallback callback, void* callback_parameter) {
return fdb_future_set_callback(future_, callback, callback_parameter);
}
[[nodiscard]] fdb_error_t Future::get_error() {
return fdb_future_get_error(future_);
return fdb_future_get_error(future_);
}
void Future::release_memory() {
fdb_future_release_memory(future_);
fdb_future_release_memory(future_);
}
void Future::cancel() {
fdb_future_cancel(future_);
fdb_future_cancel(future_);
}
// Int64Future
[[nodiscard]] fdb_error_t Int64Future::get(int64_t* out) {
return fdb_future_get_int64(future_, out);
return fdb_future_get_int64(future_, out);
}
// ValueFuture
[[nodiscard]] fdb_error_t ValueFuture::get(fdb_bool_t* out_present,
const uint8_t** out_value,
int* out_value_length) {
return fdb_future_get_value(future_, out_present, out_value,
out_value_length);
[[nodiscard]] fdb_error_t ValueFuture::get(fdb_bool_t* out_present, const uint8_t** out_value, int* out_value_length) {
return fdb_future_get_value(future_, out_present, out_value, out_value_length);
}
// KeyFuture
[[nodiscard]] fdb_error_t KeyFuture::get(const uint8_t** out_key,
int* out_key_length) {
return fdb_future_get_key(future_, out_key, out_key_length);
[[nodiscard]] fdb_error_t KeyFuture::get(const uint8_t** out_key, int* out_key_length) {
return fdb_future_get_key(future_, out_key, out_key_length);
}
// StringArrayFuture
[[nodiscard]] fdb_error_t StringArrayFuture::get(const char*** out_strings,
int* out_count) {
return fdb_future_get_string_array(future_, out_strings, out_count);
[[nodiscard]] fdb_error_t StringArrayFuture::get(const char*** out_strings, int* out_count) {
return fdb_future_get_string_array(future_, out_strings, out_count);
}
// KeyValueArrayFuture
[[nodiscard]] fdb_error_t KeyValueArrayFuture::get(const FDBKeyValue** out_kv,
int* out_count,
fdb_bool_t* out_more) {
return fdb_future_get_keyvalue_array(future_, out_kv, out_count, out_more);
[[nodiscard]] fdb_error_t KeyValueArrayFuture::get(const FDBKeyValue** out_kv, int* out_count, fdb_bool_t* out_more) {
return fdb_future_get_keyvalue_array(future_, out_kv, out_count, out_more);
}
// Database
Int64Future Database::reboot_worker(FDBDatabase* db, const uint8_t* address, int address_length, fdb_bool_t check,
int duration) {
Int64Future Database::reboot_worker(FDBDatabase* db,
const uint8_t* address,
int address_length,
fdb_bool_t check,
int duration) {
return Int64Future(fdb_database_reboot_worker(db, address, address_length, check, duration));
}
EmptyFuture Database::force_recovery_with_data_loss(FDBDatabase *db, const uint8_t *dcid, int dcid_length) {
return EmptyFuture(fdb_database_force_recovery_with_data_loss(db, dcid, dcid_length));
EmptyFuture Database::force_recovery_with_data_loss(FDBDatabase* db, const uint8_t* dcid, int dcid_length) {
return EmptyFuture(fdb_database_force_recovery_with_data_loss(db, dcid, dcid_length));
}
EmptyFuture Database::create_snapshot(FDBDatabase* db, const uint8_t* uid, int uid_length, const uint8_t* snap_command,
EmptyFuture Database::create_snapshot(FDBDatabase* db,
const uint8_t* uid,
int uid_length,
const uint8_t* snap_command,
int snap_command_length) {
return EmptyFuture(fdb_database_create_snapshot(db, uid, uid_length, snap_command, snap_command_length));
}
@ -110,61 +108,58 @@ EmptyFuture Database::create_snapshot(FDBDatabase* db, const uint8_t* uid, int u
// Transaction
Transaction::Transaction(FDBDatabase* db) {
if (fdb_error_t err = fdb_database_create_transaction(db, &tr_)) {
std::cerr << fdb_get_error(err) << std::endl;
std::abort();
}
if (fdb_error_t err = fdb_database_create_transaction(db, &tr_)) {
std::cerr << fdb_get_error(err) << std::endl;
std::abort();
}
}
Transaction::~Transaction() {
fdb_transaction_destroy(tr_);
fdb_transaction_destroy(tr_);
}
void Transaction::reset() {
fdb_transaction_reset(tr_);
fdb_transaction_reset(tr_);
}
void Transaction::cancel() {
fdb_transaction_cancel(tr_);
fdb_transaction_cancel(tr_);
}
[[nodiscard]] fdb_error_t Transaction::set_option(FDBTransactionOption option,
const uint8_t* value,
int value_length) {
return fdb_transaction_set_option(tr_, option, value, value_length);
[[nodiscard]] fdb_error_t Transaction::set_option(FDBTransactionOption option, const uint8_t* value, int value_length) {
return fdb_transaction_set_option(tr_, option, value, value_length);
}
void Transaction::set_read_version(int64_t version) {
fdb_transaction_set_read_version(tr_, version);
fdb_transaction_set_read_version(tr_, version);
}
Int64Future Transaction::get_read_version() {
return Int64Future(fdb_transaction_get_read_version(tr_));
return Int64Future(fdb_transaction_get_read_version(tr_));
}
Int64Future Transaction::get_approximate_size() {
return Int64Future(fdb_transaction_get_approximate_size(tr_));
return Int64Future(fdb_transaction_get_approximate_size(tr_));
}
KeyFuture Transaction::get_versionstamp() {
return KeyFuture(fdb_transaction_get_versionstamp(tr_));
return KeyFuture(fdb_transaction_get_versionstamp(tr_));
}
ValueFuture Transaction::get(std::string_view key, fdb_bool_t snapshot) {
return ValueFuture(fdb_transaction_get(tr_, (const uint8_t*)key.data(),
key.size(), snapshot));
return ValueFuture(fdb_transaction_get(tr_, (const uint8_t*)key.data(), key.size(), snapshot));
}
KeyFuture Transaction::get_key(const uint8_t* key_name, int key_name_length,
fdb_bool_t or_equal, int offset,
KeyFuture Transaction::get_key(const uint8_t* key_name,
int key_name_length,
fdb_bool_t or_equal,
int offset,
fdb_bool_t snapshot) {
return KeyFuture(fdb_transaction_get_key(tr_, key_name, key_name_length,
or_equal, offset, snapshot));
return KeyFuture(fdb_transaction_get_key(tr_, key_name, key_name_length, or_equal, offset, snapshot));
}
StringArrayFuture Transaction::get_addresses_for_key(std::string_view key) {
return StringArrayFuture(fdb_transaction_get_addresses_for_key(tr_,
(const uint8_t*)key.data(), key.size()));
return StringArrayFuture(fdb_transaction_get_addresses_for_key(tr_, (const uint8_t*)key.data(), key.size()));
}
KeyValueArrayFuture Transaction::get_range(const uint8_t* begin_key_name,
@ -174,71 +169,71 @@ KeyValueArrayFuture Transaction::get_range(const uint8_t* begin_key_name,
const uint8_t* end_key_name,
int end_key_name_length,
fdb_bool_t end_or_equal,
int end_offset, int limit,
int end_offset,
int limit,
int target_bytes,
FDBStreamingMode mode,
int iteration, fdb_bool_t snapshot,
int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse) {
return KeyValueArrayFuture(fdb_transaction_get_range(tr_, begin_key_name,
begin_key_name_length,
begin_or_equal,
begin_offset,
end_key_name,
end_key_name_length,
end_or_equal,
end_offset,
limit, target_bytes,
mode, iteration,
snapshot, reverse));
return KeyValueArrayFuture(fdb_transaction_get_range(tr_,
begin_key_name,
begin_key_name_length,
begin_or_equal,
begin_offset,
end_key_name,
end_key_name_length,
end_or_equal,
end_offset,
limit,
target_bytes,
mode,
iteration,
snapshot,
reverse));
}
EmptyFuture Transaction::watch(std::string_view key) {
return EmptyFuture(fdb_transaction_watch(tr_, (const uint8_t*)key.data(), key.size()));
return EmptyFuture(fdb_transaction_watch(tr_, (const uint8_t*)key.data(), key.size()));
}
EmptyFuture Transaction::commit() {
return EmptyFuture(fdb_transaction_commit(tr_));
return EmptyFuture(fdb_transaction_commit(tr_));
}
EmptyFuture Transaction::on_error(fdb_error_t err) {
return EmptyFuture(fdb_transaction_on_error(tr_, err));
return EmptyFuture(fdb_transaction_on_error(tr_, err));
}
void Transaction::clear(std::string_view key) {
return fdb_transaction_clear(tr_, (const uint8_t*)key.data(), key.size());
return fdb_transaction_clear(tr_, (const uint8_t*)key.data(), key.size());
}
void Transaction::clear_range(std::string_view begin_key,
std::string_view end_key) {
fdb_transaction_clear_range(tr_, (const uint8_t*)begin_key.data(),
begin_key.size(), (const uint8_t*)end_key.data(),
end_key.size());
void Transaction::clear_range(std::string_view begin_key, std::string_view end_key) {
fdb_transaction_clear_range(
tr_, (const uint8_t*)begin_key.data(), begin_key.size(), (const uint8_t*)end_key.data(), end_key.size());
}
void Transaction::set(std::string_view key, std::string_view value) {
fdb_transaction_set(tr_, (const uint8_t*)key.data(), key.size(),
(const uint8_t*)value.data(), value.size());
fdb_transaction_set(tr_, (const uint8_t*)key.data(), key.size(), (const uint8_t*)value.data(), value.size());
}
void Transaction::atomic_op(std::string_view key, const uint8_t* param,
int param_length, FDBMutationType operationType) {
return fdb_transaction_atomic_op(tr_, (const uint8_t*)key.data(), key.size(),
param, param_length, operationType);
void Transaction::atomic_op(std::string_view key,
const uint8_t* param,
int param_length,
FDBMutationType operationType) {
return fdb_transaction_atomic_op(tr_, (const uint8_t*)key.data(), key.size(), param, param_length, operationType);
}
[[nodiscard]] fdb_error_t Transaction::get_committed_version(int64_t* out_version) {
return fdb_transaction_get_committed_version(tr_, out_version);
return fdb_transaction_get_committed_version(tr_, out_version);
}
fdb_error_t Transaction::add_conflict_range(std::string_view begin_key,
std::string_view end_key,
FDBConflictRangeType type) {
return fdb_transaction_add_conflict_range(tr_,
(const uint8_t*)begin_key.data(),
begin_key.size(),
(const uint8_t*)end_key.data(),
end_key.size(),
type);
return fdb_transaction_add_conflict_range(
tr_, (const uint8_t*)begin_key.data(), begin_key.size(), (const uint8_t*)end_key.data(), end_key.size(), type);
}
} // namespace fdb
} // namespace fdb

286
bindings/c/test/unit/fdb_api.hpp
View File

@ -50,203 +50,207 @@ namespace fdb {
// Wrapper parent class to manage memory of an FDBFuture pointer. Cleans up
// FDBFuture when this instance goes out of scope.
class Future {
public:
virtual ~Future() = 0;
public:
virtual ~Future() = 0;
// Wrapper around fdb_future_is_ready.
bool is_ready();
// Wrapper around fdb_future_block_until_ready.
fdb_error_t block_until_ready();
// Wrapper around fdb_future_set_callback.
fdb_error_t set_callback(FDBCallback callback, void* callback_parameter);
// Wrapper around fdb_future_get_error.
fdb_error_t get_error();
// Wrapper around fdb_future_release_memory.
void release_memory();
// Wrapper around fdb_future_cancel.
void cancel();
// Wrapper around fdb_future_is_ready.
bool is_ready();
// Wrapper around fdb_future_block_until_ready.
fdb_error_t block_until_ready();
// Wrapper around fdb_future_set_callback.
fdb_error_t set_callback(FDBCallback callback, void* callback_parameter);
// Wrapper around fdb_future_get_error.
fdb_error_t get_error();
// Wrapper around fdb_future_release_memory.
void release_memory();
// Wrapper around fdb_future_cancel.
void cancel();
// Conversion operator to allow Future instances to work interchangeably as
// an FDBFuture object.
// operator FDBFuture* () const {
// return future_;
// }
// Conversion operator to allow Future instances to work interchangeably as
// an FDBFuture object.
// operator FDBFuture* () const {
// return future_;
// }
protected:
Future(FDBFuture *f) : future_(f) {}
FDBFuture* future_;
protected:
Future(FDBFuture* f) : future_(f) {}
FDBFuture* future_;
};
class Int64Future : public Future {
public:
// Call this function instead of fdb_future_get_int64 when using the
// Int64Future type. It's behavior is identical to fdb_future_get_int64.
fdb_error_t get(int64_t* out);
private:
friend class Transaction;
friend class Database;
Int64Future(FDBFuture* f) : Future(f) {}
public:
// Call this function instead of fdb_future_get_int64 when using the
// Int64Future type. It's behavior is identical to fdb_future_get_int64.
fdb_error_t get(int64_t* out);
private:
friend class Transaction;
friend class Database;
Int64Future(FDBFuture* f) : Future(f) {}
};
class KeyFuture : public Future {
public:
// Call this function instead of fdb_future_get_key when using the KeyFuture
// type. It's behavior is identical to fdb_future_get_key.
fdb_error_t get(const uint8_t** out_key, int* out_key_length);
public:
// Call this function instead of fdb_future_get_key when using the KeyFuture
// type. It's behavior is identical to fdb_future_get_key.
fdb_error_t get(const uint8_t** out_key, int* out_key_length);
private:
friend class Transaction;
KeyFuture(FDBFuture* f) : Future(f) {}
private:
friend class Transaction;
KeyFuture(FDBFuture* f) : Future(f) {}
};
class ValueFuture : public Future {
public:
// Call this function instead of fdb_future_get_value when using the
// ValueFuture type. It's behavior is identical to fdb_future_get_value.
fdb_error_t get(fdb_bool_t* out_present, const uint8_t** out_value,
int* out_value_length);
public:
// Call this function instead of fdb_future_get_value when using the
// ValueFuture type. It's behavior is identical to fdb_future_get_value.
fdb_error_t get(fdb_bool_t* out_present, const uint8_t** out_value, int* out_value_length);
private:
friend class Transaction;
ValueFuture(FDBFuture* f) : Future(f) {}
private:
friend class Transaction;
ValueFuture(FDBFuture* f) : Future(f) {}
};
class StringArrayFuture : public Future {
public:
// Call this function instead of fdb_future_get_string_array when using the
// StringArrayFuture type. It's behavior is identical to
// fdb_future_get_string_array.
fdb_error_t get(const char*** out_strings, int* out_count);
public:
// Call this function instead of fdb_future_get_string_array when using the
// StringArrayFuture type. It's behavior is identical to
// fdb_future_get_string_array.
fdb_error_t get(const char*** out_strings, int* out_count);
private:
friend class Transaction;
StringArrayFuture(FDBFuture* f) : Future(f) {}
private:
friend class Transaction;
StringArrayFuture(FDBFuture* f) : Future(f) {}
};
class KeyValueArrayFuture : public Future {
public:
// Call this function instead of fdb_future_get_keyvalue_array when using
// the KeyValueArrayFuture type. It's behavior is identical to
// fdb_future_get_keyvalue_array.
fdb_error_t get(const FDBKeyValue** out_kv, int* out_count,
fdb_bool_t* out_more);
public:
// Call this function instead of fdb_future_get_keyvalue_array when using
// the KeyValueArrayFuture type. It's behavior is identical to
// fdb_future_get_keyvalue_array.
fdb_error_t get(const FDBKeyValue** out_kv, int* out_count, fdb_bool_t* out_more);
private:
friend class Transaction;
KeyValueArrayFuture(FDBFuture* f) : Future(f) {}
private:
friend class Transaction;
KeyValueArrayFuture(FDBFuture* f) : Future(f) {}
};
class EmptyFuture : public Future {
private:
friend class Transaction;
friend class Database;
EmptyFuture(FDBFuture* f) : Future(f) {}
private:
friend class Transaction;
friend class Database;
EmptyFuture(FDBFuture* f) : Future(f) {}
};
// Wrapper around FDBDatabase, providing database-level API
class Database final {
public:
static Int64Future reboot_worker(FDBDatabase* db, const uint8_t* address, int address_length, fdb_bool_t check,
int duration);
static EmptyFuture force_recovery_with_data_loss(FDBDatabase* db, const uint8_t* dcid, int dcid_length);
static EmptyFuture create_snapshot(FDBDatabase* db, const uint8_t* uid, int uid_length, const uint8_t* snap_command,
int snap_command_length);
static Int64Future reboot_worker(FDBDatabase* db,
const uint8_t* address,
int address_length,
fdb_bool_t check,
int duration);
static EmptyFuture force_recovery_with_data_loss(FDBDatabase* db, const uint8_t* dcid, int dcid_length);
static EmptyFuture create_snapshot(FDBDatabase* db,
const uint8_t* uid,
int uid_length,
const uint8_t* snap_command,
int snap_command_length);
};
// Wrapper around FDBTransaction, providing the same set of calls as the C API.
// Handles cleanup of memory, removing the need to call
// fdb_transaction_destroy.
class Transaction final {
public:
// Given an FDBDatabase, initializes a new transaction.
Transaction(FDBDatabase* db);
~Transaction();
public:
// Given an FDBDatabase, initializes a new transaction.
Transaction(FDBDatabase* db);
~Transaction();
// Wrapper around fdb_transaction_reset.
void reset();
// Wrapper around fdb_transaction_reset.
void reset();
// Wrapper around fdb_transaction_cancel.
void cancel();
// Wrapper around fdb_transaction_cancel.
void cancel();
// Wrapper around fdb_transaction_set_option.
fdb_error_t set_option(FDBTransactionOption option, const uint8_t* value,
int value_length);
// Wrapper around fdb_transaction_set_option.
fdb_error_t set_option(FDBTransactionOption option, const uint8_t* value, int value_length);
// Wrapper around fdb_transaction_set_read_version.
void set_read_version(int64_t version);
// Wrapper around fdb_transaction_set_read_version.
void set_read_version(int64_t version);
// Returns a future which will be set to the transaction read version.
Int64Future get_read_version();
// Returns a future which will be set to the transaction read version.
Int64Future get_read_version();
// Returns a future which will be set to the approximate transaction size so far.
Int64Future get_approximate_size();
// Returns a future which will be set to the approximate transaction size so far.
Int64Future get_approximate_size();
// Returns a future which will be set to the versionstamp which was used by
// any versionstamp operations in the transaction.
KeyFuture get_versionstamp();
// Returns a future which will be set to the versionstamp which was used by
// any versionstamp operations in the transaction.
KeyFuture get_versionstamp();
// Returns a future which will be set to the value of `key` in the database.
ValueFuture get(std::string_view key, fdb_bool_t snapshot);
// Returns a future which will be set to the value of `key` in the database.
ValueFuture get(std::string_view key, fdb_bool_t snapshot);
// Returns a future which will be set to the key in the database matching the
// passed key selector.
KeyFuture get_key(const uint8_t* key_name, int key_name_length,
fdb_bool_t or_equal, int offset, fdb_bool_t snapshot);
// Returns a future which will be set to the key in the database matching the
// passed key selector.
KeyFuture get_key(const uint8_t* key_name,
int key_name_length,
fdb_bool_t or_equal,
int offset,
fdb_bool_t snapshot);
// Returns a future which will be set to an array of strings.
StringArrayFuture get_addresses_for_key(std::string_view key);
// Returns a future which will be set to an array of strings.
StringArrayFuture get_addresses_for_key(std::string_view key);
// Returns a future which will be set to an FDBKeyValue array.
KeyValueArrayFuture get_range(const uint8_t* begin_key_name,
int begin_key_name_length,
fdb_bool_t begin_or_equal, int begin_offset,
const uint8_t* end_key_name,
int end_key_name_length,
fdb_bool_t end_or_equal, int end_offset,
int limit, int target_bytes,
FDBStreamingMode mode, int iteration,
fdb_bool_t snapshot, fdb_bool_t reverse);
// Returns a future which will be set to an FDBKeyValue array.
KeyValueArrayFuture get_range(const uint8_t* begin_key_name,
int begin_key_name_length,
fdb_bool_t begin_or_equal,
int begin_offset,
const uint8_t* end_key_name,
int end_key_name_length,
fdb_bool_t end_or_equal,
int end_offset,
int limit,
int target_bytes,
FDBStreamingMode mode,
int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse);
// Wrapper around fdb_transaction_watch. Returns a future representing an
// empty value.
EmptyFuture watch(std::string_view key);
// Wrapper around fdb_transaction_watch. Returns a future representing an
// empty value.
EmptyFuture watch(std::string_view key);
// Wrapper around fdb_transaction_commit. Returns a future representing an
// empty value.
EmptyFuture commit();
// Wrapper around fdb_transaction_commit. Returns a future representing an
// empty value.
EmptyFuture commit();
// Wrapper around fdb_transaction_on_error. Returns a future representing an
// empty value.
EmptyFuture on_error(fdb_error_t err);
// Wrapper around fdb_transaction_on_error. Returns a future representing an
// empty value.
EmptyFuture on_error(fdb_error_t err);
// Wrapper around fdb_transaction_clear.
void clear(std::string_view key);
// Wrapper around fdb_transaction_clear.
void clear(std::string_view key);
// Wrapper around fdb_transaction_clear_range.
void clear_range(std::string_view begin_key, std::string_view end_key);
// Wrapper around fdb_transaction_clear_range.
void clear_range(std::string_view begin_key, std::string_view end_key);
// Wrapper around fdb_transaction_set.
void set(std::string_view key, std::string_view value);
// Wrapper around fdb_transaction_set.
void set(std::string_view key, std::string_view value);
// Wrapper around fdb_transaction_atomic_op.
void atomic_op(std::string_view key, const uint8_t* param, int param_length,
FDBMutationType operationType);
// Wrapper around fdb_transaction_atomic_op.
void atomic_op(std::string_view key, const uint8_t* param, int param_length, FDBMutationType operationType);
// Wrapper around fdb_transaction_get_committed_version.
fdb_error_t get_committed_version(int64_t* out_version);
// Wrapper around fdb_transaction_get_committed_version.
fdb_error_t get_committed_version(int64_t* out_version);
// Wrapper around fdb_transaction_add_conflict_range.
fdb_error_t add_conflict_range(std::string_view begin_key,
std::string_view end_key,
FDBConflictRangeType type);
// Wrapper around fdb_transaction_add_conflict_range.
fdb_error_t add_conflict_range(std::string_view begin_key, std::string_view end_key, FDBConflictRangeType type);
private:
FDBTransaction* tr_;
private:
FDBTransaction* tr_;
};
} // namespace fdb
} // namespace fdb

66
bindings/c/test/unit/setup_tests.cpp
View File

@ -29,47 +29,47 @@
#include "doctest.h"
void fdb_check(fdb_error_t e) {
if (e) {
std::cerr << fdb_get_error(e) << std::endl;
std::abort();
}
if (e) {
std::cerr << fdb_get_error(e) << std::endl;
std::abort();
}
}
TEST_CASE("setup") {
fdb_error_t err;
// Version passed here must be <= FDB_API_VERSION
err = fdb_select_api_version(9000);
CHECK(err);
fdb_error_t err;
// Version passed here must be <= FDB_API_VERSION
err = fdb_select_api_version(9000);
CHECK(err);
// Select current API version
fdb_check(fdb_select_api_version(700));
// Select current API version
fdb_check(fdb_select_api_version(700));
// Error to call again after a successful return
err = fdb_select_api_version(700);
CHECK(err);
// Error to call again after a successful return
err = fdb_select_api_version(700);
CHECK(err);
CHECK(fdb_get_max_api_version() >= 700);
CHECK(fdb_get_max_api_version() >= 700);
fdb_check(fdb_setup_network());
// Calling a second time should fail
err = fdb_setup_network();
CHECK(err);
fdb_check(fdb_setup_network());
// Calling a second time should fail
err = fdb_setup_network();
CHECK(err);
struct Context {
bool called = false;
};
Context context;
fdb_check(fdb_add_network_thread_completion_hook(
[](void *param) {
auto *context = static_cast<Context *>(param);
context->called = true;
},
&context));
struct Context {
bool called = false;
};
Context context;
fdb_check(fdb_add_network_thread_completion_hook(
[](void* param) {
auto* context = static_cast<Context*>(param);
context->called = true;
},
&context));
std::thread network_thread{&fdb_run_network};
std::thread network_thread{ &fdb_run_network };
CHECK(!context.called);
fdb_check(fdb_stop_network());
network_thread.join();
CHECK(context.called);
CHECK(!context.called);
fdb_check(fdb_stop_network());
network_thread.join();
CHECK(context.called);
}

3128
bindings/c/test/unit/unit_tests.cpp
View File

File diff suppressed because it is too large Load Diff

22
bindings/c/test/workloads/SimpleWorkload.cpp
View File

@ -104,7 +104,10 @@ struct SimpleWorkload : FDBWorkload {
unsigned long from, to, lastTx = 0;
std::unordered_map<State, ActorCallback> callbacks;
PopulateActor(const Callback& promise, SimpleWorkload& self, FDBDatabase* db, unsigned long from,
PopulateActor(const Callback& promise,
SimpleWorkload& self,
FDBDatabase* db,
unsigned long from,
unsigned long to)
: ActorBase(promise, self, db), from(from), to(to) {
error = fdb_database_create_transaction(db, &tx);
@ -130,8 +133,11 @@ struct SimpleWorkload : FDBWorkload {
for (; from < to && ops < self.insertsPerTx; ++ops, ++from) {
std::string value = std::to_string(from);
std::string key = KEY_PREFIX + value;
fdb_transaction_set(tx, reinterpret_cast<const uint8_t*>(key.c_str()), key.size(),
reinterpret_cast<const uint8_t*>(value.c_str()), value.size());
fdb_transaction_set(tx,
reinterpret_cast<const uint8_t*>(key.c_str()),
key.size(),
reinterpret_cast<const uint8_t*>(value.c_str()),
value.size());
}
lastTx = ops;
auto commit_future = fdb_transaction_commit(tx);
@ -154,7 +160,8 @@ struct SimpleWorkload : FDBWorkload {
run();
},
[this](fdb_error_t error) {
self.context->trace(FDBSeverity::Error, "AssertionFailure",
self.context->trace(FDBSeverity::Error,
"AssertionFailure",
{ { "Reason", "tx.onError failed" },
{ "Error", std::string(fdb_get_error(error)) } });
self.success = false;
@ -230,7 +237,8 @@ struct SimpleWorkload : FDBWorkload {
get();
},
[this](fdb_error_t) {
self.context->trace(FDBSeverity::Error, "AssertionFailure",
self.context->trace(FDBSeverity::Error,
"AssertionFailure",
{ { "Reason", "tx.onError failed" },
{ "Error", std::string(fdb_get_error(error)) } });
self.success = false;
@ -260,8 +268,8 @@ struct SimpleWorkload : FDBWorkload {
runFor = context->getOption("runFor", 10.0);
auto err = fdb_select_api_version(700);
if (err) {
context->trace(FDBSeverity::Info, "SelectAPIVersionFailed",
{ { "Error", std::string(fdb_get_error(err)) } });
context->trace(
FDBSeverity::Info, "SelectAPIVersionFailed", { { "Error", std::string(fdb_get_error(err)) } });
}
return true;
}

20
bindings/c/test/workloads/workloads.cpp
View File

@ -23,19 +23,19 @@
FDBWorkloadFactoryImpl::~FDBWorkloadFactoryImpl() {}
std::map<std::string, IFDBWorkloadFactory*>& FDBWorkloadFactoryImpl::factories() {
static std::map<std::string, IFDBWorkloadFactory*> _factories;
return _factories;
static std::map<std::string, IFDBWorkloadFactory*> _factories;
return _factories;
}
std::shared_ptr<FDBWorkload> FDBWorkloadFactoryImpl::create(const std::string &name) {
auto res = factories().find(name);
if (res == factories().end()) {
return nullptr;
}
return res->second->create();
std::shared_ptr<FDBWorkload> FDBWorkloadFactoryImpl::create(const std::string& name) {
auto res = factories().find(name);
if (res == factories().end()) {
return nullptr;
}
return res->second->create();
}
FDBWorkloadFactory* workloadFactory(FDBLogger*) {
static FDBWorkloadFactoryImpl impl;
return &impl;
static FDBWorkloadFactoryImpl impl;
return &impl;
}

10
bindings/c/test/workloads/workloads.h
View File

@ -33,15 +33,11 @@ struct FDBWorkloadFactoryImpl : FDBWorkloadFactory {
std::shared_ptr<FDBWorkload> create(const std::string& name) override;
};
template<class WorkloadType>
template <class WorkloadType>
struct FDBWorkloadFactoryT : IFDBWorkloadFactory {
explicit FDBWorkloadFactoryT(const std::string& name) {
FDBWorkloadFactoryImpl::factories()[name] = this;
}
explicit FDBWorkloadFactoryT(const std::string& name) { FDBWorkloadFactoryImpl::factories()[name] = this; }
std::shared_ptr<FDBWorkload> create() override {
return std::make_shared<WorkloadType>();
}
std::shared_ptr<FDBWorkload> create() override { return std::make_shared<WorkloadType>(); }
};
extern "C" DLLEXPORT FDBWorkloadFactory* workloadFactory(FDBLogger*);

901
bindings/flow/DirectoryLayer.actor.cpp
View File

@ -22,486 +22,541 @@
#include "DirectoryPartition.h"
namespace FDB {
const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = {1,0,0,0,0,0,0,0};
const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca");
const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer");
const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version");
const int64_t DirectoryLayer::SUB_DIR_KEY = 0;
const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = { 1, 0, 0, 0, 0, 0, 0, 0 };
const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca");
const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer");
const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version");
const int64_t DirectoryLayer::SUB_DIR_KEY = 0;
const uint32_t DirectoryLayer::VERSION[3] = {1, 0, 0};
const uint32_t DirectoryLayer::VERSION[3] = { 1, 0, 0 };
const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = LiteralStringRef("\xfe");
const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX);
const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace();
const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition");
const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = LiteralStringRef("\xfe");
const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX);
const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace();
const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition");
DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes) :
nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes),
rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY))
{ }
DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes)
: nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes),
rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY)) {}
Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const {
return nodeSubspace.get(prefix);
}
template<class T>
Optional<Subspace> DirectoryLayer::nodeWithPrefix(Optional<T> const& prefix) const {
if(!prefix.present()) {
return Optional<Subspace>();
}
return nodeWithPrefix(prefix.get());
}
ACTOR Future<DirectoryLayer::Node> find(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
state int pathIndex = 0;
state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path);
for(; pathIndex != path.size(); ++pathIndex) {
ASSERT(node.subspace.present());
Optional<FDBStandalone<ValueRef>> val = wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key()));
node.path.push_back(path[pathIndex]);
node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path);
DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
node = _node;
if(!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) {
return node;
}
}
if(!node.loadedMetadata) {
DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
node = _node;
}
return node;
}
IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const {
Path path;
path.reserve(this->path.size() + subpath.size());
path.insert(path.end(), this->path.begin(), this->path.end());
path.insert(path.end(), subpath.begin(), subpath.end());
return path;
}
Reference<DirectorySubspace> DirectoryLayer::contentsOfNode(Subspace const& node, Path const& path, Standalone<StringRef> const& layer) {
Standalone<StringRef> prefix = nodeSubspace.unpack(node.key()).getString(0);
if(layer == PARTITION_LAYER) {
return Reference<DirectorySubspace>(new DirectoryPartition(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this)));
}
else {
return Reference<DirectorySubspace>(new DirectorySubspace(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this), layer));
}
}
Reference<DirectorySubspace> DirectoryLayer::openInternal(Standalone<StringRef> const& layer, Node const& existingNode, bool allowOpen) {
if (!allowOpen) {
throw directory_already_exists();
}
if(layer.size() > 0 && layer != existingNode.layer) {
throw mismatched_layer();
}
return existingNode.getContents();
}
Future<Reference<DirectorySubspace>> DirectoryLayer::open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), false, true);
}
void DirectoryLayer::initializeDirectory(Reference<Transaction> const& tr) const {
tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12));
}
ACTOR Future<Void> checkVersionInternal(const DirectoryLayer* dirLayer, Reference<Transaction> tr, bool writeAccess) {
Optional<FDBStandalone<ValueRef>> versionBytes = wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY)));
if(!versionBytes.present()) {
if(writeAccess) {
dirLayer->initializeDirectory(tr);
}
return Void();
}
else {
if(versionBytes.get().size() != 12) {
throw invalid_directory_layer_metadata();
}
if(((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) {
throw incompatible_directory_version();
}
else if(((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) {
throw incompatible_directory_version();
}
}
return Void();
}
Future<Void> DirectoryLayer::checkVersion(Reference<Transaction> const& tr, bool writeAccess) const {
return checkVersionInternal(this, tr, writeAccess);
}
ACTOR Future<Standalone<StringRef>> getPrefix(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Optional<Standalone<StringRef>> prefix) {
if(!prefix.present()) {
Standalone<StringRef> allocated = wait(dirLayer->allocator.allocate(tr));
state Standalone<StringRef> finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key());
FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1));
if(result.size() > 0) {
throw directory_prefix_not_empty();
}
return finalPrefix;
}
return prefix.get();
}
ACTOR Future<Optional<Subspace>> nodeContainingKey(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Standalone<StringRef> key, bool snapshot) {
if(key.startsWith(dirLayer->nodeSubspace.key())) {
return dirLayer->rootNode;
}
KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key)));
FDBStandalone<RangeResultRef> result = wait(tr->getRange(range, 1, snapshot, true));
if(result.size() > 0) {
Standalone<StringRef> prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0);
if(key.startsWith(prevPrefix)) {
return dirLayer->nodeWithPrefix(prevPrefix);
}
}
Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const {
return nodeSubspace.get(prefix);
}
template <class T>
Optional<Subspace> DirectoryLayer::nodeWithPrefix(Optional<T> const& prefix) const {
if (!prefix.present()) {
return Optional<Subspace>();
}
ACTOR Future<bool> isPrefixFree(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Standalone<StringRef> prefix, bool snapshot){
if(!prefix.size()) {
return false;
}
return nodeWithPrefix(prefix.get());
}
Optional<Subspace> node = wait(nodeContainingKey(dirLayer, tr, prefix, snapshot));
if(node.present()) {
return false;
}
ACTOR Future<DirectoryLayer::Node> find(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
state int pathIndex = 0;
state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path);
FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot));
return !result.size();
for (; pathIndex != path.size(); ++pathIndex) {
ASSERT(node.subspace.present());
Optional<FDBStandalone<ValueRef>> val =
wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key()));
}
node.path.push_back(path[pathIndex]);
node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path);
ACTOR Future<Subspace> getParentNode(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
if(path.size() > 1) {
Reference<DirectorySubspace> parent = wait(dirLayer->createOrOpenInternal(tr, IDirectory::Path(path.begin(), path.end() - 1), StringRef(), Optional<Standalone<StringRef>>(), true, true));
return dirLayer->nodeWithPrefix(parent->key());
}
else {
return dirLayer->rootNode;
DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
node = _node;
if (!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) {
return node;
}
}
ACTOR Future<Reference<DirectorySubspace>> createInternal(
Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path,
Standalone<StringRef> layer, Optional<Standalone<StringRef>> prefix, bool allowCreate)
{
if(!allowCreate) {
throw directory_does_not_exist();
}
wait(dirLayer->checkVersion(tr, true));
state Standalone<StringRef> newPrefix = wait(getPrefix(dirLayer, tr, prefix));
bool isFree = wait(isPrefixFree(dirLayer, tr, newPrefix, !prefix.present()));
if(!isFree) {
throw directory_prefix_in_use();
}
Subspace parentNode = wait(getParentNode(dirLayer, tr, path));
Subspace node = dirLayer->nodeWithPrefix(newPrefix);
tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix);
tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer);
return dirLayer->contentsOfNode(node, path, layer);
if (!node.loadedMetadata) {
DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
node = _node;
}
ACTOR Future<Reference<DirectorySubspace>> _createOrOpenInternal(
Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path,
Standalone<StringRef> layer, Optional<Standalone<StringRef>> prefix, bool allowCreate, bool allowOpen)
{
ASSERT(!prefix.present() || allowCreate);
wait(dirLayer->checkVersion(tr, false));
return node;
}
if(prefix.present() && !dirLayer->allowManualPrefixes) {
if(!dirLayer->getPath().size()) {
throw manual_prefixes_not_enabled();
}
else {
throw prefix_in_partition();
}
IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const {
Path path;
path.reserve(this->path.size() + subpath.size());
path.insert(path.end(), this->path.begin(), this->path.end());
path.insert(path.end(), subpath.begin(), subpath.end());
return path;
}
Reference<DirectorySubspace> DirectoryLayer::contentsOfNode(Subspace const& node,
Path const& path,
Standalone<StringRef> const& layer) {
Standalone<StringRef> prefix = nodeSubspace.unpack(node.key()).getString(0);
if (layer == PARTITION_LAYER) {
return Reference<DirectorySubspace>(
new DirectoryPartition(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this)));
} else {
return Reference<DirectorySubspace>(
new DirectorySubspace(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this), layer));
}
}
Reference<DirectorySubspace> DirectoryLayer::openInternal(Standalone<StringRef> const& layer,
Node const& existingNode,
bool allowOpen) {
if (!allowOpen) {
throw directory_already_exists();
}
if (layer.size() > 0 && layer != existingNode.layer) {
throw mismatched_layer();
}
return existingNode.getContents();
}
Future<Reference<DirectorySubspace>> DirectoryLayer::open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), false, true);
}
void DirectoryLayer::initializeDirectory(Reference<Transaction> const& tr) const {
tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12));
}
ACTOR Future<Void> checkVersionInternal(const DirectoryLayer* dirLayer, Reference<Transaction> tr, bool writeAccess) {
Optional<FDBStandalone<ValueRef>> versionBytes =
wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY)));
if (!versionBytes.present()) {
if (writeAccess) {
dirLayer->initializeDirectory(tr);
}
return Void();
} else {
if (versionBytes.get().size() != 12) {
throw invalid_directory_layer_metadata();
}
if (((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) {
throw incompatible_directory_version();
} else if (((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) {
throw incompatible_directory_version();
}
}
return Void();
}
Future<Void> DirectoryLayer::checkVersion(Reference<Transaction> const& tr, bool writeAccess) const {
return checkVersionInternal(this, tr, writeAccess);
}
ACTOR Future<Standalone<StringRef>> getPrefix(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Optional<Standalone<StringRef>> prefix) {
if (!prefix.present()) {
Standalone<StringRef> allocated = wait(dirLayer->allocator.allocate(tr));
state Standalone<StringRef> finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key());
FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1));
if (result.size() > 0) {
throw directory_prefix_not_empty();
}
if(!path.size()){
throw cannot_open_root_directory();
}
return finalPrefix;
}
state DirectoryLayer::Node existingNode = wait(find(dirLayer, tr, path));
if(existingNode.exists()) {
if(existingNode.isInPartition()) {
IDirectory::Path subpath = existingNode.getPartitionSubpath();
Reference<DirectorySubspace> dirSpace = wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(tr, subpath, layer, prefix, allowCreate, allowOpen));
return dirSpace;
}
return dirLayer->openInternal(layer, existingNode, allowOpen);
return prefix.get();
}
ACTOR Future<Optional<Subspace>> nodeContainingKey(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Standalone<StringRef> key,
bool snapshot) {
if (key.startsWith(dirLayer->nodeSubspace.key())) {
return dirLayer->rootNode;
}
KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key)));
FDBStandalone<RangeResultRef> result = wait(tr->getRange(range, 1, snapshot, true));
if (result.size() > 0) {
Standalone<StringRef> prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0);
if (key.startsWith(prevPrefix)) {
return dirLayer->nodeWithPrefix(prevPrefix);
}
else {
Reference<DirectorySubspace> dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate));
}
return Optional<Subspace>();
}
ACTOR Future<bool> isPrefixFree(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Standalone<StringRef> prefix,
bool snapshot) {
if (!prefix.size()) {
return false;
}
Optional<Subspace> node = wait(nodeContainingKey(dirLayer, tr, prefix, snapshot));
if (node.present()) {
return false;
}
FDBStandalone<RangeResultRef> result = wait(tr->getRange(
KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot));
return !result.size();
}
ACTOR Future<Subspace> getParentNode(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
if (path.size() > 1) {
Reference<DirectorySubspace> parent =
wait(dirLayer->createOrOpenInternal(tr,
IDirectory::Path(path.begin(), path.end() - 1),
StringRef(),
Optional<Standalone<StringRef>>(),
true,
true));
return dirLayer->nodeWithPrefix(parent->key());
} else {
return dirLayer->rootNode;
}
}
ACTOR Future<Reference<DirectorySubspace>> createInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
Standalone<StringRef> layer,
Optional<Standalone<StringRef>> prefix,
bool allowCreate) {
if (!allowCreate) {
throw directory_does_not_exist();
}
wait(dirLayer->checkVersion(tr, true));
state Standalone<StringRef> newPrefix = wait(getPrefix(dirLayer, tr, prefix));
bool isFree = wait(isPrefixFree(dirLayer, tr, newPrefix, !prefix.present()));
if (!isFree) {
throw directory_prefix_in_use();
}
Subspace parentNode = wait(getParentNode(dirLayer, tr, path));
Subspace node = dirLayer->nodeWithPrefix(newPrefix);
tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix);
tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer);
return dirLayer->contentsOfNode(node, path, layer);
}
ACTOR Future<Reference<DirectorySubspace>> _createOrOpenInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
Standalone<StringRef> layer,
Optional<Standalone<StringRef>> prefix,
bool allowCreate,
bool allowOpen) {
ASSERT(!prefix.present() || allowCreate);
wait(dirLayer->checkVersion(tr, false));
if (prefix.present() && !dirLayer->allowManualPrefixes) {
if (!dirLayer->getPath().size()) {
throw manual_prefixes_not_enabled();
} else {
throw prefix_in_partition();
}
}
if (!path.size()) {
throw cannot_open_root_directory();
}
state DirectoryLayer::Node existingNode = wait(find(dirLayer, tr, path));
if (existingNode.exists()) {
if (existingNode.isInPartition()) {
IDirectory::Path subpath = existingNode.getPartitionSubpath();
Reference<DirectorySubspace> dirSpace =
wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(
tr, subpath, layer, prefix, allowCreate, allowOpen));
return dirSpace;
}
return dirLayer->openInternal(layer, existingNode, allowOpen);
} else {
Reference<DirectorySubspace> dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate));
return dirSpace;
}
}
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpenInternal(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix,
bool allowCreate,
bool allowOpen) {
return _createOrOpenInternal(
Reference<DirectoryLayer>::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::create(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix) {
return createOrOpenInternal(tr, path, layer, prefix, true, false);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), true, true);
}
ACTOR Future<Standalone<VectorRef<StringRef>>> listInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if (!node.exists()) {
throw directory_does_not_exist();
}
if (node.isInPartition(true)) {
Standalone<VectorRef<StringRef>> partitionList =
wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath()));
return partitionList;
}
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpenInternal(
Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix, bool allowCreate, bool allowOpen)
{
return _createOrOpenInternal(Reference<DirectoryLayer>::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen);
}
state Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY);
state Key begin = subdir.range().begin;
state Standalone<VectorRef<StringRef>> subdirectories;
Future<Reference<DirectorySubspace>> DirectoryLayer::create(
Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix)
{
return createOrOpenInternal(tr, path, layer, prefix, true, false);
}
loop {
FDBStandalone<RangeResultRef> subdirRange = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), true, true);
}
ACTOR Future<Standalone<VectorRef<StringRef>>> listInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if(!node.exists()) {
throw directory_does_not_exist();
}
if(node.isInPartition(true)) {
Standalone<VectorRef<StringRef>> partitionList = wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath()));
return partitionList;
for (int i = 0; i < subdirRange.size(); ++i) {
subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0));
}
state Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY);
state Key begin = subdir.range().begin;
state Standalone<VectorRef<StringRef>> subdirectories;
loop {
FDBStandalone<RangeResultRef> subdirRange = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
for(int i = 0; i < subdirRange.size(); ++i) {
subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0));
}
if(!subdirRange.more) {
return subdirectories;
}
begin = keyAfter(subdirRange.back().key);
if (!subdirRange.more) {
return subdirectories;
}
}
Future<Standalone<VectorRef<StringRef>>> DirectoryLayer::list(Reference<Transaction> const& tr, Path const& path) {
return listInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
begin = keyAfter(subdirRange.back().key);
}
}
bool pathsEqual(IDirectory::Path const& path1, IDirectory::Path const& path2, size_t maxElementsToCheck = std::numeric_limits<size_t>::max()) {
if(std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) {
Future<Standalone<VectorRef<StringRef>>> DirectoryLayer::list(Reference<Transaction> const& tr, Path const& path) {
return listInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
}
bool pathsEqual(IDirectory::Path const& path1,
IDirectory::Path const& path2,
size_t maxElementsToCheck = std::numeric_limits<size_t>::max()) {
if (std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) {
return false;
}
for (int i = 0; i < path1.size() && i < maxElementsToCheck; ++i) {
if (path1[i] != path2[i]) {
return false;
}
for(int i = 0; i < path1.size() && i < maxElementsToCheck; ++i) {
if(path1[i] != path2[i]) {
return false;
}
}
return true;
}
ACTOR Future<Void> removeFromParent(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
ASSERT(path.size() >= 1);
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1)));
if(parentNode.subspace.present()) {
tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key());
}
return true;
}
return Void();
ACTOR Future<Void> removeFromParent(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
ASSERT(path.size() >= 1);
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1)));
if (parentNode.subspace.present()) {
tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key());
}
ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path oldPath, IDirectory::Path newPath) {
wait(dirLayer->checkVersion(tr, true));
return Void();
}
if(oldPath.size() <= newPath.size()) {
if(pathsEqual(oldPath, newPath, oldPath.size())) {
throw invalid_destination_directory();
}
ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path oldPath,
IDirectory::Path newPath) {
wait(dirLayer->checkVersion(tr, true));
if (oldPath.size() <= newPath.size()) {
if (pathsEqual(oldPath, newPath, oldPath.size())) {
throw invalid_destination_directory();
}
std::vector<Future<DirectoryLayer::Node>> futures;
futures.push_back(find(dirLayer, tr, oldPath));
futures.push_back(find(dirLayer, tr, newPath));
std::vector<DirectoryLayer::Node> nodes = wait(getAll(futures));
state DirectoryLayer::Node oldNode = nodes[0];
state DirectoryLayer::Node newNode = nodes[1];
if(!oldNode.exists()) {
throw directory_does_not_exist();
}
if(oldNode.isInPartition() || newNode.isInPartition()) {
if(!oldNode.isInPartition() || !newNode.isInPartition() || !pathsEqual(oldNode.path, newNode.path)) {
throw cannot_move_directory_between_partitions();
}
Reference<DirectorySubspace> partitionMove = wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath()));
return partitionMove;
}
if(newNode.exists() || newPath.empty()) {
throw directory_already_exists();
}
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1)));
if(!parentNode.exists()) {
throw parent_directory_does_not_exist();
}
tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(), dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0));
wait(removeFromParent(dirLayer, tr, oldPath));
return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) {
return moveInternal(Reference<DirectoryLayer>::addRef(this), tr, oldPath, newPath);
std::vector<Future<DirectoryLayer::Node>> futures;
futures.push_back(find(dirLayer, tr, oldPath));
futures.push_back(find(dirLayer, tr, newPath));
std::vector<DirectoryLayer::Node> nodes = wait(getAll(futures));
state DirectoryLayer::Node oldNode = nodes[0];
state DirectoryLayer::Node newNode = nodes[1];
if (!oldNode.exists()) {
throw directory_does_not_exist();
}
Future<Reference<DirectorySubspace>> DirectoryLayer::moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) {
if (oldNode.isInPartition() || newNode.isInPartition()) {
if (!oldNode.isInPartition() || !newNode.isInPartition() || !pathsEqual(oldNode.path, newNode.path)) {
throw cannot_move_directory_between_partitions();
}
Reference<DirectorySubspace> partitionMove =
wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath()));
return partitionMove;
}
if (newNode.exists() || newPath.empty()) {
throw directory_already_exists();
}
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1)));
if (!parentNode.exists()) {
throw parent_directory_does_not_exist();
}
tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(),
dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0));
wait(removeFromParent(dirLayer, tr, oldPath));
return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) {
return moveInternal(Reference<DirectoryLayer>::addRef(this), tr, oldPath, newPath);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) {
throw cannot_modify_root_directory();
}
Future<Void> removeRecursive(Reference<DirectoryLayer> const&, Reference<Transaction> const&, Subspace const&);
ACTOR Future<Void> removeRecursive(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Subspace nodeSub) {
state Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY);
state Key begin = subdir.range().begin;
state std::vector<Future<Void>> futures;
loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
for (int i = 0; i < range.size(); ++i) {
Subspace subNode = dirLayer->nodeWithPrefix(range[i].value);
futures.push_back(removeRecursive(dirLayer, tr, subNode));
}
if (!range.more) {
break;
}
begin = keyAfter(range.back().key);
}
// waits are done concurrently
wait(waitForAll(futures));
Standalone<StringRef> nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0);
tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix)));
tr->clear(nodeSub.range());
return Void();
}
Future<bool> removeInternal(Reference<DirectoryLayer> const&,
Reference<Transaction> const&,
IDirectory::Path const&,
bool const&);
ACTOR Future<bool> removeInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
bool failOnNonexistent) {
wait(dirLayer->checkVersion(tr, true));
if (path.empty()) {
throw cannot_modify_root_directory();
}
Future<Void> removeRecursive(Reference<DirectoryLayer> const&, Reference<Transaction> const&, Subspace const&);
ACTOR Future<Void> removeRecursive(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Subspace nodeSub) {
state Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY);
state Key begin = subdir.range().begin;
state std::vector<Future<Void>> futures;
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
for (int i = 0; i < range.size(); ++i) {
Subspace subNode = dirLayer->nodeWithPrefix(range[i].value);
futures.push_back(removeRecursive(dirLayer, tr, subNode));
}
if(!range.more) {
break;
}
begin = keyAfter(range.back().key);
}
// waits are done concurrently
wait(waitForAll(futures));
Standalone<StringRef> nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0);
tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix)));
tr->clear(nodeSub.range());
return Void();
}
Future<bool> removeInternal(Reference<DirectoryLayer> const&, Reference<Transaction> const&, IDirectory::Path const&, bool const&);
ACTOR Future<bool> removeInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path, bool failOnNonexistent) {
wait(dirLayer->checkVersion(tr, true));
if(path.empty()) {
throw cannot_modify_root_directory();
}
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if(!node.exists()) {
if(failOnNonexistent) {
throw directory_does_not_exist();
}
else {
return false;
}
}
if(node.isInPartition()) {
bool recurse = wait(removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent));
return recurse;
}
state std::vector<Future<Void>> futures;
futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get()));
futures.push_back(removeFromParent(dirLayer, tr, path));
wait(waitForAll(futures));
return true;
}
Future<Void> DirectoryLayer::remove(Reference<Transaction> const& tr, Path const& path) {
return success(removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, true));
}
Future<bool> DirectoryLayer::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
return removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, false);
}
ACTOR Future<bool> existsInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if(!node.exists()) {
if (!node.exists()) {
if (failOnNonexistent) {
throw directory_does_not_exist();
} else {
return false;
}
if(node.isInPartition()) {
bool exists = wait(node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath()));
return exists;
}
return true;
}
Future<bool> DirectoryLayer::exists(Reference<Transaction> const& tr, Path const& path) {
return existsInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
if (node.isInPartition()) {
bool recurse = wait(
removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent));
return recurse;
}
Reference<DirectoryLayer> DirectoryLayer::getDirectoryLayer() {
return Reference<DirectoryLayer>::addRef(this);
}
state std::vector<Future<Void>> futures;
futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get()));
futures.push_back(removeFromParent(dirLayer, tr, path));
const Standalone<StringRef> DirectoryLayer::getLayer() const {
return StringRef();
}
wait(waitForAll(futures));
const IDirectory::Path DirectoryLayer::getPath() const {
return path;
}
return true;
}
Future<Void> DirectoryLayer::remove(Reference<Transaction> const& tr, Path const& path) {
return success(removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, true));
}
Future<bool> DirectoryLayer::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
return removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, false);
}
ACTOR Future<bool> existsInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if (!node.exists()) {
return false;
}
if (node.isInPartition()) {
bool exists = wait(node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath()));
return exists;
}
return true;
}
Future<bool> DirectoryLayer::exists(Reference<Transaction> const& tr, Path const& path) {
return existsInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
}
Reference<DirectoryLayer> DirectoryLayer::getDirectoryLayer() {
return Reference<DirectoryLayer>::addRef(this);
}
const Standalone<StringRef> DirectoryLayer::getLayer() const {
return StringRef();
}
const IDirectory::Path DirectoryLayer::getPath() const {
return path;
}
} // namespace FDB

154
bindings/flow/DirectoryLayer.h Executable file → Normal file
View File

@ -28,84 +28,108 @@
#include "HighContentionAllocator.h"
namespace FDB {
class DirectoryLayer : public IDirectory {
public:
DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE, Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE, bool allowManualPrefixes = false);
class DirectoryLayer : public IDirectory {
public:
DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE,
Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE,
bool allowManualPrefixes = false);
Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(), Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<Reference<DirectorySubspace>> create(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath);
Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath);
Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path());
Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path());
Reference<DirectoryLayer> getDirectoryLayer();
const Standalone<StringRef> getLayer() const;
const Path getPath() const;
Reference<DirectoryLayer> getDirectoryLayer();
const Standalone<StringRef> getLayer() const;
const Path getPath() const;
static const Subspace DEFAULT_NODE_SUBSPACE;
static const Subspace DEFAULT_CONTENT_SUBSPACE;
static const StringRef PARTITION_LAYER;
static const Subspace DEFAULT_NODE_SUBSPACE;
static const Subspace DEFAULT_CONTENT_SUBSPACE;
static const StringRef PARTITION_LAYER;
//private:
static const uint8_t LITTLE_ENDIAN_LONG_ONE[8];
static const StringRef HIGH_CONTENTION_KEY;
static const StringRef LAYER_KEY;
static const StringRef VERSION_KEY;
static const int64_t SUB_DIR_KEY;
static const uint32_t VERSION[3];
static const StringRef DEFAULT_NODE_SUBSPACE_PREFIX;
// private:
static const uint8_t LITTLE_ENDIAN_LONG_ONE[8];
static const StringRef HIGH_CONTENTION_KEY;
static const StringRef LAYER_KEY;
static const StringRef VERSION_KEY;
static const int64_t SUB_DIR_KEY;
static const uint32_t VERSION[3];
static const StringRef DEFAULT_NODE_SUBSPACE_PREFIX;
struct Node {
Node() {}
Node(Reference<DirectoryLayer> const& directoryLayer, Optional<Subspace> const& subspace, Path const& path, Path const& targetPath);
struct Node {
Node() {}
Node(Reference<DirectoryLayer> const& directoryLayer,
Optional<Subspace> const& subspace,
Path const& path,
Path const& targetPath);
bool exists() const;
bool exists() const;
Future<Node> loadMetadata(Reference<Transaction> tr);
void ensureMetadataLoaded() const;
Future<Node> loadMetadata(Reference<Transaction> tr);
void ensureMetadataLoaded() const;
bool isInPartition(bool includeEmptySubpath = false) const;
Path getPartitionSubpath() const;
Reference<DirectorySubspace> getContents() const;
Reference<DirectoryLayer> directoryLayer;
Optional<Subspace> subspace;
Path path;
Path targetPath;
Standalone<StringRef> layer;
bool loadedMetadata;
};
Reference<DirectorySubspace> openInternal(Standalone<StringRef> const& layer, Node const& existingNode, bool allowOpen);
Future<Reference<DirectorySubspace>> createOrOpenInternal(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer, Optional<Standalone<StringRef>> const& prefix, bool allowCreate, bool allowOpen);
void initializeDirectory(Reference<Transaction> const& tr) const;
Future<Void> checkVersion(Reference<Transaction> const& tr, bool writeAccess) const;
template <class T>
Optional<Subspace> nodeWithPrefix(Optional<T> const& prefix) const;
Subspace nodeWithPrefix(StringRef const& prefix) const;
Reference<DirectorySubspace> contentsOfNode(Subspace const& node, Path const& path, Standalone<StringRef> const& layer);
Path toAbsolutePath(Path const& subpath) const;
Subspace rootNode;
Subspace nodeSubspace;
Subspace contentSubspace;
HighContentionAllocator allocator;
bool allowManualPrefixes;
bool isInPartition(bool includeEmptySubpath = false) const;
Path getPartitionSubpath() const;
Reference<DirectorySubspace> getContents() const;
Reference<DirectoryLayer> directoryLayer;
Optional<Subspace> subspace;
Path path;
Path targetPath;
Standalone<StringRef> layer;
bool loadedMetadata;
};
}
Reference<DirectorySubspace> openInternal(Standalone<StringRef> const& layer,
Node const& existingNode,
bool allowOpen);
Future<Reference<DirectorySubspace>> createOrOpenInternal(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix,
bool allowCreate,
bool allowOpen);
void initializeDirectory(Reference<Transaction> const& tr) const;
Future<Void> checkVersion(Reference<Transaction> const& tr, bool writeAccess) const;
template <class T>
Optional<Subspace> nodeWithPrefix(Optional<T> const& prefix) const;
Subspace nodeWithPrefix(StringRef const& prefix) const;
Reference<DirectorySubspace> contentsOfNode(Subspace const& node,
Path const& path,
Standalone<StringRef> const& layer);
Path toAbsolutePath(Path const& subpath) const;
Subspace rootNode;
Subspace nodeSubspace;
Subspace contentSubspace;
HighContentionAllocator allocator;
bool allowManualPrefixes;
Path path;
};
} // namespace FDB
#endif

50
bindings/flow/DirectoryPartition.h
View File

@ -28,34 +28,38 @@
#include "DirectoryLayer.h"
namespace FDB {
class DirectoryPartition : public DirectorySubspace {
class DirectoryPartition : public DirectorySubspace {
public:
DirectoryPartition(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> parentDirectoryLayer)
: DirectorySubspace(path, prefix, Reference<DirectoryLayer>(new DirectoryLayer(Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)), Subspace(prefix))), DirectoryLayer::PARTITION_LAYER),
parentDirectoryLayer(parentDirectoryLayer)
{
this->directoryLayer->path = path;
}
virtual ~DirectoryPartition() {}
public:
DirectoryPartition(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> parentDirectoryLayer)
: DirectorySubspace(path,
prefix,
Reference<DirectoryLayer>(new DirectoryLayer(
Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)),
Subspace(prefix))),
DirectoryLayer::PARTITION_LAYER),
parentDirectoryLayer(parentDirectoryLayer) {
this->directoryLayer->path = path;
}
virtual ~DirectoryPartition() {}
virtual Key key() const { throw cannot_use_partition_as_subspace(); }
virtual bool contains(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); }
virtual Key key() const { throw cannot_use_partition_as_subspace(); }
virtual bool contains(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); }
virtual Key pack(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); }
virtual Tuple unpack(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); }
virtual KeyRange range(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); }
virtual Key pack(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); }
virtual Tuple unpack(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); }
virtual KeyRange range(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); }
virtual Subspace subspace(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); }
virtual Subspace get(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); }
virtual Subspace subspace(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); }
virtual Subspace get(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); }
protected:
Reference<DirectoryLayer> parentDirectoryLayer;
protected:
Reference<DirectoryLayer> parentDirectoryLayer;
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const {
return path.empty() ? parentDirectoryLayer : directoryLayer;
}
};
}
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const {
return path.empty() ? parentDirectoryLayer : directoryLayer;
}
};
} // namespace FDB
#endif

165
bindings/flow/DirectorySubspace.cpp Executable file → Normal file
View File

@ -21,89 +21,100 @@
#include "DirectorySubspace.h"
namespace FDB {
DirectorySubspace::DirectorySubspace(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> directoryLayer, Standalone<StringRef> const& layer)
: Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) { }
DirectorySubspace::DirectorySubspace(Path const& path,
StringRef const& prefix,
Reference<DirectoryLayer> directoryLayer,
Standalone<StringRef> const& layer)
: Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) {}
Future<Reference<DirectorySubspace>> DirectorySubspace::create(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix) {
return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix);
}
Future<Reference<DirectorySubspace>> DirectorySubspace::create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix)
{
return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix);
Future<Reference<DirectorySubspace>> DirectorySubspace::open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return directoryLayer->open(tr, getPartitionSubpath(path), layer);
}
Future<Reference<DirectorySubspace>> DirectorySubspace::createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer);
}
Future<bool> DirectorySubspace::exists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->exists(tr, getPartitionSubpath(path, directoryLayer));
}
Future<Standalone<VectorRef<StringRef>>> DirectorySubspace::list(Reference<Transaction> const& tr, Path const& path) {
return directoryLayer->list(tr, getPartitionSubpath(path));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) {
return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(Path());
Path directoryLayerPath = directoryLayer->getPath();
if (directoryLayerPath.size() > newAbsolutePath.size()) {
return cannot_move_directory_between_partitions();
}
Future<Reference<DirectorySubspace>> DirectorySubspace::open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
return directoryLayer->open(tr, getPartitionSubpath(path), layer);
}
Future<Reference<DirectorySubspace>> DirectorySubspace::createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer);
}
Future<bool> DirectorySubspace::exists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->exists(tr, getPartitionSubpath(path, directoryLayer));
}
Future<Standalone<VectorRef<StringRef>>> DirectorySubspace::list(Reference<Transaction> const& tr, Path const& path) {
return directoryLayer->list(tr, getPartitionSubpath(path));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) {
return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(Path());
Path directoryLayerPath = directoryLayer->getPath();
if(directoryLayerPath.size() > newAbsolutePath.size()) {
for (int i = 0; i < directoryLayerPath.size(); ++i) {
if (directoryLayerPath[i] != newAbsolutePath[i]) {
return cannot_move_directory_between_partitions();
}
for(int i = 0; i < directoryLayerPath.size(); ++i) {
if(directoryLayerPath[i] != newAbsolutePath[i]) {
return cannot_move_directory_between_partitions();
}
}
Path newRelativePath(newAbsolutePath.begin() + directoryLayerPath.size(), newAbsolutePath.end());
return directoryLayer->move(tr, getPartitionSubpath(Path(), directoryLayer), newRelativePath);
}
Future<Void> DirectorySubspace::remove(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->remove(tr, getPartitionSubpath(path, directoryLayer));
}
Future<bool> DirectorySubspace::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->removeIfExists(tr, getPartitionSubpath(path, directoryLayer));
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayer() {
return directoryLayer;
}
const Standalone<StringRef> DirectorySubspace::getLayer() const {
return layer;
}
const IDirectory::Path DirectorySubspace::getPath() const {
return path;
}
IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path, Reference<DirectoryLayer> directoryLayer) const {
if(!directoryLayer) {
directoryLayer = this->directoryLayer;
}
Path newPath(this->path.begin() + directoryLayer->getPath().size(), this->path.end());
newPath.insert(newPath.end(), path.begin(), path.end());
return newPath;
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayerForPath(Path const& path) const {
return directoryLayer;
}
Path newRelativePath(newAbsolutePath.begin() + directoryLayerPath.size(), newAbsolutePath.end());
return directoryLayer->move(tr, getPartitionSubpath(Path(), directoryLayer), newRelativePath);
}
Future<Void> DirectorySubspace::remove(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->remove(tr, getPartitionSubpath(path, directoryLayer));
}
Future<bool> DirectorySubspace::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->removeIfExists(tr, getPartitionSubpath(path, directoryLayer));
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayer() {
return directoryLayer;
}
const Standalone<StringRef> DirectorySubspace::getLayer() const {
return layer;
}
const IDirectory::Path DirectorySubspace::getPath() const {
return path;
}
IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path,
Reference<DirectoryLayer> directoryLayer) const {
if (!directoryLayer) {
directoryLayer = this->directoryLayer;
}
Path newPath(this->path.begin() + directoryLayer->getPath().size(), this->path.end());
newPath.insert(newPath.end(), path.begin(), path.end());
return newPath;
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayerForPath(Path const& path) const {
return directoryLayer;
}
} // namespace FDB

64
bindings/flow/DirectorySubspace.h Executable file → Normal file
View File

@ -28,39 +28,53 @@
#include "Subspace.h"
namespace FDB {
class DirectorySubspace : public IDirectory, public Subspace {
class DirectorySubspace : public IDirectory, public Subspace {
public:
DirectorySubspace(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> directorLayer, Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual ~DirectorySubspace() {}
public:
DirectorySubspace(Path const& path,
StringRef const& prefix,
Reference<DirectoryLayer> directorLayer,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual ~DirectorySubspace() {}
virtual Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
virtual Future<Reference<DirectorySubspace>> create(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<Reference<DirectorySubspace>> createOrOpen(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath);
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath);
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Reference<DirectoryLayer> getDirectoryLayer();
virtual const Standalone<StringRef> getLayer() const;
virtual const Path getPath() const;
virtual Reference<DirectoryLayer> getDirectoryLayer();
virtual const Standalone<StringRef> getLayer() const;
virtual const Path getPath() const;
protected:
Reference<DirectoryLayer> directoryLayer;
Path path;
Standalone<StringRef> layer;
protected:
Reference<DirectoryLayer> directoryLayer;
Path path;
Standalone<StringRef> layer;
virtual Path getPartitionSubpath(Path const& path, Reference<DirectoryLayer> directoryLayer = Reference<DirectoryLayer>()) const;
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const;
};
}
virtual Path getPartitionSubpath(Path const& path,
Reference<DirectoryLayer> directoryLayer = Reference<DirectoryLayer>()) const;
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const;
};
} // namespace FDB
#endif

576
bindings/flow/FDBLoanerTypes.h Executable file → Normal file
View File

@ -22,292 +22,304 @@
#define FDB_FLOW_LOANER_TYPES_H
namespace FDB {
typedef StringRef KeyRef;
typedef StringRef ValueRef;
typedef StringRef KeyRef;
typedef StringRef ValueRef;
typedef int64_t Version;
typedef int64_t Version;
typedef Standalone<KeyRef> Key;
typedef Standalone<ValueRef> Value;
typedef Standalone<KeyRef> Key;
typedef Standalone<ValueRef> Value;
inline Key keyAfter( const KeyRef& key ) {
if(key == LiteralStringRef("\xff\xff"))
return key;
inline Key keyAfter(const KeyRef& key) {
if (key == LiteralStringRef("\xff\xff"))
return key;
Standalone<StringRef> r;
uint8_t* s = new (r.arena()) uint8_t[ key.size() + 1 ];
memcpy(s, key.begin(), key.size() );
s[key.size()] = 0;
((StringRef&) r) = StringRef( s, key.size() + 1 );
return r;
}
inline KeyRef keyAfter( const KeyRef& key, Arena& arena ) {
if(key == LiteralStringRef("\xff\xff"))
return key;
uint8_t* t = new ( arena ) uint8_t[ key.size()+1 ];
memcpy(t, key.begin(), key.size() );
t[key.size()] = 0;
return KeyRef(t,key.size()+1);
}
struct KeySelectorRef {
KeyRef key; // Find the last item less than key
bool orEqual; // (or equal to key, if this is true)
int offset; // and then move forward this many items (or backward if negative)
KeySelectorRef() {}
KeySelectorRef( const KeyRef& key, bool orEqual, int offset ) : key(key), orEqual(orEqual), offset(offset) {}
KeySelectorRef( Arena& arena, const KeySelectorRef& copyFrom ) : key(arena,copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {}
int expectedSize() const { return key.expectedSize(); }
// std::string toString() const {
// if (offset > 0) {
// if (orEqual) return format("firstGreaterThan(%s)%+d", printable(key).c_str(), offset-1);
// else return format("firstGreaterOrEqual(%s)%+d", printable(key).c_str(), offset-1);
// } else {
// if (orEqual) return format("lastLessOrEqual(%s)%+d", printable(key).c_str(), offset);
// else return format("lastLessThan(%s)%+d", printable(key).c_str(), offset);
// }
// }
bool isBackward() const { return !orEqual && offset<=0; } // True if the resolution of the KeySelector depends only on keys less than key
bool isFirstGreaterOrEqual() const { return !orEqual && offset==1; }
bool isFirstGreaterThan() const { return orEqual && offset==1; }
bool isLastLessOrEqual() const { return orEqual && offset==0; }
// True iff, regardless of the contents of the database, lhs must resolve to a key > rhs
bool isDefinitelyGreater( KeyRef const& k ) {
return offset >= 1 && ( isFirstGreaterOrEqual() ? key > k : key >= k );
}
// True iff, regardless of the contents of the database, lhs must resolve to a key < rhs
bool isDefinitelyLess( KeyRef const& k ) {
return offset <= 0 && ( isLastLessOrEqual() ? key < k : key <= k );
}
template <class Ar>
void serialize( Ar& ar ) {
serializer(ar, key, orEqual, offset);
}
};
inline bool operator == (const KeySelectorRef& lhs, const KeySelectorRef& rhs) { return lhs.key == rhs.key && lhs.orEqual==rhs.orEqual && lhs.offset==rhs.offset; }
inline KeySelectorRef lastLessThan( const KeyRef& k ) {
return KeySelectorRef( k, false, 0 );
}
inline KeySelectorRef lastLessOrEqual( const KeyRef& k ) {
return KeySelectorRef( k, true, 0 );
}
inline KeySelectorRef firstGreaterThan( const KeyRef& k ) {
return KeySelectorRef( k, true, +1 );
}
inline KeySelectorRef firstGreaterOrEqual( const KeyRef& k ) {
return KeySelectorRef( k, false, +1 );
}
inline KeySelectorRef operator + (const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset+off);
}
inline KeySelectorRef operator - (const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset-off);
}
typedef Standalone<KeySelectorRef> KeySelector;
struct KeyValueRef {
KeyRef key;
ValueRef value;
KeyValueRef() {}
KeyValueRef( const KeyRef& key, const ValueRef& value ) : key(key), value(value) {}
KeyValueRef( Arena& a, const KeyValueRef& copyFrom ) : key(a, copyFrom.key), value(a, copyFrom.value) {}
bool operator == ( const KeyValueRef& r ) const { return key == r.key && value == r.value; }
int expectedSize() const { return key.expectedSize() + value.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) { serializer(ar, key, value); }
struct OrderByKey {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const {
return a.key < b.key;
}
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a < b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key < b;
}
};
struct OrderByKeyBack {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const {
return a.key > b.key;
}
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a > b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key > b;
}
};
};
typedef Standalone<KeyValueRef> KeyValue;
struct RangeResultRef : VectorRef<KeyValueRef> {
bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the limits requested)
// False implies that no such values remain
Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or beginning if reverse) of the range
// which was read to produce these results. This is guarenteed to be less than the requested range.
bool readToBegin;
bool readThroughEnd;
RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {}
RangeResultRef( Arena& p, const RangeResultRef& toCopy ) : more( toCopy.more ), readToBegin( toCopy.readToBegin ), readThroughEnd( toCopy.readThroughEnd ), readThrough( toCopy.readThrough.present() ? KeyRef( p, toCopy.readThrough.get() ) : Optional<KeyRef>() ), VectorRef<KeyValueRef>( p, toCopy ) {}
RangeResultRef( const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>() ) : VectorRef<KeyValueRef>( value ), more( more ), readThrough( readThrough ), readToBegin( false ), readThroughEnd( false ) {}
RangeResultRef( bool readToBegin, bool readThroughEnd ) : more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) { }
template <class Ar>
void serialize( Ar& ar ) {
serializer(ar, ((VectorRef<KeyValueRef>&)*this), more, readThrough, readToBegin, readThroughEnd);
}
};
struct GetRangeLimits {
enum { ROW_LIMIT_UNLIMITED = -1, BYTE_LIMIT_UNLIMITED = -1 };
int rows;
int minRows;
int bytes;
GetRangeLimits() : rows( ROW_LIMIT_UNLIMITED ), minRows(1), bytes( BYTE_LIMIT_UNLIMITED ) {}
explicit GetRangeLimits( int rowLimit ) : rows( rowLimit ), minRows(1), bytes( BYTE_LIMIT_UNLIMITED ) {}
GetRangeLimits( int rowLimit, int byteLimit ) : rows( rowLimit ), minRows(1), bytes( byteLimit ) {}
void decrement( VectorRef<KeyValueRef> const& data );
void decrement( KeyValueRef const& data );
// True if either the row or byte limit has been reached
bool isReached();
// True if data would cause the row or byte limit to be reached
bool reachedBy( VectorRef<KeyValueRef> const& data );
bool hasByteLimit();
bool hasRowLimit();
bool hasSatisfiedMinRows();
bool isValid() { return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED)
&& (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED)
&& minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED); }
};
struct KeyRangeRef {
const KeyRef begin, end;
KeyRangeRef() {}
KeyRangeRef( const KeyRef& begin, const KeyRef& end ) : begin(begin), end(end) {
if( begin > end ) {
throw inverted_range();
}
}
KeyRangeRef( Arena& a, const KeyRangeRef& copyFrom ) : begin(a, copyFrom.begin), end(a, copyFrom.end) {}
bool operator == ( const KeyRangeRef& r ) const { return begin == r.begin && end == r.end; }
bool operator != ( const KeyRangeRef& r ) const { return begin != r.begin || end != r.end; }
bool contains( const KeyRef& key ) const { return begin <= key && key < end; }
bool contains( const KeyRangeRef& keys ) const { return begin <= keys.begin && keys.end <= end; }
bool intersects( const KeyRangeRef& keys ) const { return begin < keys.end && keys.begin < end; }
bool empty() const { return begin == end; }
Standalone<KeyRangeRef> withPrefix( const StringRef& prefix ) const {
return KeyRangeRef( begin.withPrefix(prefix), end.withPrefix(prefix) );
}
const KeyRangeRef& operator = (const KeyRangeRef& rhs) {
const_cast<KeyRef&>(begin) = rhs.begin;
const_cast<KeyRef&>(end) = rhs.end;
return *this;
}
int expectedSize() const { return begin.expectedSize() + end.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) {
serializer(ar, const_cast<KeyRef&>(begin), const_cast<KeyRef&>(end));
if( begin > end ) {
throw inverted_range();
};
}
struct ArbitraryOrder {
bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const {
if (a.begin < b.begin) return true;
if (a.begin > b.begin) return false;
return a.end < b.end;
}
};
};
inline KeyRangeRef operator & (const KeyRangeRef& lhs, const KeyRangeRef& rhs) {
KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end);
if (e < b)
return KeyRangeRef();
return KeyRangeRef(b,e);
}
typedef Standalone<KeyRangeRef> KeyRange;
template <class T>
static std::string describe(T const& item) {
return item.toString();
}
template <class K, class V>
static std::string describe(std::map<K, V> const& items, int max_items = -1) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto it = items.begin(); it != items.end(); it++) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1) s += ",";
s += describe(it->first) + "=>" + describe(it->second);
}
return s;
}
template <class T>
static std::string describeList(T const& items, int max_items) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto const& item : items) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1) s += ",";
s += describe(item);
}
return s;
}
template <class T>
static std::string describe(std::vector<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T>
static std::string describe(std::set<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T1, class T2>
static std::string describe(std::pair<T1, T2> const& pair) {
return "first: " + describe(pair.first) + " second: " + describe(pair.second);
}
Standalone<StringRef> r;
uint8_t* s = new (r.arena()) uint8_t[key.size() + 1];
memcpy(s, key.begin(), key.size());
s[key.size()] = 0;
((StringRef&)r) = StringRef(s, key.size() + 1);
return r;
}
inline KeyRef keyAfter(const KeyRef& key, Arena& arena) {
if (key == LiteralStringRef("\xff\xff"))
return key;
uint8_t* t = new (arena) uint8_t[key.size() + 1];
memcpy(t, key.begin(), key.size());
t[key.size()] = 0;
return KeyRef(t, key.size() + 1);
}
struct KeySelectorRef {
KeyRef key; // Find the last item less than key
bool orEqual; // (or equal to key, if this is true)
int offset; // and then move forward this many items (or backward if negative)
KeySelectorRef() {}
KeySelectorRef(const KeyRef& key, bool orEqual, int offset) : key(key), orEqual(orEqual), offset(offset) {}
KeySelectorRef(Arena& arena, const KeySelectorRef& copyFrom)
: key(arena, copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {}
int expectedSize() const { return key.expectedSize(); }
// std::string toString() const {
// if (offset > 0) {
// if (orEqual) return format("firstGreaterThan(%s)%+d", printable(key).c_str(), offset-1);
// else return format("firstGreaterOrEqual(%s)%+d", printable(key).c_str(), offset-1);
// } else {
// if (orEqual) return format("lastLessOrEqual(%s)%+d", printable(key).c_str(), offset);
// else return format("lastLessThan(%s)%+d", printable(key).c_str(), offset);
// }
// }
bool isBackward() const {
return !orEqual && offset <= 0;
} // True if the resolution of the KeySelector depends only on keys less than key
bool isFirstGreaterOrEqual() const { return !orEqual && offset == 1; }
bool isFirstGreaterThan() const { return orEqual && offset == 1; }
bool isLastLessOrEqual() const { return orEqual && offset == 0; }
// True iff, regardless of the contents of the database, lhs must resolve to a key > rhs
bool isDefinitelyGreater(KeyRef const& k) { return offset >= 1 && (isFirstGreaterOrEqual() ? key > k : key >= k); }
// True iff, regardless of the contents of the database, lhs must resolve to a key < rhs
bool isDefinitelyLess(KeyRef const& k) { return offset <= 0 && (isLastLessOrEqual() ? key < k : key <= k); }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, key, orEqual, offset);
}
};
inline bool operator==(const KeySelectorRef& lhs, const KeySelectorRef& rhs) {
return lhs.key == rhs.key && lhs.orEqual == rhs.orEqual && lhs.offset == rhs.offset;
}
inline KeySelectorRef lastLessThan(const KeyRef& k) {
return KeySelectorRef(k, false, 0);
}
inline KeySelectorRef lastLessOrEqual(const KeyRef& k) {
return KeySelectorRef(k, true, 0);
}
inline KeySelectorRef firstGreaterThan(const KeyRef& k) {
return KeySelectorRef(k, true, +1);
}
inline KeySelectorRef firstGreaterOrEqual(const KeyRef& k) {
return KeySelectorRef(k, false, +1);
}
inline KeySelectorRef operator+(const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset + off);
}
inline KeySelectorRef operator-(const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset - off);
}
typedef Standalone<KeySelectorRef> KeySelector;
struct KeyValueRef {
KeyRef key;
ValueRef value;
KeyValueRef() {}
KeyValueRef(const KeyRef& key, const ValueRef& value) : key(key), value(value) {}
KeyValueRef(Arena& a, const KeyValueRef& copyFrom) : key(a, copyFrom.key), value(a, copyFrom.value) {}
bool operator==(const KeyValueRef& r) const { return key == r.key && value == r.value; }
int expectedSize() const { return key.expectedSize() + value.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) {
serializer(ar, key, value);
}
struct OrderByKey {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key < b.key; }
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a < b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key < b;
}
};
struct OrderByKeyBack {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key > b.key; }
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a > b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key > b;
}
};
};
typedef Standalone<KeyValueRef> KeyValue;
struct RangeResultRef : VectorRef<KeyValueRef> {
bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the
// limits requested)
// False implies that no such values remain
Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or
// beginning if reverse) of the range
// which was read to produce these results. This is guarenteed to be less than the requested range.
bool readToBegin;
bool readThroughEnd;
RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {}
RangeResultRef(Arena& p, const RangeResultRef& toCopy)
: more(toCopy.more), readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd),
readThrough(toCopy.readThrough.present() ? KeyRef(p, toCopy.readThrough.get()) : Optional<KeyRef>()),
VectorRef<KeyValueRef>(p, toCopy) {}
RangeResultRef(const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>())
: VectorRef<KeyValueRef>(value), more(more), readThrough(readThrough), readToBegin(false), readThroughEnd(false) {
}
RangeResultRef(bool readToBegin, bool readThroughEnd)
: more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, ((VectorRef<KeyValueRef>&)*this), more, readThrough, readToBegin, readThroughEnd);
}
};
struct GetRangeLimits {
enum { ROW_LIMIT_UNLIMITED = -1, BYTE_LIMIT_UNLIMITED = -1 };
int rows;
int minRows;
int bytes;
GetRangeLimits() : rows(ROW_LIMIT_UNLIMITED), minRows(1), bytes(BYTE_LIMIT_UNLIMITED) {}
explicit GetRangeLimits(int rowLimit) : rows(rowLimit), minRows(1), bytes(BYTE_LIMIT_UNLIMITED) {}
GetRangeLimits(int rowLimit, int byteLimit) : rows(rowLimit), minRows(1), bytes(byteLimit) {}
void decrement(VectorRef<KeyValueRef> const& data);
void decrement(KeyValueRef const& data);
// True if either the row or byte limit has been reached
bool isReached();
// True if data would cause the row or byte limit to be reached
bool reachedBy(VectorRef<KeyValueRef> const& data);
bool hasByteLimit();
bool hasRowLimit();
bool hasSatisfiedMinRows();
bool isValid() {
return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED) && (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED) &&
minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED);
}
};
struct KeyRangeRef {
const KeyRef begin, end;
KeyRangeRef() {}
KeyRangeRef(const KeyRef& begin, const KeyRef& end) : begin(begin), end(end) {
if (begin > end) {
throw inverted_range();
}
}
KeyRangeRef(Arena& a, const KeyRangeRef& copyFrom) : begin(a, copyFrom.begin), end(a, copyFrom.end) {}
bool operator==(const KeyRangeRef& r) const { return begin == r.begin && end == r.end; }
bool operator!=(const KeyRangeRef& r) const { return begin != r.begin || end != r.end; }
bool contains(const KeyRef& key) const { return begin <= key && key < end; }
bool contains(const KeyRangeRef& keys) const { return begin <= keys.begin && keys.end <= end; }
bool intersects(const KeyRangeRef& keys) const { return begin < keys.end && keys.begin < end; }
bool empty() const { return begin == end; }
Standalone<KeyRangeRef> withPrefix(const StringRef& prefix) const {
return KeyRangeRef(begin.withPrefix(prefix), end.withPrefix(prefix));
}
const KeyRangeRef& operator=(const KeyRangeRef& rhs) {
const_cast<KeyRef&>(begin) = rhs.begin;
const_cast<KeyRef&>(end) = rhs.end;
return *this;
}
int expectedSize() const { return begin.expectedSize() + end.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) {
serializer(ar, const_cast<KeyRef&>(begin), const_cast<KeyRef&>(end));
if (begin > end) {
throw inverted_range();
};
}
struct ArbitraryOrder {
bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const {
if (a.begin < b.begin)
return true;
if (a.begin > b.begin)
return false;
return a.end < b.end;
}
};
};
inline KeyRangeRef operator&(const KeyRangeRef& lhs, const KeyRangeRef& rhs) {
KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end);
if (e < b)
return KeyRangeRef();
return KeyRangeRef(b, e);
}
typedef Standalone<KeyRangeRef> KeyRange;
template <class T>
static std::string describe(T const& item) {
return item.toString();
}
template <class K, class V>
static std::string describe(std::map<K, V> const& items, int max_items = -1) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto it = items.begin(); it != items.end(); it++) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1)
s += ",";
s += describe(it->first) + "=>" + describe(it->second);
}
return s;
}
template <class T>
static std::string describeList(T const& items, int max_items) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto const& item : items) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1)
s += ",";
s += describe(item);
}
return s;
}
template <class T>
static std::string describe(std::vector<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T>
static std::string describe(std::set<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T1, class T2>
static std::string describe(std::pair<T1, T2> const& pair) {
return "first: " + describe(pair.first) + " second: " + describe(pair.second);
}
} // namespace FDB
#endif /* FDB_LOANER_TYPES_H */

152
bindings/flow/HighContentionAllocator.actor.cpp
View File

@ -21,90 +21,90 @@
#include "HighContentionAllocator.h"
namespace FDB {
ACTOR Future<Standalone<StringRef>> _allocate(Reference<Transaction> tr, Subspace counters, Subspace recent){
state int64_t start = 0;
state int64_t window = 0;
ACTOR Future<Standalone<StringRef>> _allocate(Reference<Transaction> tr, Subspace counters, Subspace recent) {
state int64_t start = 0;
state int64_t window = 0;
loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(counters.range(), 1, true, true));
if (range.size() > 0) {
start = counters.unpack(range[0].key).getInt(0);
}
state bool windowAdvanced = false;
loop {
// if thread safety is needed, this should be locked {
if (windowAdvanced) {
tr->clear(KeyRangeRef(counters.key(), counters.get(start).key()));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->clear(KeyRangeRef(recent.key(), recent.get(start).key()));
}
int64_t inc = 1;
tr->atomicOp(counters.get(start).key(), StringRef((uint8_t*)&inc, 8), FDB_MUTATION_TYPE_ADD);
Future<Optional<FDBStandalone<ValueRef>>> countFuture = tr->get(counters.get(start).key(), true);
// }
Optional<FDBStandalone<ValueRef>> countValue = wait(countFuture);
int64_t count = 0;
if (countValue.present()) {
if (countValue.get().size() != 8) {
throw invalid_directory_layer_metadata();
}
count = *(int64_t*)countValue.get().begin();
}
window = HighContentionAllocator::windowSize(start);
if (count * 2 < window) {
break;
}
start += window;
windowAdvanced = true;
}
loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(counters.range(), 1, true, true));
state int64_t candidate = deterministicRandom()->randomInt(start, start + window);
if(range.size() > 0) {
start = counters.unpack(range[0].key).getInt(0);
// if thread safety is needed, this should be locked {
state Future<FDBStandalone<RangeResultRef>> latestCounter = tr->getRange(counters.range(), 1, true, true);
state Future<Optional<FDBStandalone<ValueRef>>> candidateValue = tr->get(recent.get(candidate).key());
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->set(recent.get(candidate).key(), ValueRef());
// }
wait(success(latestCounter) && success(candidateValue));
int64_t currentWindowStart = 0;
if (latestCounter.get().size() > 0) {
currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0);
}
state bool windowAdvanced = false;
loop {
// if thread safety is needed, this should be locked {
if(windowAdvanced) {
tr->clear(KeyRangeRef(counters.key(), counters.get(start).key()));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->clear(KeyRangeRef(recent.key(), recent.get(start).key()));
}
int64_t inc = 1;
tr->atomicOp(counters.get(start).key(), StringRef((uint8_t*)&inc, 8), FDB_MUTATION_TYPE_ADD);
Future<Optional<FDBStandalone<ValueRef>>> countFuture = tr->get(counters.get(start).key(), true);
// }
Optional<FDBStandalone<ValueRef>> countValue = wait(countFuture);
int64_t count = 0;
if(countValue.present()) {
if(countValue.get().size() != 8) {
throw invalid_directory_layer_metadata();
}
count = *(int64_t*)countValue.get().begin();
}
window = HighContentionAllocator::windowSize(start);
if(count * 2 < window) {
break;
}
start += window;
windowAdvanced = true;
if (currentWindowStart > start) {
break;
}
loop {
state int64_t candidate = deterministicRandom()->randomInt(start, start + window);
// if thread safety is needed, this should be locked {
state Future<FDBStandalone<RangeResultRef>> latestCounter = tr->getRange(counters.range(), 1, true, true);
state Future<Optional<FDBStandalone<ValueRef>>> candidateValue = tr->get(recent.get(candidate).key());
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->set(recent.get(candidate).key(), ValueRef());
// }
wait(success(latestCounter) && success(candidateValue));
int64_t currentWindowStart = 0;
if(latestCounter.get().size() > 0) {
currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0);
}
if(currentWindowStart > start) {
break;
}
if(!candidateValue.get().present()) {
tr->addWriteConflictKey(recent.get(candidate).key());
return Tuple().append(candidate).pack();
}
if (!candidateValue.get().present()) {
tr->addWriteConflictKey(recent.get(candidate).key());
return Tuple().append(candidate).pack();
}
}
}
Future<Standalone<StringRef>> HighContentionAllocator::allocate(Reference<Transaction> const& tr) const {
return _allocate(tr, counters, recent);
}
int64_t HighContentionAllocator::windowSize(int64_t start) {
if (start < 255) {
return 64;
}
if (start < 65535) {
return 1024;
}
return 8192;
}
}
Future<Standalone<StringRef>> HighContentionAllocator::allocate(Reference<Transaction> const& tr) const {
return _allocate(tr, counters, recent);
}
int64_t HighContentionAllocator::windowSize(int64_t start) {
if (start < 255) {
return 64;
}
if (start < 65535) {
return 1024;
}
return 8192;
}
} // namespace FDB

21
bindings/flow/HighContentionAllocator.h
View File

@ -26,16 +26,17 @@
#include "Subspace.h"
namespace FDB {
class HighContentionAllocator {
public:
HighContentionAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {}
Future<Standalone<StringRef>> allocate(Reference<Transaction> const& tr) const;
class HighContentionAllocator {
public:
HighContentionAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {}
Future<Standalone<StringRef>> allocate(Reference<Transaction> const& tr) const;
static int64_t windowSize(int64_t start);
private:
Subspace counters;
Subspace recent;
};
}
static int64_t windowSize(int64_t start);
private:
Subspace counters;
Subspace recent;
};
} // namespace FDB
#endif

54
bindings/flow/IDirectory.h Executable file → Normal file
View File

@ -27,34 +27,46 @@
#include "bindings/flow/fdb_flow.h"
namespace FDB {
class DirectoryLayer;
class DirectorySubspace;
class DirectoryLayer;
class DirectorySubspace;
class IDirectory : public ReferenceCounted<IDirectory> {
public:
typedef std::vector<Standalone<StringRef>> Path;
class IDirectory : public ReferenceCounted<IDirectory> {
public:
typedef std::vector<Standalone<StringRef>> Path;
virtual Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()) = 0;
virtual Future<Reference<DirectorySubspace>> create(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()) = 0;
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<Reference<DirectorySubspace>> createOrOpen(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr,
Path const& path = Path()) = 0;
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) = 0;
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) = 0;
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) = 0;
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) = 0;
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Reference<DirectoryLayer> getDirectoryLayer() = 0;
virtual const Standalone<StringRef> getLayer() const = 0;
virtual const Path getPath() const = 0;
virtual Reference<DirectoryLayer> getDirectoryLayer() = 0;
virtual const Standalone<StringRef> getLayer() const = 0;
virtual const Path getPath() const = 0;
virtual ~IDirectory() {};
};
}
virtual ~IDirectory(){};
};
} // namespace FDB
#endif

73
bindings/flow/Node.actor.cpp Executable file → Normal file
View File

@ -21,50 +21,49 @@
#include "DirectoryLayer.h"
namespace FDB {
DirectoryLayer::Node::Node(Reference<DirectoryLayer> const& directoryLayer, Optional<Subspace> const& subspace, IDirectory::Path const& path, IDirectory::Path const& targetPath)
: directoryLayer(directoryLayer),
subspace(subspace),
path(path),
targetPath(targetPath),
loadedMetadata(false)
{ }
DirectoryLayer::Node::Node(Reference<DirectoryLayer> const& directoryLayer,
Optional<Subspace> const& subspace,
IDirectory::Path const& path,
IDirectory::Path const& targetPath)
: directoryLayer(directoryLayer), subspace(subspace), path(path), targetPath(targetPath), loadedMetadata(false) {}
bool DirectoryLayer::Node::exists() const {
return subspace.present();
}
bool DirectoryLayer::Node::exists() const {
return subspace.present();
}
ACTOR Future<DirectoryLayer::Node> loadMetadata(DirectoryLayer::Node *n, Reference<Transaction> tr) {
if(!n->exists()){
n->loadedMetadata = true;
return *n;
}
Optional<FDBStandalone<ValueRef>> layer = wait(tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY)));
n->layer = layer.present() ? layer.get() : Standalone<StringRef>();
ACTOR Future<DirectoryLayer::Node> loadMetadata(DirectoryLayer::Node* n, Reference<Transaction> tr) {
if (!n->exists()) {
n->loadedMetadata = true;
return *n;
}
//Calls to loadMetadata must keep the Node alive while the future is outstanding
Future<DirectoryLayer::Node> DirectoryLayer::Node::loadMetadata(Reference<Transaction> tr) {
return FDB::loadMetadata(this, tr);
}
Optional<FDBStandalone<ValueRef>> layer = wait(tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY)));
bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const {
ASSERT(loadedMetadata);
return exists() && layer == DirectoryLayer::PARTITION_LAYER && (includeEmptySubpath || targetPath.size() > path.size());
}
n->layer = layer.present() ? layer.get() : Standalone<StringRef>();
n->loadedMetadata = true;
IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const {
return Path(targetPath.begin() + path.size(), targetPath.end());
}
Reference<DirectorySubspace> DirectoryLayer::Node::getContents() const {
ASSERT(exists());
ASSERT(loadedMetadata);
return directoryLayer->contentsOfNode(subspace.get(), path, layer);
return *n;
}
// Calls to loadMetadata must keep the Node alive while the future is outstanding
Future<DirectoryLayer::Node> DirectoryLayer::Node::loadMetadata(Reference<Transaction> tr) {
return FDB::loadMetadata(this, tr);
}
bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const {
ASSERT(loadedMetadata);
return exists() && layer == DirectoryLayer::PARTITION_LAYER &&
(includeEmptySubpath || targetPath.size() > path.size());
}
IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const {
return Path(targetPath.begin() + path.size(), targetPath.end());
}
Reference<DirectorySubspace> DirectoryLayer::Node::getContents() const {
ASSERT(exists());
ASSERT(loadedMetadata);
return directoryLayer->contentsOfNode(subspace.get(), path, layer);
}
} // namespace FDB

117
bindings/flow/Subspace.cpp Executable file → Normal file
View File

@ -21,71 +21,72 @@
#include "Subspace.h"
namespace FDB {
Subspace::Subspace(Tuple const& tuple, StringRef const& rawPrefix){
StringRef packed = tuple.pack();
Subspace::Subspace(Tuple const& tuple, StringRef const& rawPrefix) {
StringRef packed = tuple.pack();
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + packed.size());
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
this->rawPrefix.append(this->rawPrefix.arena(), packed.begin(), packed.size());
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + packed.size());
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
this->rawPrefix.append(this->rawPrefix.arena(), packed.begin(), packed.size());
}
Subspace::Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix) {
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + tuple.pack().size());
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
this->rawPrefix.append(this->rawPrefix.arena(), tuple.pack().begin(), tuple.pack().size());
}
Subspace::Subspace(StringRef const& rawPrefix) {
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
}
Subspace::~Subspace() {}
Key Subspace::key() const {
return StringRef(rawPrefix.begin(), rawPrefix.size());
}
Key Subspace::pack(const Tuple& tuple) const {
return tuple.pack().withPrefix(StringRef(rawPrefix.begin(), rawPrefix.size()));
}
Tuple Subspace::unpack(StringRef const& key) const {
if (!contains(key)) {
throw key_not_in_subspace();
}
return Tuple::unpack(key.substr(rawPrefix.size()));
}
Subspace::Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix) {
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + tuple.pack().size());
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
this->rawPrefix.append(this->rawPrefix.arena(), tuple.pack().begin(), tuple.pack().size());
}
KeyRange Subspace::range(Tuple const& tuple) const {
VectorRef<uint8_t> begin;
VectorRef<uint8_t> end;
Subspace::Subspace(StringRef const& rawPrefix){
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
}
KeyRange keyRange;
Subspace::~Subspace() { }
begin.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
begin.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
begin.push_back(keyRange.arena(), uint8_t('\x00'));
Key Subspace::key() const {
return StringRef(rawPrefix.begin(), rawPrefix.size());
}
end.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
end.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
end.push_back(keyRange.arena(), uint8_t('\xff'));
Key Subspace::pack(const Tuple& tuple) const {
return tuple.pack().withPrefix(StringRef(rawPrefix.begin(), rawPrefix.size()));
}
// FIXME: test that this uses the keyRange arena and doesn't create another one
keyRange.KeyRangeRef::operator=(
KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size())));
return keyRange;
}
Tuple Subspace::unpack(StringRef const& key) const {
if (!contains(key)) {
throw key_not_in_subspace();
}
return Tuple::unpack(key.substr(rawPrefix.size()));
}
bool Subspace::contains(KeyRef const& key) const {
return key.startsWith(StringRef(rawPrefix.begin(), rawPrefix.size()));
}
KeyRange Subspace::range(Tuple const& tuple) const {
VectorRef<uint8_t> begin;
VectorRef<uint8_t> end;
Subspace Subspace::subspace(Tuple const& tuple) const {
return Subspace(tuple, rawPrefix);
}
KeyRange keyRange;
begin.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
begin.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
begin.push_back(keyRange.arena(), uint8_t('\x00'));
end.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
end.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
end.push_back(keyRange.arena(), uint8_t('\xff'));
// FIXME: test that this uses the keyRange arena and doesn't create another one
keyRange.KeyRangeRef::operator=(KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size())));
return keyRange;
}
bool Subspace::contains(KeyRef const& key) const {
return key.startsWith(StringRef(rawPrefix.begin(), rawPrefix.size()));
}
Subspace Subspace::subspace(Tuple const& tuple) const {
return Subspace(tuple, rawPrefix);
}
Subspace Subspace::get(Tuple const& tuple) const {
return subspace(tuple);
}
}
Subspace Subspace::get(Tuple const& tuple) const {
return subspace(tuple);
}
} // namespace FDB

98
bindings/flow/Subspace.h Executable file → Normal file
View File

@ -28,65 +28,65 @@
#include "Tuple.h"
namespace FDB {
class Subspace {
public:
Subspace(Tuple const& tuple = Tuple(), StringRef const& rawPrefix = StringRef());
Subspace(StringRef const& rawPrefix);
class Subspace {
public:
Subspace(Tuple const& tuple = Tuple(), StringRef const& rawPrefix = StringRef());
Subspace(StringRef const& rawPrefix);
virtual ~Subspace();
virtual ~Subspace();
virtual Key key() const;
virtual bool contains(KeyRef const& key) const;
virtual Key key() const;
virtual bool contains(KeyRef const& key) const;
virtual Key pack(Tuple const& tuple = Tuple()) const;
virtual Tuple unpack(KeyRef const& key) const;
virtual KeyRange range(Tuple const& tuple = Tuple()) const;
virtual Key pack(Tuple const& tuple = Tuple()) const;
virtual Tuple unpack(KeyRef const& key) const;
virtual KeyRange range(Tuple const& tuple = Tuple()) const;
template <class T>
Key pack(T const& item) const {
Tuple t;
t.append(item);
return pack(t);
}
template <class T>
Key pack(T const& item) const {
Tuple t;
t.append(item);
return pack(t);
}
Key packNested(Tuple const& item) const {
Tuple t;
t.appendNested(item);
return pack(t);
}
Key packNested(Tuple const& item) const {
Tuple t;
t.appendNested(item);
return pack(t);
}
Key pack(StringRef const& item, bool utf8=false) const {
Tuple t;
t.append(item, utf8);
return pack(t);
}
Key pack(StringRef const& item, bool utf8 = false) const {
Tuple t;
t.append(item, utf8);
return pack(t);
}
virtual Subspace subspace(Tuple const& tuple) const;
virtual Subspace get(Tuple const& tuple) const;
virtual Subspace subspace(Tuple const& tuple) const;
virtual Subspace get(Tuple const& tuple) const;
template <class T>
Subspace get(T const& item) const {
Tuple t;
t.append(item);
return get(t);
}
template <class T>
Subspace get(T const& item) const {
Tuple t;
t.append(item);
return get(t);
}
Subspace getNested(Tuple const& item) const {
Tuple t;
t.appendNested(item);
return get(t);
}
Subspace getNested(Tuple const& item) const {
Tuple t;
t.appendNested(item);
return get(t);
}
Subspace get(StringRef const& item, bool utf8=false) const {
Tuple t;
t.append(item, utf8);
return get(t);
}
Subspace get(StringRef const& item, bool utf8 = false) const {
Tuple t;
t.append(item, utf8);
return get(t);
}
private:
Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix);
Standalone<VectorRef<uint8_t>> rawPrefix;
};
}
private:
Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix);
Standalone<VectorRef<uint8_t>> rawPrefix;
};
} // namespace FDB
#endif

1147
bindings/flow/Tuple.cpp Executable file → Normal file
View File

File diff suppressed because it is too large Load Diff

143
bindings/flow/Tuple.h Executable file → Normal file
View File

@ -26,92 +26,93 @@
#include "bindings/flow/fdb_flow.h"
namespace FDB {
struct Uuid {
const static size_t SIZE;
struct Uuid {
const static size_t SIZE;
Uuid(StringRef const& data);
Uuid(StringRef const& data);
StringRef getData() const;
StringRef getData() const;
// Comparisons
bool operator==(Uuid const& other) const;
bool operator!=(Uuid const& other) const;
bool operator<(Uuid const& other) const;
bool operator<=(Uuid const& other) const;
bool operator>(Uuid const& other) const;
bool operator>=(Uuid const& other) const;
private:
Standalone<StringRef> data;
};
// Comparisons
bool operator==(Uuid const& other) const;
bool operator!=(Uuid const& other) const;
bool operator<(Uuid const& other) const;
bool operator<=(Uuid const& other) const;
bool operator>(Uuid const& other) const;
bool operator>=(Uuid const& other) const;
struct Tuple {
Tuple() {}
private:
Standalone<StringRef> data;
};
static Tuple unpack(StringRef const& str);
struct Tuple {
Tuple() {}
Tuple& append(Tuple const& tuple);
Tuple& append(StringRef const& str, bool utf8=false);
Tuple& append(int32_t);
Tuple& append(int64_t);
Tuple& append(bool);
Tuple& append(float);
Tuple& append(double);
Tuple& append(Uuid);
Tuple& appendNested(Tuple const&);
Tuple& appendNull();
static Tuple unpack(StringRef const& str);
StringRef pack() const { return StringRef(data.begin(), data.size()); }
Tuple& append(Tuple const& tuple);
Tuple& append(StringRef const& str, bool utf8 = false);
Tuple& append(int32_t);
Tuple& append(int64_t);
Tuple& append(bool);
Tuple& append(float);
Tuple& append(double);
Tuple& append(Uuid);
Tuple& appendNested(Tuple const&);
Tuple& appendNull();
template <typename T>
Tuple& operator<<(T const& t) {
return append(t);
}
StringRef pack() const { return StringRef(data.begin(), data.size()); }
enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED };
template <typename T>
Tuple& operator<<(T const& t) {
return append(t);
}
// this is number of elements, not length of data
size_t size() const { return offsets.size(); }
enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED };
ElementType getType(size_t index) const;
Standalone<StringRef> getString(size_t index) const;
int64_t getInt(size_t index) const;
bool getBool(size_t index) const;
float getFloat(size_t index) const;
double getDouble(size_t index) const;
Uuid getUuid(size_t index) const;
Tuple getNested(size_t index) const;
// this is number of elements, not length of data
size_t size() const { return offsets.size(); }
KeyRange range(Tuple const& tuple = Tuple()) const;
ElementType getType(size_t index) const;
Standalone<StringRef> getString(size_t index) const;
int64_t getInt(size_t index) const;
bool getBool(size_t index) const;
float getFloat(size_t index) const;
double getDouble(size_t index) const;
Uuid getUuid(size_t index) const;
Tuple getNested(size_t index) const;
Tuple subTuple(size_t beginIndex, size_t endIndex = std::numeric_limits<size_t>::max()) const;
KeyRange range(Tuple const& tuple = Tuple()) const;
// Comparisons
bool operator==(Tuple const& other) const;
bool operator!=(Tuple const& other) const;
bool operator<(Tuple const& other) const;
bool operator<=(Tuple const& other) const;
bool operator>(Tuple const& other) const;
bool operator>=(Tuple const& other) const;
Tuple subTuple(size_t beginIndex, size_t endIndex = std::numeric_limits<size_t>::max()) const;
private:
static const uint8_t NULL_CODE;
static const uint8_t BYTES_CODE;
static const uint8_t STRING_CODE;
static const uint8_t NESTED_CODE;
static const uint8_t INT_ZERO_CODE;
static const uint8_t POS_INT_END;
static const uint8_t NEG_INT_START;
static const uint8_t FLOAT_CODE;
static const uint8_t DOUBLE_CODE;
static const uint8_t FALSE_CODE;
static const uint8_t TRUE_CODE;
static const uint8_t UUID_CODE;
// Comparisons
bool operator==(Tuple const& other) const;
bool operator!=(Tuple const& other) const;
bool operator<(Tuple const& other) const;
bool operator<=(Tuple const& other) const;
bool operator>(Tuple const& other) const;
bool operator>=(Tuple const& other) const;
Tuple(const StringRef& data);
Tuple(Standalone<VectorRef<uint8_t>> data, std::vector<size_t> offsets);
Standalone<VectorRef<uint8_t>> data;
std::vector<size_t> offsets;
};
}
private:
static const uint8_t NULL_CODE;
static const uint8_t BYTES_CODE;
static const uint8_t STRING_CODE;
static const uint8_t NESTED_CODE;
static const uint8_t INT_ZERO_CODE;
static const uint8_t POS_INT_END;
static const uint8_t NEG_INT_START;
static const uint8_t FLOAT_CODE;
static const uint8_t DOUBLE_CODE;
static const uint8_t FALSE_CODE;
static const uint8_t TRUE_CODE;
static const uint8_t UUID_CODE;
Tuple(const StringRef& data);
Tuple(Standalone<VectorRef<uint8_t>> data, std::vector<size_t> offsets);
Standalone<VectorRef<uint8_t>> data;
std::vector<size_t> offsets;
};
} // namespace FDB
#endif /* _FDB_TUPLE_H_ */

778
bindings/flow/fdb_flow.actor.cpp
View File

@ -37,41 +37,42 @@ THREAD_FUNC networkThread(void* fdb) {
}
ACTOR Future<Void> _test() {
API *fdb = FDB::API::selectAPIVersion(700);
API* fdb = FDB::API::selectAPIVersion(700);
auto db = fdb->createDatabase();
state Reference<Transaction> tr = db->createTransaction();
// tr->setVersion(1);
Version ver = wait( tr->getReadVersion() );
Version ver = wait(tr->getReadVersion());
printf("%" PRId64 "\n", ver);
state std::vector< Future<Version> > versions;
state std::vector<Future<Version>> versions;
state double starttime = timer_monotonic();
state int i;
// for (i = 0; i < 100000; i++) {
// Version v = wait( tr->getReadVersion() );
// }
for ( i = 0; i < 100000; i++ ) {
versions.push_back( tr->getReadVersion() );
for (i = 0; i < 100000; i++) {
versions.push_back(tr->getReadVersion());
}
for ( i = 0; i < 100000; i++ ) {
Version v = wait( versions[i] );
for (i = 0; i < 100000; i++) {
Version v = wait(versions[i]);
}
// wait( waitForAllReady( versions ) );
printf("Elapsed: %lf\n", timer_monotonic() - starttime );
printf("Elapsed: %lf\n", timer_monotonic() - starttime);
tr->set( LiteralStringRef("foo"), LiteralStringRef("bar") );
tr->set(LiteralStringRef("foo"), LiteralStringRef("bar"));
Optional< FDBStandalone<ValueRef> > v = wait( tr->get( LiteralStringRef("foo") ) );
if ( v.present() ) {
printf("%s\n", v.get().toString().c_str() );
Optional<FDBStandalone<ValueRef>> v = wait(tr->get(LiteralStringRef("foo")));
if (v.present()) {
printf("%s\n", v.get().toString().c_str());
}
FDBStandalone<RangeResultRef> r = wait( tr->getRange( KeyRangeRef( LiteralStringRef("a"), LiteralStringRef("z") ), 100 ) );
FDBStandalone<RangeResultRef> r =
wait(tr->getRange(KeyRangeRef(LiteralStringRef("a"), LiteralStringRef("z")), 100));
for ( auto kv : r ) {
for (auto kv : r) {
printf("%s is %s\n", kv.key.toString().c_str(), kv.value.toString().c_str());
}
@ -80,7 +81,7 @@ ACTOR Future<Void> _test() {
}
void fdb_flow_test() {
API *fdb = FDB::API::selectAPIVersion(700);
API* fdb = FDB::API::selectAPIVersion(700);
fdb->setupNetwork();
startThread(networkThread, fdb);
@ -97,393 +98,430 @@ void fdb_flow_test() {
// FDB object used by bindings
namespace FDB {
class DatabaseImpl : public Database, NonCopyable {
public:
virtual ~DatabaseImpl() { fdb_database_destroy(db); }
class DatabaseImpl : public Database, NonCopyable {
public:
virtual ~DatabaseImpl() { fdb_database_destroy(db); }
Reference<Transaction> createTransaction() override;
void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) override;
Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
Future<Void> createSnapshot(const StringRef& uid, const StringRef& snap_command) override;
Reference<Transaction> createTransaction() override;
void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) override;
Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
Future<Void> createSnapshot(const StringRef& uid, const StringRef& snap_command) override;
private:
FDBDatabase* db;
explicit DatabaseImpl(FDBDatabase* db) : db(db) {}
private:
FDBDatabase* db;
explicit DatabaseImpl(FDBDatabase* db) : db(db) {}
friend class API;
};
friend class API;
};
class TransactionImpl : public Transaction, private NonCopyable, public FastAllocated<TransactionImpl> {
friend class DatabaseImpl;
class TransactionImpl : public Transaction, private NonCopyable, public FastAllocated<TransactionImpl> {
friend class DatabaseImpl;
public:
virtual ~TransactionImpl() {
if (tr) {
fdb_transaction_destroy(tr);
}
public:
virtual ~TransactionImpl() {
if (tr) {
fdb_transaction_destroy(tr);
}
void setReadVersion(Version v) override;
Future<Version> getReadVersion() override;
Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) override;
Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) override;
Future<Void> watch(const Key& key) override;
using Transaction::getRange;
Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin, const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(), bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override;
Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) override;
Future<FDBStandalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) override;
void addReadConflictRange(KeyRangeRef const& keys) override;
void addReadConflictKey(KeyRef const& key) override;
void addWriteConflictRange(KeyRangeRef const& keys) override;
void addWriteConflictKey(KeyRef const& key) override;
void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) override;
void set(const KeyRef& key, const ValueRef& value) override;
void clear(const KeyRangeRef& range) override;
void clear(const KeyRef& key) override;
Future<Void> commit() override;
Version getCommittedVersion() override;
Future<FDBStandalone<StringRef>> getVersionstamp() override;
void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Future<int64_t> getApproximateSize() override;
Future<Void> onError(Error const& e) override;
void cancel() override;
void reset() override;
TransactionImpl() : tr(nullptr) {}
TransactionImpl(TransactionImpl&& r) noexcept {
tr = r.tr;
r.tr = nullptr;
}
TransactionImpl& operator=(TransactionImpl&& r) noexcept {
tr = r.tr;
r.tr = nullptr;
return *this;
}
private:
FDBTransaction* tr;
explicit TransactionImpl(FDBDatabase* db);
};
static inline void throw_on_error( fdb_error_t e ) {
if (e)
throw Error(e);
}
void CFuture::blockUntilReady() {
throw_on_error( fdb_future_block_until_ready( f ) );
void setReadVersion(Version v) override;
Future<Version> getReadVersion() override;
Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) override;
Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) override;
Future<Void> watch(const Key& key) override;
using Transaction::getRange;
Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override;
Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) override;
Future<FDBStandalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) override;
void addReadConflictRange(KeyRangeRef const& keys) override;
void addReadConflictKey(KeyRef const& key) override;
void addWriteConflictRange(KeyRangeRef const& keys) override;
void addWriteConflictKey(KeyRef const& key) override;
void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) override;
void set(const KeyRef& key, const ValueRef& value) override;
void clear(const KeyRangeRef& range) override;
void clear(const KeyRef& key) override;
Future<Void> commit() override;
Version getCommittedVersion() override;
Future<FDBStandalone<StringRef>> getVersionstamp() override;
void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Future<int64_t> getApproximateSize() override;
Future<Void> onError(Error const& e) override;
void cancel() override;
void reset() override;
TransactionImpl() : tr(nullptr) {}
TransactionImpl(TransactionImpl&& r) noexcept {
tr = r.tr;
r.tr = nullptr;
}
TransactionImpl& operator=(TransactionImpl&& r) noexcept {
tr = r.tr;
r.tr = nullptr;
return *this;
}
void backToFutureCallback( FDBFuture* f, void* data ) {
g_network->onMainThread( Promise<Void>((SAV<Void>*)data), TaskPriority::DefaultOnMainThread ); // SOMEDAY: think about this priority
}
private:
FDBTransaction* tr;
// backToFuture<Type>( FDBFuture*, (FDBFuture* -> Type) ) -> Future<Type>
// Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread)
// and converts it into a Future<T> (with callbacks working on this thread, cancellation etc).
// You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type
ACTOR template<class T, class Function> static Future<T> backToFuture( FDBFuture* _f, Function convertValue ) {
state Reference<CFuture> f( new CFuture(_f) );
explicit TransactionImpl(FDBDatabase* db);
};
Promise<Void> ready;
Future<Void> onReady = ready.getFuture();
static inline void throw_on_error(fdb_error_t e) {
if (e)
throw Error(e);
}
throw_on_error( fdb_future_set_callback( f->f, backToFutureCallback, ready.extractRawPointer() ) );
wait( onReady );
void CFuture::blockUntilReady() {
throw_on_error(fdb_future_block_until_ready(f));
}
return convertValue( f );
}
void backToFutureCallback(FDBFuture* f, void* data) {
g_network->onMainThread(Promise<Void>((SAV<Void>*)data),
TaskPriority::DefaultOnMainThread); // SOMEDAY: think about this priority
}
void API::setNetworkOption( FDBNetworkOption option, Optional<StringRef> value ) {
if ( value.present() )
throw_on_error( fdb_network_set_option( option, value.get().begin(), value.get().size() ) );
else
throw_on_error( fdb_network_set_option( option, nullptr, 0 ) );
}
// backToFuture<Type>( FDBFuture*, (FDBFuture* -> Type) ) -> Future<Type>
// Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread)
// and converts it into a Future<T> (with callbacks working on this thread, cancellation etc).
// You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type
ACTOR template <class T, class Function>
static Future<T> backToFuture(FDBFuture* _f, Function convertValue) {
state Reference<CFuture> f(new CFuture(_f));
API* API::instance = nullptr;
API::API(int version) : version(version) {}
Promise<Void> ready;
Future<Void> onReady = ready.getFuture();
API* API::selectAPIVersion(int apiVersion) {
if(API::instance) {
if(apiVersion != API::instance->version) {
throw api_version_already_set();
}
else {
return API::instance;
}
}
throw_on_error(fdb_future_set_callback(f->f, backToFutureCallback, ready.extractRawPointer()));
wait(onReady);
if(apiVersion < 500 || apiVersion > FDB_API_VERSION) {
throw api_version_not_supported();
}
return convertValue(f);
}
throw_on_error( fdb_select_api_version_impl(apiVersion, FDB_API_VERSION) );
void API::setNetworkOption(FDBNetworkOption option, Optional<StringRef> value) {
if (value.present())
throw_on_error(fdb_network_set_option(option, value.get().begin(), value.get().size()));
else
throw_on_error(fdb_network_set_option(option, nullptr, 0));
}
API::instance = new API(apiVersion);
return API::instance;
}
API* API::instance = nullptr;
API::API(int version) : version(version) {}
bool API::isAPIVersionSelected() {
return API::instance != nullptr;
}
API* API::getInstance() {
if(API::instance == nullptr) {
throw api_version_unset();
}
else {
API* API::selectAPIVersion(int apiVersion) {
if (API::instance) {
if (apiVersion != API::instance->version) {
throw api_version_already_set();
} else {
return API::instance;
}
}
void API::setupNetwork() {
throw_on_error( fdb_setup_network() );
if (apiVersion < 500 || apiVersion > FDB_API_VERSION) {
throw api_version_not_supported();
}
void API::runNetwork() {
throw_on_error( fdb_run_network() );
throw_on_error(fdb_select_api_version_impl(apiVersion, FDB_API_VERSION));
API::instance = new API(apiVersion);
return API::instance;
}
bool API::isAPIVersionSelected() {
return API::instance != nullptr;
}
API* API::getInstance() {
if (API::instance == nullptr) {
throw api_version_unset();
} else {
return API::instance;
}
}
void API::stopNetwork() {
throw_on_error( fdb_stop_network() );
void API::setupNetwork() {
throw_on_error(fdb_setup_network());
}
void API::runNetwork() {
throw_on_error(fdb_run_network());
}
void API::stopNetwork() {
throw_on_error(fdb_stop_network());
}
bool API::evaluatePredicate(FDBErrorPredicate pred, Error const& e) {
return fdb_error_predicate(pred, e.code());
}
Reference<Database> API::createDatabase(std::string const& connFilename) {
FDBDatabase* db;
throw_on_error(fdb_create_database(connFilename.c_str(), &db));
return Reference<Database>(new DatabaseImpl(db));
}
int API::getAPIVersion() const {
return version;
}
Reference<Transaction> DatabaseImpl::createTransaction() {
return Reference<Transaction>(new TransactionImpl(db));
}
void DatabaseImpl::setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value) {
if (value.present())
throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size()));
else
throw_on_error(fdb_database_set_option(db, option, nullptr, 0));
}
Future<int64_t> DatabaseImpl::rebootWorker(const StringRef& address, bool check, int duration) {
return backToFuture<int64_t>(fdb_database_reboot_worker(db, address.begin(), address.size(), check, duration),
[](Reference<CFuture> f) {
int64_t res;
throw_on_error(fdb_future_get_int64(f->f, &res));
return res;
});
}
Future<Void> DatabaseImpl::forceRecoveryWithDataLoss(const StringRef& dcid) {
return backToFuture<Void>(fdb_database_force_recovery_with_data_loss(db, dcid.begin(), dcid.size()),
[](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
Future<Void> DatabaseImpl::createSnapshot(const StringRef& uid, const StringRef& snap_command) {
return backToFuture<Void>(
fdb_database_create_snapshot(db, uid.begin(), uid.size(), snap_command.begin(), snap_command.size()),
[](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
TransactionImpl::TransactionImpl(FDBDatabase* db) {
throw_on_error(fdb_database_create_transaction(db, &tr));
}
void TransactionImpl::setReadVersion(Version v) {
fdb_transaction_set_read_version(tr, v);
}
Future<Version> TransactionImpl::getReadVersion() {
return backToFuture<Version>(fdb_transaction_get_read_version(tr), [](Reference<CFuture> f) {
Version value;
throw_on_error(fdb_future_get_int64(f->f, &value));
return value;
});
}
Future<Optional<FDBStandalone<ValueRef>>> TransactionImpl::get(const Key& key, bool snapshot) {
return backToFuture<Optional<FDBStandalone<ValueRef>>>(
fdb_transaction_get(tr, key.begin(), key.size(), snapshot), [](Reference<CFuture> f) {
fdb_bool_t present;
uint8_t const* value;
int value_length;
throw_on_error(fdb_future_get_value(f->f, &present, &value, &value_length));
if (present) {
return Optional<FDBStandalone<ValueRef>>(FDBStandalone<ValueRef>(f, ValueRef(value, value_length)));
} else {
return Optional<FDBStandalone<ValueRef>>();
}
});
}
Future<Void> TransactionImpl::watch(const Key& key) {
return backToFuture<Void>(fdb_transaction_watch(tr, key.begin(), key.size()), [](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
Future<FDBStandalone<KeyRef>> TransactionImpl::getKey(const KeySelector& key, bool snapshot) {
return backToFuture<FDBStandalone<KeyRef>>(
fdb_transaction_get_key(tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot),
[](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error(fdb_future_get_key(f->f, &key, &key_length));
return FDBStandalone<KeyRef>(f, KeyRef(key, key_length));
});
}
Future<FDBStandalone<RangeResultRef>> TransactionImpl::getRange(const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits,
bool snapshot,
bool reverse,
FDBStreamingMode streamingMode) {
// FIXME: iteration
return backToFuture<FDBStandalone<RangeResultRef>>(
fdb_transaction_get_range(tr,
begin.key.begin(),
begin.key.size(),
begin.orEqual,
begin.offset,
end.key.begin(),
end.key.size(),
end.orEqual,
end.offset,
limits.rows,
limits.bytes,
streamingMode,
1,
snapshot,
reverse),
[](Reference<CFuture> f) {
FDBKeyValue const* kv;
int count;
fdb_bool_t more;
throw_on_error(fdb_future_get_keyvalue_array(f->f, &kv, &count, &more));
return FDBStandalone<RangeResultRef>(f,
RangeResultRef(VectorRef<KeyValueRef>((KeyValueRef*)kv, count), more));
});
}
Future<int64_t> TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) {
return backToFuture<int64_t>(fdb_transaction_get_estimated_range_size_bytes(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()),
[](Reference<CFuture> f) {
int64_t bytes;
throw_on_error(fdb_future_get_int64(f->f, &bytes));
return bytes;
});
}
Future<FDBStandalone<VectorRef<KeyRef>>> TransactionImpl::getRangeSplitPoints(const KeyRange& range,
int64_t chunkSize) {
return backToFuture<FDBStandalone<VectorRef<KeyRef>>>(
fdb_transaction_get_range_split_points(
tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size(), chunkSize),
[](Reference<CFuture> f) {
FDBKey const* ks;
int count;
throw_on_error(fdb_future_get_key_array(f->f, &ks, &count));
return FDBStandalone<VectorRef<KeyRef>>(f, VectorRef<KeyRef>((KeyRef*)ks, count));
});
}
void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) {
throw_on_error(fdb_transaction_add_conflict_range(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ));
}
void TransactionImpl::addReadConflictKey(KeyRef const& key) {
return addReadConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) {
throw_on_error(fdb_transaction_add_conflict_range(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE));
}
void TransactionImpl::addWriteConflictKey(KeyRef const& key) {
return addWriteConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) {
fdb_transaction_atomic_op(tr, key.begin(), key.size(), operand.begin(), operand.size(), operationType);
}
void TransactionImpl::set(const KeyRef& key, const ValueRef& value) {
fdb_transaction_set(tr, key.begin(), key.size(), value.begin(), value.size());
}
void TransactionImpl::clear(const KeyRangeRef& range) {
fdb_transaction_clear_range(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size());
}
void TransactionImpl::clear(const KeyRef& key) {
fdb_transaction_clear(tr, key.begin(), key.size());
}
Future<Void> TransactionImpl::commit() {
return backToFuture<Void>(fdb_transaction_commit(tr), [](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
Version TransactionImpl::getCommittedVersion() {
Version v;
throw_on_error(fdb_transaction_get_committed_version(tr, &v));
return v;
}
Future<FDBStandalone<StringRef>> TransactionImpl::getVersionstamp() {
return backToFuture<FDBStandalone<KeyRef>>(fdb_transaction_get_versionstamp(tr), [](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error(fdb_future_get_key(f->f, &key, &key_length));
return FDBStandalone<StringRef>(f, StringRef(key, key_length));
});
}
void TransactionImpl::setOption(FDBTransactionOption option, Optional<StringRef> value) {
if (value.present()) {
throw_on_error(fdb_transaction_set_option(tr, option, value.get().begin(), value.get().size()));
} else {
throw_on_error(fdb_transaction_set_option(tr, option, nullptr, 0));
}
}
bool API::evaluatePredicate(FDBErrorPredicate pred, Error const& e) {
return fdb_error_predicate( pred, e.code() );
}
Future<int64_t> TransactionImpl::getApproximateSize() {
return backToFuture<int64_t>(fdb_transaction_get_approximate_size(tr), [](Reference<CFuture> f) {
int64_t size = 0;
throw_on_error(fdb_future_get_int64(f->f, &size));
return size;
});
}
Reference<Database> API::createDatabase(std::string const& connFilename) {
FDBDatabase *db;
throw_on_error(fdb_create_database(connFilename.c_str(), &db));
return Reference<Database>(new DatabaseImpl(db));
}
Future<Void> TransactionImpl::onError(Error const& e) {
return backToFuture<Void>(fdb_transaction_on_error(tr, e.code()), [](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
int API::getAPIVersion() const {
return version;
}
void TransactionImpl::cancel() {
fdb_transaction_cancel(tr);
}
Reference<Transaction> DatabaseImpl::createTransaction() {
return Reference<Transaction>(new TransactionImpl(db));
}
void TransactionImpl::reset() {
fdb_transaction_reset(tr);
}
void DatabaseImpl::setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value) {
if (value.present())
throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size()));
else
throw_on_error(fdb_database_set_option(db, option, nullptr, 0));
}
Future<int64_t> DatabaseImpl::rebootWorker(const StringRef &address, bool check, int duration) {
return backToFuture<int64_t>( fdb_database_reboot_worker(db, address.begin(), address.size(), check, duration), [](Reference<CFuture> f) {
int64_t res;
throw_on_error(fdb_future_get_int64( f->f, &res ) );
return res;
} );
}
Future<Void> DatabaseImpl::forceRecoveryWithDataLoss(const StringRef &dcid) {
return backToFuture< Void > ( fdb_database_force_recovery_with_data_loss(db, dcid.begin(), dcid.size()), [](Reference<CFuture> f){
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
});
}
Future<Void> DatabaseImpl::createSnapshot(const StringRef& uid, const StringRef& snap_command) {
return backToFuture<Void>(
fdb_database_create_snapshot(db, uid.begin(), uid.size(), snap_command.begin(), snap_command.size()),
[](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
TransactionImpl::TransactionImpl(FDBDatabase* db) {
throw_on_error(fdb_database_create_transaction(db, &tr));
}
void TransactionImpl::setReadVersion(Version v) {
fdb_transaction_set_read_version( tr, v );
}
Future<Version> TransactionImpl::getReadVersion() {
return backToFuture<Version>( fdb_transaction_get_read_version( tr ), [](Reference<CFuture> f){
Version value;
throw_on_error( fdb_future_get_int64( f->f, &value ) );
return value;
} );
}
Future<Optional<FDBStandalone<ValueRef>>> TransactionImpl::get(const Key& key, bool snapshot) {
return backToFuture< Optional<FDBStandalone<ValueRef>> >( fdb_transaction_get( tr, key.begin(), key.size(), snapshot ), [](Reference<CFuture> f) {
fdb_bool_t present;
uint8_t const* value;
int value_length;
throw_on_error( fdb_future_get_value( f->f, &present, &value, &value_length ) );
if ( present ) {
return Optional<FDBStandalone<ValueRef>>( FDBStandalone<ValueRef>( f, ValueRef( value, value_length ) ) );
} else {
return Optional<FDBStandalone<ValueRef>>();
}
} );
}
Future<Void> TransactionImpl::watch(const Key& key) {
return backToFuture< Void >( fdb_transaction_watch( tr, key.begin(), key.size() ), [](Reference<CFuture> f) {
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
} );
}
Future<FDBStandalone<KeyRef>> TransactionImpl::getKey(const KeySelector& key, bool snapshot) {
return backToFuture< FDBStandalone<KeyRef> >( fdb_transaction_get_key( tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot ), [](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error( fdb_future_get_key( f->f, &key, &key_length ) );
return FDBStandalone<KeyRef>( f, KeyRef( key, key_length ) );
} );
}
Future<FDBStandalone<RangeResultRef>> TransactionImpl::getRange(const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, bool snapshot, bool reverse, FDBStreamingMode streamingMode) {
// FIXME: iteration
return backToFuture< FDBStandalone<RangeResultRef> >( fdb_transaction_get_range( tr, begin.key.begin(), begin.key.size(), begin.orEqual, begin.offset, end.key.begin(), end.key.size(), end.orEqual, end.offset, limits.rows, limits.bytes, streamingMode, 1, snapshot, reverse ), [](Reference<CFuture> f) {
FDBKeyValue const* kv;
int count;
fdb_bool_t more;
throw_on_error( fdb_future_get_keyvalue_array( f->f, &kv, &count, &more ) );
return FDBStandalone<RangeResultRef>( f, RangeResultRef( VectorRef<KeyValueRef>( (KeyValueRef*)kv, count ), more ) );
} );
}
Future<int64_t> TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) {
return backToFuture<int64_t>(fdb_transaction_get_estimated_range_size_bytes(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()), [](Reference<CFuture> f) {
int64_t bytes;
throw_on_error(fdb_future_get_int64(f->f, &bytes));
return bytes;
});
}
Future<FDBStandalone<VectorRef<KeyRef>>> TransactionImpl::getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) {
return backToFuture<FDBStandalone<VectorRef<KeyRef>>>(fdb_transaction_get_range_split_points(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size(), chunkSize), [](Reference<CFuture> f) {
FDBKey const* ks;
int count;
throw_on_error(fdb_future_get_key_array(f->f, &ks, &count));
return FDBStandalone<VectorRef<KeyRef>>(f, VectorRef<KeyRef>((KeyRef*)ks, count));
});
}
void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) {
throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ ) );
}
void TransactionImpl::addReadConflictKey(KeyRef const& key) {
return addReadConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) {
throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE ) );
}
void TransactionImpl::addWriteConflictKey(KeyRef const& key) {
return addWriteConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) {
fdb_transaction_atomic_op( tr, key.begin(), key.size(), operand.begin(), operand.size(), operationType );
}
void TransactionImpl::set(const KeyRef& key, const ValueRef& value) {
fdb_transaction_set( tr, key.begin(), key.size(), value.begin(), value.size() );
}
void TransactionImpl::clear(const KeyRangeRef& range) {
fdb_transaction_clear_range( tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size() );
}
void TransactionImpl::clear(const KeyRef& key) {
fdb_transaction_clear( tr, key.begin(), key.size() );
}
Future<Void> TransactionImpl::commit() {
return backToFuture< Void >( fdb_transaction_commit( tr ), [](Reference<CFuture> f) {
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
} );
}
Version TransactionImpl::getCommittedVersion() {
Version v;
throw_on_error( fdb_transaction_get_committed_version( tr, &v ) );
return v;
}
Future<FDBStandalone<StringRef>> TransactionImpl::getVersionstamp() {
return backToFuture<FDBStandalone<KeyRef>>(fdb_transaction_get_versionstamp(tr), [](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error( fdb_future_get_key( f->f, &key, &key_length ) );
return FDBStandalone<StringRef>( f, StringRef( key, key_length ) );
});
}
void TransactionImpl::setOption(FDBTransactionOption option, Optional<StringRef> value) {
if ( value.present() ) {
throw_on_error( fdb_transaction_set_option( tr, option, value.get().begin(), value.get().size() ) );
} else {
throw_on_error( fdb_transaction_set_option( tr, option, nullptr, 0 ) );
}
}
Future<int64_t> TransactionImpl::getApproximateSize() {
return backToFuture<int64_t>(fdb_transaction_get_approximate_size(tr), [](Reference<CFuture> f) {
int64_t size = 0;
throw_on_error(fdb_future_get_int64(f->f, &size));
return size;
});
}
Future<Void> TransactionImpl::onError(Error const& e) {
return backToFuture< Void >( fdb_transaction_on_error( tr, e.code() ), [](Reference<CFuture> f) {
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
} );
}
void TransactionImpl::cancel() {
fdb_transaction_cancel( tr );
}
void TransactionImpl::reset() {
fdb_transaction_reset( tr );
}
} // namespace FDB
} // namespace FDB

214
bindings/flow/fdb_flow.h
View File

@ -30,127 +30,143 @@
#include "FDBLoanerTypes.h"
namespace FDB {
struct CFuture : NonCopyable, ReferenceCounted<CFuture>, FastAllocated<CFuture> {
CFuture() : f(nullptr) {}
explicit CFuture(FDBFuture* f) : f(f) {}
~CFuture() {
if (f) {
fdb_future_destroy(f);
}
struct CFuture : NonCopyable, ReferenceCounted<CFuture>, FastAllocated<CFuture> {
CFuture() : f(nullptr) {}
explicit CFuture(FDBFuture* f) : f(f) {}
~CFuture() {
if (f) {
fdb_future_destroy(f);
}
}
void blockUntilReady();
void blockUntilReady();
FDBFuture* f;
};
FDBFuture* f;
};
template <class T>
class FDBStandalone : public T {
public:
FDBStandalone() {}
FDBStandalone(Reference<CFuture> f, T const& t) : T(t), f(f) {}
FDBStandalone(FDBStandalone const& o) : T((T const&)o), f(o.f) {}
template <class T>
class FDBStandalone : public T {
public:
FDBStandalone() {}
FDBStandalone(Reference<CFuture> f, T const& t) : T(t), f(f) {}
FDBStandalone(FDBStandalone const& o) : T((T const&)o), f(o.f) {}
private:
Reference<CFuture> f;
};
private:
Reference<CFuture> f;
};
class ReadTransaction : public ReferenceCounted<ReadTransaction> {
public:
virtual ~ReadTransaction(){};
virtual void setReadVersion(Version v) = 0;
virtual Future<Version> getReadVersion() = 0;
class ReadTransaction : public ReferenceCounted<ReadTransaction> {
public:
virtual ~ReadTransaction(){};
virtual void setReadVersion(Version v) = 0;
virtual Future<Version> getReadVersion() = 0;
virtual Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) = 0;
virtual Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) = 0;
virtual Future<Void> watch(const Key& key) = 0;
virtual Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) = 0;
virtual Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) = 0;
virtual Future<Void> watch(const Key& key) = 0;
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeySelector& begin, const KeySelector& end, GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false, bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0;
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeySelector& begin, const KeySelector& end, int limit, bool snapshot = false, bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeyRange& keys, int limit, bool snapshot = false, bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limit, snapshot, reverse,
streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeyRange& keys, GetRangeLimits limits = GetRangeLimits(), bool snapshot = false, bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limits, snapshot, reverse,
streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0;
virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
int limit,
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeyRange& keys,
int limit,
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limit,
snapshot,
reverse,
streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeyRange& keys,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limits,
snapshot,
reverse,
streamingMode);
}
virtual Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) = 0;
virtual Future<FDBStandalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) = 0;
virtual Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) = 0;
virtual Future<FDBStandalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) = 0;
virtual void addReadConflictRange(KeyRangeRef const& keys) = 0;
virtual void addReadConflictKey(KeyRef const& key) = 0;
virtual void addReadConflictRange(KeyRangeRef const& keys) = 0;
virtual void addReadConflictKey(KeyRef const& key) = 0;
virtual void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual Future<Void> onError(Error const& e) = 0;
virtual Future<Void> onError(Error const& e) = 0;
virtual void cancel() = 0;
virtual void reset() = 0;
};
virtual void cancel() = 0;
virtual void reset() = 0;
};
class Transaction : public ReadTransaction {
public:
virtual void addWriteConflictRange(KeyRangeRef const& keys) = 0;
virtual void addWriteConflictKey(KeyRef const& key) = 0;
class Transaction : public ReadTransaction {
public:
virtual void addWriteConflictRange(KeyRangeRef const& keys) = 0;
virtual void addWriteConflictKey(KeyRef const& key) = 0;
virtual void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) = 0;
virtual void set(const KeyRef& key, const ValueRef& value) = 0;
virtual void clear(const KeyRangeRef& range) = 0;
virtual void clear(const KeyRef& key) = 0;
virtual void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) = 0;
virtual void set(const KeyRef& key, const ValueRef& value) = 0;
virtual void clear(const KeyRangeRef& range) = 0;
virtual void clear(const KeyRef& key) = 0;
virtual Future<Void> commit() = 0;
virtual Version getCommittedVersion() = 0;
virtual Future<int64_t> getApproximateSize() = 0;
virtual Future<FDBStandalone<StringRef>> getVersionstamp() = 0;
};
virtual Future<Void> commit() = 0;
virtual Version getCommittedVersion() = 0;
virtual Future<int64_t> getApproximateSize() = 0;
virtual Future<FDBStandalone<StringRef>> getVersionstamp() = 0;
};
class Database : public ReferenceCounted<Database> {
public:
virtual ~Database(){};
virtual Reference<Transaction> createTransaction() = 0;
virtual void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) = 0;
virtual Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0;
virtual Future<Void> createSnapshot(const StringRef& uid, const StringRef& snap_command) = 0;
};
class Database : public ReferenceCounted<Database> {
public:
virtual ~Database(){};
virtual Reference<Transaction> createTransaction() = 0;
virtual void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) = 0;
virtual Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0;
virtual Future<Void> createSnapshot(const StringRef& uid, const StringRef& snap_command) = 0;
};
class API {
public:
static API* selectAPIVersion(int apiVersion);
static API* getInstance();
static bool isAPIVersionSelected();
class API {
public:
static API* selectAPIVersion(int apiVersion);
static API* getInstance();
static bool isAPIVersionSelected();
void setNetworkOption(FDBNetworkOption option, Optional<StringRef> value = Optional<StringRef>());
void setNetworkOption(FDBNetworkOption option, Optional<StringRef> value = Optional<StringRef>());
void setupNetwork();
void runNetwork();
void stopNetwork();
void setupNetwork();
void runNetwork();
void stopNetwork();
Reference<Database> createDatabase(std::string const& connFilename = "");
Reference<Database> createDatabase(std::string const& connFilename = "");
bool evaluatePredicate(FDBErrorPredicate pred, Error const& e);
int getAPIVersion() const;
bool evaluatePredicate(FDBErrorPredicate pred, Error const& e);
int getAPIVersion() const;
private:
static API* instance;
private:
static API* instance;
API(int version);
int version;
};
} // namespace FDB
API(int version);
int version;
};
} // namespace FDB
#endif // FDB_FLOW_FDB_FLOW_H

201
bindings/flow/tester/DirectoryTester.actor.cpp
View File

@ -19,14 +19,14 @@
*/
#include "Tester.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
using namespace FDB;
ACTOR Future<std::vector<Tuple>> popTuples(Reference<FlowTesterData> data, int count = 1) {
state std::vector<Tuple> tuples;
while(tuples.size() < count) {
while (tuples.size() < count) {
Standalone<StringRef> sizeStr = wait(data->stack.pop()[0].value);
int size = Tuple::unpack(sizeStr).getInt(0);
@ -34,7 +34,7 @@ ACTOR Future<std::vector<Tuple>> popTuples(Reference<FlowTesterData> data, int c
state Tuple tuple;
state int index;
for(index = 0; index < tupleItems.size(); ++index) {
for (index = 0; index < tupleItems.size(); ++index) {
Standalone<StringRef> itemStr = wait(tupleItems[index].value);
tuple.append(Tuple::unpack(itemStr));
}
@ -54,9 +54,9 @@ ACTOR Future<std::vector<IDirectory::Path>> popPaths(Reference<FlowTesterData> d
std::vector<Tuple> tuples = wait(popTuples(data, count));
std::vector<IDirectory::Path> paths;
for(auto &tuple : tuples) {
for (auto& tuple : tuples) {
IDirectory::Path path;
for(int i = 0; i < tuple.size(); ++i) {
for (int i = 0; i < tuple.size(); ++i) {
path.push_back(tuple.getString(i));
}
@ -74,9 +74,9 @@ ACTOR Future<IDirectory::Path> popPath(Reference<FlowTesterData> data) {
std::string pathToString(IDirectory::Path const& path) {
std::string str;
str += "[";
for(int i = 0; i < path.size(); ++i) {
for (int i = 0; i < path.size(); ++i) {
str += path[i].toString();
if(i < path.size() - 1) {
if (i < path.size() - 1) {
str += ", ";
}
}
@ -86,21 +86,21 @@ std::string pathToString(IDirectory::Path const& path) {
IDirectory::Path combinePaths(IDirectory::Path const& path1, IDirectory::Path const& path2) {
IDirectory::Path outPath(path1.begin(), path1.end());
for(auto p : path2) {
for (auto p : path2) {
outPath.push_back(p);
}
return outPath;
}
void logOp(std::string message, bool force=false) {
if(LOG_OPS || force) {
void logOp(std::string message, bool force = false) {
if (LOG_OPS || force) {
printf("%s\n", message.c_str());
fflush(stdout);
}
}
//DIRECTORY_CREATE_SUBSPACE
// DIRECTORY_CREATE_SUBSPACE
struct DirectoryCreateSubspaceFunc : InstructionFunc {
static const char* name;
@ -108,7 +108,8 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc {
state Tuple path = wait(popTuple(data));
Tuple rawPrefix = wait(data->stack.waitAndPop());
logOp(format("Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str()));
logOp(format(
"Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str()));
data->directoryData.push(new Subspace(path, rawPrefix.getString(0)));
return Void();
}
@ -116,7 +117,7 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc {
const char* DirectoryCreateSubspaceFunc::name = "DIRECTORY_CREATE_SUBSPACE";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateSubspaceFunc);
//DIRECTORY_CREATE_LAYER
// DIRECTORY_CREATE_LAYER
struct DirectoryCreateLayerFunc : InstructionFunc {
static const char* name;
@ -127,15 +128,21 @@ struct DirectoryCreateLayerFunc : InstructionFunc {
int index2 = args[1].getInt(0);
bool allowManualPrefixes = args[2].getInt(0) != 0;
if(!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) {
if (!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) {
logOp("Create directory layer: None");
data->directoryData.push();
}
else {
} else {
Subspace* nodeSubspace = data->directoryData.directoryList[index1].subspace.get();
Subspace* contentSubspace = data->directoryData.directoryList[index2].subspace.get();
logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, allow_manual_prefixes = %d", index1, nodeSubspace->key().printable().c_str(), index2, nodeSubspace->key().printable().c_str(), allowManualPrefixes));
data->directoryData.push(Reference<IDirectory>(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes)));
logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, "
"allow_manual_prefixes = %d",
index1,
nodeSubspace->key().printable().c_str(),
index2,
nodeSubspace->key().printable().c_str(),
allowManualPrefixes));
data->directoryData.push(
Reference<IDirectory>(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes)));
}
return Void();
@ -144,7 +151,7 @@ struct DirectoryCreateLayerFunc : InstructionFunc {
const char* DirectoryCreateLayerFunc::name = "DIRECTORY_CREATE_LAYER";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateLayerFunc);
//DIRECTORY_CHANGE
// DIRECTORY_CHANGE
struct DirectoryChangeFunc : InstructionFunc {
static const char* name;
@ -153,13 +160,17 @@ struct DirectoryChangeFunc : InstructionFunc {
data->directoryData.directoryListIndex = index.getInt(0);
ASSERT(data->directoryData.directoryListIndex < data->directoryData.directoryList.size());
if(!data->directoryData.directoryList[data->directoryData.directoryListIndex].valid()) {
if (!data->directoryData.directoryList[data->directoryData.directoryListIndex].valid()) {
data->directoryData.directoryListIndex = data->directoryData.directoryErrorIndex;
}
if(LOG_DIRS) {
if (LOG_DIRS) {
DirectoryOrSubspace d = data->directoryData.directoryList[data->directoryData.directoryListIndex];
printf("Changed directory to %d (%s @\'%s\')\n", data->directoryData.directoryListIndex, d.typeString().c_str(), d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str() : d.subspace.get()->key().printable().c_str());
printf("Changed directory to %d (%s @\'%s\')\n",
data->directoryData.directoryListIndex,
d.typeString().c_str(),
d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str()
: d.subspace.get()->key().printable().c_str());
fflush(stdout);
}
@ -169,7 +180,7 @@ struct DirectoryChangeFunc : InstructionFunc {
const char* DirectoryChangeFunc::name = "DIRECTORY_CHANGE";
REGISTER_INSTRUCTION_FUNC(DirectoryChangeFunc);
//DIRECTORY_SET_ERROR_INDEX
// DIRECTORY_SET_ERROR_INDEX
struct DirectorySetErrorIndexFunc : InstructionFunc {
static const char* name;
@ -183,7 +194,7 @@ struct DirectorySetErrorIndexFunc : InstructionFunc {
const char* DirectorySetErrorIndexFunc::name = "DIRECTORY_SET_ERROR_INDEX";
REGISTER_INSTRUCTION_FUNC(DirectorySetErrorIndexFunc);
//DIRECTORY_CREATE_OR_OPEN
// DIRECTORY_CREATE_OR_OPEN
struct DirectoryCreateOrOpenFunc : InstructionFunc {
static const char* name;
@ -193,11 +204,12 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc {
Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("create_or_open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str()));
logOp(format("create_or_open %s: layer=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, layer] () {
return directory->createOrOpen(instruction->tr, path, layer);
}));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
instruction, [this, directory, layer]() { return directory->createOrOpen(instruction->tr, path, layer); }));
data->directoryData.push(dirSubspace);
@ -207,7 +219,7 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc {
const char* DirectoryCreateOrOpenFunc::name = "DIRECTORY_CREATE_OR_OPEN";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateOrOpenFunc);
//DIRECTORY_CREATE
// DIRECTORY_CREATE
struct DirectoryCreateFunc : InstructionFunc {
static const char* name;
@ -215,14 +227,19 @@ struct DirectoryCreateFunc : InstructionFunc {
state IDirectory::Path path = wait(popPath(data));
std::vector<Tuple> args = wait(data->stack.waitAndPop(2));
Standalone<StringRef> layer = args[0].getType(0) == Tuple::NULL_TYPE ? StringRef() : args[0].getString(0);
Optional<Standalone<StringRef>> prefix = args[1].getType(0) == Tuple::NULL_TYPE ? Optional<Standalone<StringRef>>() : args[1].getString(0);
Optional<Standalone<StringRef>> prefix =
args[1].getType(0) == Tuple::NULL_TYPE ? Optional<Standalone<StringRef>>() : args[1].getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("create %s: layer=%s, prefix=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str(), prefix.present() ? prefix.get().printable().c_str() : "<not present>"));
logOp(format("create %s: layer=%s, prefix=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str(),
prefix.present() ? prefix.get().printable().c_str() : "<not present>"));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, layer, prefix] () {
return directory->create(instruction->tr, path, layer, prefix);
}));
Reference<DirectorySubspace> dirSubspace =
wait(executeMutation(instruction, [this, directory, layer, prefix]() {
return directory->create(instruction->tr, path, layer, prefix);
}));
data->directoryData.push(dirSubspace);
@ -232,7 +249,7 @@ struct DirectoryCreateFunc : InstructionFunc {
const char* DirectoryCreateFunc::name = "DIRECTORY_CREATE";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateFunc);
//DIRECTORY_OPEN
// DIRECTORY_OPEN
struct DirectoryOpenFunc : InstructionFunc {
static const char* name;
@ -242,7 +259,9 @@ struct DirectoryOpenFunc : InstructionFunc {
Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str()));
logOp(format("open %s: layer=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str()));
Reference<DirectorySubspace> dirSubspace = wait(directory->open(instruction->tr, path, layer));
data->directoryData.push(dirSubspace);
@ -252,7 +271,7 @@ struct DirectoryOpenFunc : InstructionFunc {
const char* DirectoryOpenFunc::name = "DIRECTORY_OPEN";
REGISTER_INSTRUCTION_FUNC(DirectoryOpenFunc);
//DIRECTORY_MOVE
// DIRECTORY_MOVE
struct DirectoryMoveFunc : InstructionFunc {
static const char* name;
@ -260,11 +279,12 @@ struct DirectoryMoveFunc : InstructionFunc {
std::vector<IDirectory::Path> paths = wait(popPaths(data, 2));
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("move %s to %s", pathToString(combinePaths(directory->getPath(), paths[0])).c_str(), pathToString(combinePaths(directory->getPath(), paths[1])).c_str()));
logOp(format("move %s to %s",
pathToString(combinePaths(directory->getPath(), paths[0])).c_str(),
pathToString(combinePaths(directory->getPath(), paths[1])).c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, paths] () {
return directory->move(instruction->tr, paths[0], paths[1]);
}));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
instruction, [this, directory, paths]() { return directory->move(instruction->tr, paths[0], paths[1]); }));
data->directoryData.push(dirSubspace);
@ -274,7 +294,7 @@ struct DirectoryMoveFunc : InstructionFunc {
const char* DirectoryMoveFunc::name = "DIRECTORY_MOVE";
REGISTER_INSTRUCTION_FUNC(DirectoryMoveFunc);
//DIRECTORY_MOVE_TO
// DIRECTORY_MOVE_TO
struct DirectoryMoveToFunc : InstructionFunc {
static const char* name;
@ -284,9 +304,8 @@ struct DirectoryMoveToFunc : InstructionFunc {
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("move %s to %s", pathToString(directory->getPath()).c_str(), pathToString(path).c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, path] () {
return directory->moveTo(instruction->tr, path);
}));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
instruction, [this, directory, path]() { return directory->moveTo(instruction->tr, path); }));
data->directoryData.push(dirSubspace);
@ -296,27 +315,22 @@ struct DirectoryMoveToFunc : InstructionFunc {
const char* DirectoryMoveToFunc::name = "DIRECTORY_MOVE_TO";
REGISTER_INSTRUCTION_FUNC(DirectoryMoveToFunc);
//DIRECTORY_REMOVE
// DIRECTORY_REMOVE
struct DirectoryRemoveFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory();
if(count.getInt(0) == 0) {
if (count.getInt(0) == 0) {
logOp(format("remove %s", pathToString(directory->getPath()).c_str()));
wait(executeMutation(instruction, [this] () {
return directory->remove(instruction->tr);
}));
}
else {
wait(executeMutation(instruction, [this]() { return directory->remove(instruction->tr); }));
} else {
IDirectory::Path path = wait(popPath(data));
logOp(format("remove %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
wait(executeMutation(instruction, [this, path] () {
return directory->remove(instruction->tr, path);
}));
wait(executeMutation(instruction, [this, path]() { return directory->remove(instruction->tr, path); }));
}
return Void();
@ -325,27 +339,24 @@ struct DirectoryRemoveFunc : InstructionFunc {
const char* DirectoryRemoveFunc::name = "DIRECTORY_REMOVE";
REGISTER_INSTRUCTION_FUNC(DirectoryRemoveFunc);
//DIRECTORY_REMOVE_IF_EXISTS
// DIRECTORY_REMOVE_IF_EXISTS
struct DirectoryRemoveIfExistsFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory();
if(count.getInt(0) == 0) {
if (count.getInt(0) == 0) {
logOp(format("remove_if_exists %s", pathToString(directory->getPath()).c_str()));
wait(success(executeMutation(instruction, [this] () {
return directory->removeIfExists(instruction->tr);
})));
}
else {
wait(
success(executeMutation(instruction, [this]() { return directory->removeIfExists(instruction->tr); })));
} else {
IDirectory::Path path = wait(popPath(data));
logOp(format("remove_if_exists %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
wait(success(executeMutation(instruction, [this, path] () {
return directory->removeIfExists(instruction->tr, path);
})));
wait(success(executeMutation(instruction,
[this, path]() { return directory->removeIfExists(instruction->tr, path); })));
}
return Void();
@ -354,7 +365,7 @@ struct DirectoryRemoveIfExistsFunc : InstructionFunc {
const char* DirectoryRemoveIfExistsFunc::name = "DIRECTORY_REMOVE_IF_EXISTS";
REGISTER_INSTRUCTION_FUNC(DirectoryRemoveIfExistsFunc);
//DIRECTORY_LIST
// DIRECTORY_LIST
struct DirectoryListFunc : InstructionFunc {
static const char* name;
@ -362,12 +373,11 @@ struct DirectoryListFunc : InstructionFunc {
Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory();
state Standalone<VectorRef<StringRef>> subdirs;
if(count.getInt(0) == 0) {
if (count.getInt(0) == 0) {
logOp(format("list %s", pathToString(directory->getPath()).c_str()));
Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr));
subdirs = _subdirs;
}
else {
} else {
IDirectory::Path path = wait(popPath(data));
logOp(format("list %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr, path));
@ -375,7 +385,7 @@ struct DirectoryListFunc : InstructionFunc {
}
Tuple subdirTuple;
for(auto &sd : subdirs) {
for (auto& sd : subdirs) {
subdirTuple.append(sd, true);
}
@ -386,7 +396,7 @@ struct DirectoryListFunc : InstructionFunc {
const char* DirectoryListFunc::name = "DIRECTORY_LIST";
REGISTER_INSTRUCTION_FUNC(DirectoryListFunc);
//DIRECTORY_EXISTS
// DIRECTORY_EXISTS
struct DirectoryExistsFunc : InstructionFunc {
static const char* name;
@ -394,12 +404,11 @@ struct DirectoryExistsFunc : InstructionFunc {
Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory();
state bool result;
if(count.getInt(0) == 0) {
if (count.getInt(0) == 0) {
bool _result = wait(directory->exists(instruction->tr));
result = _result;
logOp(format("exists %s: %d", pathToString(directory->getPath()).c_str(), result));
}
else {
} else {
state IDirectory::Path path = wait(popPath(data));
bool _result = wait(directory->exists(instruction->tr, path));
result = _result;
@ -413,7 +422,7 @@ struct DirectoryExistsFunc : InstructionFunc {
const char* DirectoryExistsFunc::name = "DIRECTORY_EXISTS";
REGISTER_INSTRUCTION_FUNC(DirectoryExistsFunc);
//DIRECTORY_PACK_KEY
// DIRECTORY_PACK_KEY
struct DirectoryPackKeyFunc : InstructionFunc {
static const char* name;
@ -427,17 +436,19 @@ struct DirectoryPackKeyFunc : InstructionFunc {
const char* DirectoryPackKeyFunc::name = "DIRECTORY_PACK_KEY";
REGISTER_INSTRUCTION_FUNC(DirectoryPackKeyFunc);
//DIRECTORY_UNPACK_KEY
// DIRECTORY_UNPACK_KEY
struct DirectoryUnpackKeyFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple key = wait(data->stack.waitAndPop());
Subspace *subspace = data->directoryData.subspace();
logOp(format("Unpack %s in subspace with prefix %s", key.getString(0).printable().c_str(), subspace->key().printable().c_str()));
Subspace* subspace = data->directoryData.subspace();
logOp(format("Unpack %s in subspace with prefix %s",
key.getString(0).printable().c_str(),
subspace->key().printable().c_str()));
Tuple tuple = subspace->unpack(key.getString(0));
for(int i = 0; i < tuple.size(); ++i) {
data->stack.push(tuple.subTuple(i, i+1).pack());
for (int i = 0; i < tuple.size(); ++i) {
data->stack.push(tuple.subTuple(i, i + 1).pack());
}
return Void();
@ -446,7 +457,7 @@ struct DirectoryUnpackKeyFunc : InstructionFunc {
const char* DirectoryUnpackKeyFunc::name = "DIRECTORY_UNPACK_KEY";
REGISTER_INSTRUCTION_FUNC(DirectoryUnpackKeyFunc);
//DIRECTORY_RANGE
// DIRECTORY_RANGE
struct DirectoryRangeFunc : InstructionFunc {
static const char* name;
@ -462,7 +473,7 @@ struct DirectoryRangeFunc : InstructionFunc {
const char* DirectoryRangeFunc::name = "DIRECTORY_RANGE";
REGISTER_INSTRUCTION_FUNC(DirectoryRangeFunc);
//DIRECTORY_CONTAINS
// DIRECTORY_CONTAINS
struct DirectoryContainsFunc : InstructionFunc {
static const char* name;
@ -477,15 +488,15 @@ struct DirectoryContainsFunc : InstructionFunc {
const char* DirectoryContainsFunc::name = "DIRECTORY_CONTAINS";
REGISTER_INSTRUCTION_FUNC(DirectoryContainsFunc);
//DIRECTORY_OPEN_SUBSPACE
// DIRECTORY_OPEN_SUBSPACE
struct DirectoryOpenSubspaceFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple tuple = wait(popTuple(data));
Subspace *subspace = data->directoryData.subspace();
Subspace* subspace = data->directoryData.subspace();
logOp(format("open_subspace %s (at %s)", tupleToString(tuple).c_str(), subspace->key().printable().c_str()));
Subspace *child = new Subspace(subspace->subspace(tuple));
Subspace* child = new Subspace(subspace->subspace(tuple));
data->directoryData.push(child);
return Void();
@ -494,7 +505,7 @@ struct DirectoryOpenSubspaceFunc : InstructionFunc {
const char* DirectoryOpenSubspaceFunc::name = "DIRECTORY_OPEN_SUBSPACE";
REGISTER_INSTRUCTION_FUNC(DirectoryOpenSubspaceFunc);
//DIRECTORY_LOG_SUBSPACE
// DIRECTORY_LOG_SUBSPACE
struct DirectoryLogSubspaceFunc : InstructionFunc {
static const char* name;
@ -510,7 +521,7 @@ struct DirectoryLogSubspaceFunc : InstructionFunc {
const char* DirectoryLogSubspaceFunc::name = "DIRECTORY_LOG_SUBSPACE";
REGISTER_INSTRUCTION_FUNC(DirectoryLogSubspaceFunc);
//DIRECTORY_LOG_DIRECTORY
// DIRECTORY_LOG_DIRECTORY
struct DirectoryLogDirectoryFunc : InstructionFunc {
static const char* name;
@ -520,22 +531,23 @@ struct DirectoryLogDirectoryFunc : InstructionFunc {
state bool exists = wait(directory->exists(instruction->tr));
state Tuple childrenTuple;
if(exists) {
if (exists) {
Standalone<VectorRef<StringRef>> children = wait(directory->list(instruction->tr));
for(auto &c : children) {
for (auto& c : children) {
childrenTuple.append(c, true);
}
}
}
Subspace logSubspace(Tuple().append(data->directoryData.directoryListIndex), prefix.getString(0));
Tuple pathTuple;
for(auto &p : directory->getPath()) {
for (auto& p : directory->getPath()) {
pathTuple.append(p, true);
}
instruction->tr->set(logSubspace.pack(LiteralStringRef("path"), true), pathTuple.pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true), Tuple().append(directory->getLayer()).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true),
Tuple().append(directory->getLayer()).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append(exists ? 1 : 0).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("children"), true), childrenTuple.pack());
@ -545,13 +557,13 @@ struct DirectoryLogDirectoryFunc : InstructionFunc {
const char* DirectoryLogDirectoryFunc::name = "DIRECTORY_LOG_DIRECTORY";
REGISTER_INSTRUCTION_FUNC(DirectoryLogDirectoryFunc);
//DIRECTORY_STRIP_PREFIX
// DIRECTORY_STRIP_PREFIX
struct DirectoryStripPrefixFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple str = wait(data->stack.waitAndPop());
Subspace *subspace = data->directoryData.subspace();
Subspace* subspace = data->directoryData.subspace();
ASSERT(str.getString(0).startsWith(subspace->key()));
data->stack.pushTuple(str.getString(0).substr(subspace->key().size()));
return Void();
@ -559,4 +571,3 @@ struct DirectoryStripPrefixFunc : InstructionFunc {
};
const char* DirectoryStripPrefixFunc::name = "DIRECTORY_STRIP_PREFIX";
REGISTER_INSTRUCTION_FUNC(DirectoryStripPrefixFunc);

482
bindings/flow/tester/Tester.actor.cpp
View File

File diff suppressed because it is too large Load Diff

84
bindings/flow/tester/Tester.actor.h
View File

@ -18,12 +18,13 @@
* limitations under the License.
*/
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version.
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDB_FLOW_TESTER_TESTER_ACTOR_G_H)
#define FDB_FLOW_TESTER_TESTER_ACTOR_G_H
#include "Tester.actor.g.h"
#define FDB_FLOW_TESTER_TESTER_ACTOR_G_H
#include "Tester.actor.g.h"
#elif !defined(FDB_FLOW_TESTER_TESTER_ACTOR_H)
#define FDB_FLOW_TESTER_TESTER_ACTOR_H
#define FDB_FLOW_TESTER_TESTER_ACTOR_H
#pragma once
@ -34,7 +35,7 @@
#include "bindings/flow/IDirectory.h"
#include "bindings/flow/Subspace.h"
#include "bindings/flow/DirectoryLayer.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
constexpr bool LOG_ALL = false;
constexpr bool LOG_INSTRUCTIONS = LOG_ALL || false;
@ -56,19 +57,13 @@ struct FlowTesterStack {
uint32_t index;
std::vector<StackItem> data;
void push(Future<Standalone<StringRef>> value) {
data.push_back(StackItem(index, value));
}
void push(Future<Standalone<StringRef>> value) { data.push_back(StackItem(index, value)); }
void push(Standalone<StringRef> value) {
push(Future<Standalone<StringRef>>(value));
}
void push(Standalone<StringRef> value) { push(Future<Standalone<StringRef>>(value)); }
void push(const StackItem& item) {
data.push_back(item);
}
void push(const StackItem& item) { data.push_back(item); }
void pushTuple(StringRef value, bool utf8=false) {
void pushTuple(StringRef value, bool utf8 = false) {
FDB::Tuple t;
t.append(value, utf8);
data.push_back(StackItem(index, t.pack()));
@ -101,9 +96,7 @@ struct FlowTesterStack {
data.push_back(data.back());
}
void clear() {
data.clear();
}
void clear() { data.clear(); }
};
struct InstructionData : public ReferenceCounted<InstructionData> {
@ -113,21 +106,21 @@ struct InstructionData : public ReferenceCounted<InstructionData> {
Reference<FDB::Transaction> tr;
InstructionData(bool _isDatabase, bool _isSnapshot, StringRef _instruction, Reference<FDB::Transaction> _tr)
: isDatabase(_isDatabase)
, isSnapshot(_isSnapshot)
, instruction(_instruction)
, tr(_tr) {}
: isDatabase(_isDatabase), isSnapshot(_isSnapshot), instruction(_instruction), tr(_tr) {}
};
struct FlowTesterData;
struct InstructionFunc : IDispatched<InstructionFunc, std::string, std::function<Future<Void>(Reference<FlowTesterData> data, Reference<InstructionData> instruction)>> {
struct InstructionFunc
: IDispatched<InstructionFunc,
std::string,
std::function<Future<Void>(Reference<FlowTesterData> data, Reference<InstructionData> instruction)>> {
static Future<Void> call(std::string op, Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
ASSERT(data);
ASSERT(instruction);
auto it = dispatches().find(op);
if(it == dispatches().end()) {
if (it == dispatches().end()) {
fprintf(stderr, "Unrecognized instruction: %s\n", op.c_str());
ASSERT(false);
}
@ -143,24 +136,20 @@ struct DirectoryOrSubspace {
DirectoryOrSubspace() {}
DirectoryOrSubspace(Reference<FDB::IDirectory> directory) : directory(directory) {}
DirectoryOrSubspace(FDB::Subspace *subspace) : subspace(subspace) {}
DirectoryOrSubspace(Reference<FDB::DirectorySubspace> dirSubspace) : directory(dirSubspace), subspace(dirSubspace.getPtr()) {}
DirectoryOrSubspace(FDB::Subspace* subspace) : subspace(subspace) {}
DirectoryOrSubspace(Reference<FDB::DirectorySubspace> dirSubspace)
: directory(dirSubspace), subspace(dirSubspace.getPtr()) {}
bool valid() {
return directory.present() || subspace.present();
}
bool valid() { return directory.present() || subspace.present(); }
std::string typeString() {
if(directory.present() && subspace.present()) {
if (directory.present() && subspace.present()) {
return "DirectorySubspace";
}
else if(directory.present()) {
} else if (directory.present()) {
return "IDirectory";
}
else if(subspace.present()) {
} else if (subspace.present()) {
return "Subspace";
}
else {
} else {
return "InvalidDirectory";
}
}
@ -190,8 +179,8 @@ struct DirectoryTesterData {
template <class T>
void push(T item) {
directoryList.push_back(DirectoryOrSubspace(item));
if(LOG_DIRS) {
printf("Pushed %s at %lu\n", directoryList.back().typeString().c_str(), directoryList.size()-1);
if (LOG_DIRS) {
printf("Pushed %s at %lu\n", directoryList.back().typeString().c_str(), directoryList.size() - 1);
fflush(stdout);
}
}
@ -200,7 +189,7 @@ struct DirectoryTesterData {
};
struct FlowTesterData : public ReferenceCounted<FlowTesterData> {
FDB::API *api;
FDB::API* api;
Reference<FDB::Database> db;
Standalone<FDB::RangeResultRef> instructions;
Standalone<StringRef> trName;
@ -211,12 +200,11 @@ struct FlowTesterData : public ReferenceCounted<FlowTesterData> {
std::vector<Future<Void>> subThreads;
Future<Void> processInstruction(Reference<InstructionData> instruction) {
return InstructionFunc::call(instruction->instruction.toString(), Reference<FlowTesterData>::addRef(this), instruction);
return InstructionFunc::call(
instruction->instruction.toString(), Reference<FlowTesterData>::addRef(this), instruction);
}
FlowTesterData(FDB::API *api) {
this->api = api;
}
FlowTesterData(FDB::API* api) { this->api = api; }
};
std::string tupleToString(FDB::Tuple const& tuple);
@ -226,16 +214,14 @@ Future<decltype(std::declval<F>()().getValue())> executeMutation(Reference<Instr
loop {
try {
state decltype(std::declval<F>()().getValue()) result = wait(func());
if(instruction->isDatabase) {
if (instruction->isDatabase) {
wait(instruction->tr->commit());
}
return result;
}
catch(Error &e) {
if(instruction->isDatabase) {
} catch (Error& e) {
if (instruction->isDatabase) {
wait(instruction->tr->onError(e));
}
else {
} else {
throw;
}
}

36
bindings/java/JavaWorkload.cpp
View File

@ -280,7 +280,8 @@ struct JVM {
w.name = name;
w.signature = sig;
w.fnPtr = std::get<2>(t);
log->trace(info, "PreparedNativeMethod",
log->trace(info,
"PreparedNativeMethod",
{ { "Name", w.name },
{ "Signature", w.signature },
{ "Ptr", std::to_string(reinterpret_cast<uintptr_t>(w.fnPtr)) } });
@ -362,7 +363,8 @@ struct JVM {
{ "getOption", "(JLjava/lang/String;Z)Z", reinterpret_cast<void*>(&getOptionBool) },
{ "getOption", "(JLjava/lang/String;J)J", reinterpret_cast<void*>(&getOptionLong) },
{ "getOption", "(JLjava/lang/String;D)D", reinterpret_cast<void*>(&getOptionDouble) },
{ "getOption", "(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;",
{ "getOption",
"(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;",
reinterpret_cast<void*>(&getOptionString) },
{ "getClientID", "(J)I", reinterpret_cast<void*>(&getClientID) },
{ "getClientCount", "(J)I", reinterpret_cast<void*>(&getClientCount) },
@ -391,7 +393,8 @@ struct JVM {
auto impl = env->GetLongField(res, field);
checkException();
if (impl != jContext) {
log->trace(error, "ContextNotCorrect",
log->trace(error,
"ContextNotCorrect",
{ { "Expected", std::to_string(jContext) }, { "Impl", std::to_string(impl) } });
std::terminate();
}
@ -471,14 +474,16 @@ struct JVM {
}
jobject createDatabase(jobject workload, FDBDatabase* db) {
auto executor =
env->CallObjectMethod(workload, getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"),
"getExecutor", "()Ljava/util/concurrent/Executor;"));
auto executor = env->CallObjectMethod(workload,
getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"),
"getExecutor",
"()Ljava/util/concurrent/Executor;"));
auto databaseClass = getClass("com/apple/foundationdb/FDBDatabase");
jlong databasePtr = reinterpret_cast<jlong>(db);
jobject javaDatabase =
env->NewObject(databaseClass, getMethod(databaseClass, "<init>", "(JLjava/util/concurrent/Executor;)V"),
databasePtr, executor);
jobject javaDatabase = env->NewObject(databaseClass,
getMethod(databaseClass, "<init>", "(JLjava/util/concurrent/Executor;)V"),
databasePtr,
executor);
env->DeleteLocalRef(executor);
return javaDatabase;
}
@ -491,9 +496,10 @@ struct JVM {
jPromise = createPromise(std::move(promise));
env->CallVoidMethod(
workload,
getMethod(clazz, method,
"(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"),
jdb, jPromise);
getMethod(
clazz, method, "(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"),
jdb,
jPromise);
env->DeleteLocalRef(jdb);
env->DeleteLocalRef(jPromise);
jPromise = nullptr;
@ -515,7 +521,7 @@ struct JavaWorkload : FDBWorkload {
bool failed = false;
jobject workload = nullptr;
JavaWorkload(const std::shared_ptr<JVM>& jvm, FDBLogger& log, const std::string& name)
: jvm(jvm), log(log), name(name) {
: jvm(jvm), log(log), name(name) {
boost::replace_all(this->name, ".", "/");
}
~JavaWorkload() {
@ -588,9 +594,7 @@ struct JavaWorkload : FDBWorkload {
log.trace(error, "CheckFailedWithJNIError", { { "Error", e.toString() }, { "Location", e.location() } });
}
}
void getMetrics(std::vector<FDBPerfMetric>& out) const override {
jvm->getMetrics(workload, name, out);
}
void getMetrics(std::vector<FDBPerfMetric>& out) const override { jvm->getMetrics(workload, name, out); }
};
struct JavaWorkloadFactory : FDBWorkloadFactory {

1037
bindings/java/fdbJNI.cpp
View File

File diff suppressed because it is too large Load Diff

7
contrib/monitoring/actor_flamegraph.cpp
View File

@ -24,7 +24,8 @@ struct Error {
struct Actor {
template <class Str>
explicit Actor(std::unordered_map<std::string, unsigned long>& results, unsigned long id, Str&& name) : results(results), id(id), name(std::forward<Str>(name)) {}
explicit Actor(std::unordered_map<std::string, unsigned long>& results, unsigned long id, Str&& name)
: results(results), id(id), name(std::forward<Str>(name)) {}
Actor(const Actor&) = delete;
~Actor() { collect(); }
std::unordered_map<std::string, unsigned long>& results;
@ -40,12 +41,12 @@ struct Actor {
for (auto i = stack.begin(); i != stack.end();) {
int num = 0;
auto name = *i;
for (; i != stack.end() && *i == name ; ++i) {
for (; i != stack.end() && *i == name; ++i) {
++num;
}
ss << name;
if (num > 1) {
ss << " ("<< num << ')';
ss << " (" << num << ')';
}
ss << ';';
}

22
documentation/tutorial/tutorial.actor.cpp
View File

@ -394,16 +394,18 @@ ACTOR Future<Void> fdbStatusStresser() {
}
}
std::unordered_map<std::string, std::function<Future<Void>()>> actors = { { "timer", &simpleTimer }, // ./tutorial timer
{ "promiseDemo", &promiseDemo }, // ./tutorial promiseDemo
{ "triggerDemo", &triggerDemo }, // ./tutorial triggerDemo
{ "echoServer", &echoServer }, // ./tutorial -p 6666 echoServer
{ "echoClient", &echoClient }, // ./tutorial -s 127.0.0.1:6666 echoClient
{ "kvStoreServer", &kvStoreServer }, // ./tutorial -p 6666 kvStoreServer
{ "kvSimpleClient", &kvSimpleClient }, // ./tutorial -s 127.0.0.1:6666 kvSimpleClient
{ "multipleClients", &multipleClients }, // ./tutorial -s 127.0.0.1:6666 multipleClients
{ "fdbClient", &fdbClient }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClient
{ "fdbStatusStresser", &fdbStatusStresser } }; // ./tutorial -C $CLUSTER_FILE_PATH fdbStatusStresser
std::unordered_map<std::string, std::function<Future<Void>()>> actors = {
{ "timer", &simpleTimer }, // ./tutorial timer
{ "promiseDemo", &promiseDemo }, // ./tutorial promiseDemo
{ "triggerDemo", &triggerDemo }, // ./tutorial triggerDemo
{ "echoServer", &echoServer }, // ./tutorial -p 6666 echoServer
{ "echoClient", &echoClient }, // ./tutorial -s 127.0.0.1:6666 echoClient
{ "kvStoreServer", &kvStoreServer }, // ./tutorial -p 6666 kvStoreServer
{ "kvSimpleClient", &kvSimpleClient }, // ./tutorial -s 127.0.0.1:6666 kvSimpleClient
{ "multipleClients", &multipleClients }, // ./tutorial -s 127.0.0.1:6666 multipleClients
{ "fdbClient", &fdbClient }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClient
{ "fdbStatusStresser", &fdbStatusStresser }
}; // ./tutorial -C $CLUSTER_FILE_PATH fdbStatusStresser
int main(int argc, char* argv[]) {
bool isServer = false;

55
fdbbackup/FileConverter.actor.cpp
View File

@ -148,13 +148,17 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
FileProgress(Reference<IAsyncFile> f, int index) : fd(f), idx(index), offset(0), eof(false) {}
bool operator<(const FileProgress& rhs) const {
if (rhs.mutations.empty()) return true;
if (mutations.empty()) return false;
if (rhs.mutations.empty())
return true;
if (mutations.empty())
return false;
return mutations[0].version < rhs.mutations[0].version;
}
bool operator<=(const FileProgress& rhs) const {
if (rhs.mutations.empty()) return true;
if (mutations.empty()) return false;
if (rhs.mutations.empty())
return true;
if (mutations.empty())
return false;
return mutations[0].version <= rhs.mutations[0].version;
}
bool empty() { return eof && mutations.empty(); }
@ -169,11 +173,13 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
try {
// Read block header
if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version();
if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION)
throw restore_unsupported_file_version();
while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF) break;
if (reader.eof() || *reader.rptr == 0xFF)
break;
// Deserialize messages written in saveMutationsToFile().
msgVersion = bigEndian64(reader.consume<Version>());
@ -181,7 +187,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
int msgSize = bigEndian32(reader.consume<int>());
const uint8_t* message = reader.consume(msgSize);
ArenaReader rd(buf.arena(), StringRef(message, msgSize), AssumeVersion(g_network->protocolVersion()));
ArenaReader rd(
buf.arena(), StringRef(message, msgSize), AssumeVersion(g_network->protocolVersion()));
MutationRef m;
rd >> m;
count++;
@ -194,8 +201,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
break; // skip
}
if (msgVersion >= minVersion) {
mutations.emplace_back(LogMessageVersion(msgVersion, sub), StringRef(message, msgSize),
buf.arena());
mutations.emplace_back(
LogMessageVersion(msgVersion, sub), StringRef(message, msgSize), buf.arena());
inserted++;
}
}
@ -232,7 +239,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
bool hasMutations() {
for (const auto& fp : fileProgress) {
if (!fp->empty()) return true;
if (!fp->empty())
return true;
}
return false;
}
@ -252,7 +260,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
// Sorts files according to their first mutation version and removes files without mutations.
void sortAndRemoveEmpty() {
std::sort(fileProgress.begin(), fileProgress.end(),
std::sort(fileProgress.begin(),
fileProgress.end(),
[](const Reference<FileProgress>& a, const Reference<FileProgress>& b) { return (*a) < (*b); });
while (!fileProgress.empty() && fileProgress.back()->empty()) {
fileProgress.pop_back();
@ -319,11 +328,15 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
// Decodes the file until EOF or an mutation >= minVersion and saves these mutations.
// Skip mutations >= maxVersion.
ACTOR static Future<Void> decodeToVersion(Reference<FileProgress> fp, Version minVersion, Version maxVersion,
ACTOR static Future<Void> decodeToVersion(Reference<FileProgress> fp,
Version minVersion,
Version maxVersion,
LogFile file) {
if (fp->empty()) return Void();
if (fp->empty())
return Void();
if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion) return Void();
if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion)
return Void();
state int64_t len;
try {
@ -337,13 +350,15 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
state Standalone<StringRef> buf = makeString(len);
int rLen = wait(fp->fd->read(mutateString(buf), len, fp->offset));
if (len != rLen) throw restore_bad_read();
if (len != rLen)
throw restore_bad_read();
TraceEvent("ReadFile")
.detail("Name", fp->fd->getFilename())
.detail("Length", rLen)
.detail("Offset", fp->offset);
if (fp->decodeBlock(buf, rLen, minVersion, maxVersion)) break;
if (fp->decodeBlock(buf, rLen, minVersion, maxVersion))
break;
}
return Void();
} catch (Error& e) {
@ -402,7 +417,8 @@ struct LogFileWriter {
wait(self->file->appendStringRefWithLen(v));
// At this point we should be in whatever the current block is or the block size is too small
if (self->file->size() > self->blockEnd) throw backup_bad_block_size();
if (self->file->size() > self->blockEnd)
throw backup_bad_block_size();
return Void();
}
@ -439,7 +455,8 @@ ACTOR Future<Void> convert(ConvertParams params) {
state BackupDescription desc = wait(container->describeBackup());
std::cout << "\n" << desc.toString() << "\n";
// std::cout << "Using Protocol Version: 0x" << std::hex << g_network->protocolVersion().version() << std::dec << "\n";
// std::cout << "Using Protocol Version: 0x" << std::hex << g_network->protocolVersion().version() << std::dec <<
// "\n";
std::vector<LogFile> logs = getRelevantLogFiles(listing.logs, params.begin, params.end);
printLogFiles("Range has", logs);
@ -550,7 +567,7 @@ int parseCommandLine(ConvertParams* param, CSimpleOpt* args) {
return FDB_EXIT_SUCCESS;
}
} // namespace file_converter
} // namespace file_converter
int main(int argc, char** argv) {
try {

4
fdbbackup/FileConverter.h
View File

@ -61,6 +61,6 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
{ OPT_HELP, "--help", SO_NONE },
SO_END_OF_OPTIONS };
} // namespace file_converter
} // namespace file_converter
#endif // FDBBACKUP_FILECONVERTER_H
#endif // FDBBACKUP_FILECONVERTER_H

24
fdbbackup/FileDecoder.actor.cpp
View File

@ -189,7 +189,8 @@ std::vector<MutationRef> decode_value(const StringRef& value) {
std::vector<MutationRef> mutations;
while (1) {
if (reader.eof()) break;
if (reader.eof())
break;
// Deserialization of a MutationRef, which was packed by MutationListRef::push_back_deep()
uint32_t type, p1len, p2len;
@ -242,8 +243,7 @@ class DecodeProgress {
public:
DecodeProgress() = default;
template <class U>
DecodeProgress(const LogFile& file, U &&values)
: file(file), keyValues(std::forward<U>(values)) {}
DecodeProgress(const LogFile& file, U&& values) : file(file), keyValues(std::forward<U>(values)) {}
// If there are no more mutations to pull from the file.
// However, we could have unfinished version in the buffer when EOF is true,
@ -289,7 +289,8 @@ public:
int idx = 1; // next kv pair in "keyValues"
int bufSize = kv.kv.size();
for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) {
if (idx == self->keyValues.size()) break;
if (idx == self->keyValues.size())
break;
const auto& nextKV = self->keyValues[idx];
if (kv.version != nextKV.version) {
@ -355,12 +356,14 @@ public:
try {
// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION) throw restore_unsupported_file_version();
if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION)
throw restore_unsupported_file_version();
// Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte.
while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF) break;
if (reader.eof() || *reader.rptr == 0xFF)
break;
// Read key and value. If anything throws then there is a problem.
uint32_t kLen = reader.consumeNetworkUInt32();
@ -379,7 +382,8 @@ public:
// Make sure any remaining bytes in the block are 0xFF
for (auto b : reader.remainder()) {
if (b != 0xFF) throw restore_corrupted_data_padding();
if (b != 0xFF)
throw restore_corrupted_data_padding();
}
// The (version, part) in a block can be out of order, i.e., (3, 0)
@ -445,7 +449,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
state BackupFileList listing = wait(container->dumpFileList());
// remove partitioned logs
listing.logs.erase(std::remove_if(listing.logs.begin(), listing.logs.end(),
listing.logs.erase(std::remove_if(listing.logs.begin(),
listing.logs.end(),
[](const LogFile& file) {
std::string prefix("plogs/");
return file.fileName.substr(0, prefix.size()) == prefix;
@ -464,7 +469,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
// Previous file's unfinished version data
state std::vector<VersionedKVPart> left;
for (; i < logs.size(); i++) {
if (logs[i].fileSize == 0) continue;
if (logs[i].fileSize == 0)
continue;
state DecodeProgress progress(logs[i], std::move(left));
wait(progress.openFile(container));

2868
fdbbackup/backup.actor.cpp
View File

File diff suppressed because it is too large Load Diff

221
fdbcli/FlowLineNoise.actor.cpp
View File

@ -29,153 +29,152 @@
#include "flow/ThreadHelper.actor.h"
#if __unixish__
#define HAVE_LINENOISE 1
#include "fdbcli/linenoise/linenoise.h"
#define HAVE_LINENOISE 1
#include "fdbcli/linenoise/linenoise.h"
#else
#define HAVE_LINENOISE 0
#define HAVE_LINENOISE 0
#endif
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
struct LineNoiseReader final : IThreadPoolReceiver {
void init() override {}
struct Read final : TypedAction<LineNoiseReader, Read> {
std::string prompt;
ThreadReturnPromise<Optional<std::string>> result;
ThreadReturnPromise<Optional<std::string>> result;
double getTimeEstimate() const override { return 0.0; }
explicit Read(std::string const& prompt) : prompt(prompt) {}
};
void action(Read& r) {
try {
r.result.send( read(r.prompt) );
} catch (Error& e) {
r.result.sendError(e);
} catch (...) {
r.result.sendError(unknown_error());
}
}
try {
r.result.send(read(r.prompt));
} catch (Error& e) {
r.result.sendError(e);
} catch (...) {
r.result.sendError(unknown_error());
}
}
private:
Optional<std::string> read(std::string const& prompt) {
#if HAVE_LINENOISE
errno = 0;
char* line = linenoise(prompt.c_str());
if (line) {
std::string s(line);
free(line);
return s;
} else {
if (errno == EAGAIN) // Ctrl-C
return std::string();
return Optional<std::string>();
}
#else
std::string line;
std::fputs( prompt.c_str(), stdout );
if (!std::getline( std::cin, line ).eof()) {
return line;
} else
return Optional<std::string>();
#endif
}
Optional<std::string> read(std::string const& prompt) {
#if HAVE_LINENOISE
errno = 0;
char* line = linenoise(prompt.c_str());
if (line) {
std::string s(line);
free(line);
return s;
} else {
if (errno == EAGAIN) // Ctrl-C
return std::string();
return Optional<std::string>();
}
#else
std::string line;
std::fputs(prompt.c_str(), stdout);
if (!std::getline(std::cin, line).eof()) {
return line;
} else
return Optional<std::string>();
#endif
}
};
LineNoise::LineNoise(
std::function< void(std::string const&, std::vector<std::string>&) > _completion_callback,
std::function< Hint(std::string const&) > _hint_callback,
int maxHistoryLines,
bool multiline )
: threadPool( createGenericThreadPool() )
{
reader = new LineNoiseReader();
LineNoise::LineNoise(std::function<void(std::string const&, std::vector<std::string>&)> _completion_callback,
std::function<Hint(std::string const&)> _hint_callback,
int maxHistoryLines,
bool multiline)
: threadPool(createGenericThreadPool()) {
reader = new LineNoiseReader();
#if HAVE_LINENOISE
// It should be OK to call these functions from this thread, since read() can't be called yet
// The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely invoke the callbacks we've been given
#if HAVE_LINENOISE
// It should be OK to call these functions from this thread, since read() can't be called yet
// The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely
// invoke the callbacks we've been given
// linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables
static std::function< void(std::string const&, std::vector<std::string>&) > completion_callback;
static std::function< Hint(std::string const&) > hint_callback;
completion_callback = _completion_callback;
hint_callback = _hint_callback;
// linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables
static std::function<void(std::string const&, std::vector<std::string>&)> completion_callback;
static std::function<Hint(std::string const&)> hint_callback;
completion_callback = _completion_callback;
hint_callback = _hint_callback;
linenoiseHistorySetMaxLen( maxHistoryLines );
linenoiseSetMultiLine( multiline );
linenoiseSetCompletionCallback( [](const char* line, linenoiseCompletions* lc) {
// This code will run in the thread pool
std::vector<std::string> completions;
onMainThread( [line, &completions]() -> Future<Void> {
completion_callback(line, completions);
return Void();
}).getBlocking();
for( auto const& c : completions )
linenoiseAddCompletion( lc, c.c_str() );
});
linenoiseSetHintsCallback( [](const char* line, int* color, int*bold) -> char* {
Hint h = onMainThread( [line]() -> Future<Hint> {
return hint_callback(line);
}).getBlocking();
if (!h.valid) return nullptr;
*color = h.color;
*bold = h.bold;
return strdup( h.text.c_str() );
});
linenoiseSetFreeHintsCallback( free );
#endif
linenoiseHistorySetMaxLen(maxHistoryLines);
linenoiseSetMultiLine(multiline);
linenoiseSetCompletionCallback([](const char* line, linenoiseCompletions* lc) {
// This code will run in the thread pool
std::vector<std::string> completions;
onMainThread([line, &completions]() -> Future<Void> {
completion_callback(line, completions);
return Void();
}).getBlocking();
for (auto const& c : completions)
linenoiseAddCompletion(lc, c.c_str());
});
linenoiseSetHintsCallback([](const char* line, int* color, int* bold) -> char* {
Hint h = onMainThread([line]() -> Future<Hint> { return hint_callback(line); }).getBlocking();
if (!h.valid)
return nullptr;
*color = h.color;
*bold = h.bold;
return strdup(h.text.c_str());
});
linenoiseSetFreeHintsCallback(free);
#endif
threadPool->addThread(reader);
threadPool->addThread(reader);
}
LineNoise::~LineNoise() {
threadPool.clear();
threadPool.clear();
}
Future<Optional<std::string>> LineNoise::read( std::string const& prompt ) {
auto r = new LineNoiseReader::Read(prompt);
auto f = r->result.getFuture();
threadPool->post(r);
return f;
Future<Optional<std::string>> LineNoise::read(std::string const& prompt) {
auto r = new LineNoiseReader::Read(prompt);
auto f = r->result.getFuture();
threadPool->post(r);
return f;
}
ACTOR Future<Void> waitKeyboardInterrupt(boost::asio::io_service* ios) {
state boost::asio::signal_set signals(*ios, SIGINT);
Promise<Void> result;
signals.async_wait([result](const boost::system::error_code& error, int signal_number) {
if (error) {
result.sendError(io_error());
} else {
result.send(Void());
}
});
state boost::asio::signal_set signals(*ios, SIGINT);
Promise<Void> result;
signals.async_wait([result](const boost::system::error_code& error, int signal_number) {
if (error) {
result.sendError(io_error());
} else {
result.send(Void());
}
});
wait(result.getFuture());
return Void();
wait(result.getFuture());
return Void();
}
Future<Void> LineNoise::onKeyboardInterrupt() {
boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService);
if (!ios) return Never();
return waitKeyboardInterrupt(ios);
boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService);
if (!ios)
return Never();
return waitKeyboardInterrupt(ios);
}
void LineNoise::historyAdd( std::string const& line ) {
#if HAVE_LINENOISE
linenoiseHistoryAdd( line.c_str() );
#endif
void LineNoise::historyAdd(std::string const& line) {
#if HAVE_LINENOISE
linenoiseHistoryAdd(line.c_str());
#endif
}
void LineNoise::historyLoad( std::string const& filename ) {
#if HAVE_LINENOISE
if(linenoiseHistoryLoad(filename.c_str()) != 0) {
throw io_error();
}
#endif
void LineNoise::historyLoad(std::string const& filename) {
#if HAVE_LINENOISE
if (linenoiseHistoryLoad(filename.c_str()) != 0) {
throw io_error();
}
#endif
}
void LineNoise::historySave( std::string const& filename ) {
#if HAVE_LINENOISE
if(linenoiseHistorySave(filename.c_str()) != 0) {
throw io_error();
}
#endif
void LineNoise::historySave(std::string const& filename) {
#if HAVE_LINENOISE
if (linenoiseHistorySave(filename.c_str()) != 0) {
throw io_error();
}
#endif
}

52
fdbcli/FlowLineNoise.h
View File

@ -26,39 +26,37 @@
#include <functional>
struct LineNoise : NonCopyable {
// Wraps the linenoise library so that it can be called from asynchronous Flow code
// Only create one of these at a time; the linenoise library only supports one history
//
// The current implementation does not support calling read concurrently with any other
// function (or itself).
// Wraps the linenoise library so that it can be called from asynchronous Flow code
// Only create one of these at a time; the linenoise library only supports one history
//
// The current implementation does not support calling read concurrently with any other
// function (or itself).
struct Hint {
std::string text;
int color;
bool bold;
bool valid;
Hint() : text(), color(), bold(), valid() {}
Hint( std::string const& text, int color, bool bold ) : text(text), color(color), bold(bold), valid(true) {}
};
struct Hint {
std::string text;
int color;
bool bold;
bool valid;
Hint() : text(), color(), bold(), valid() {}
Hint(std::string const& text, int color, bool bold) : text(text), color(color), bold(bold), valid(true) {}
};
LineNoise(
std::function< void(std::string const&, std::vector<std::string>&) > completion_callback,
std::function< Hint(std::string const&) > hint_callback,
int maxHistoryLines,
bool multiline
);
~LineNoise();
LineNoise(std::function<void(std::string const&, std::vector<std::string>&)> completion_callback,
std::function<Hint(std::string const&)> hint_callback,
int maxHistoryLines,
bool multiline);
~LineNoise();
Future< Optional<std::string> > read( std::string const& prompt ); // Returns "nothing" on EOF
void historyAdd( std::string const& line );
Future<Optional<std::string>> read(std::string const& prompt); // Returns "nothing" on EOF
void historyAdd(std::string const& line);
void historyLoad( std::string const& filename );
void historySave( std::string const& filename );
void historyLoad(std::string const& filename);
void historySave(std::string const& filename);
static Future<Void> onKeyboardInterrupt(); // Returns when Ctrl-C is next pressed (i.e. SIGINT)
static Future<Void> onKeyboardInterrupt(); // Returns when Ctrl-C is next pressed (i.e. SIGINT)
Reference<class IThreadPool> threadPool;
struct LineNoiseReader* reader;
Reference<class IThreadPool> threadPool;
struct LineNoiseReader* reader;
};
#endif

2376
fdbcli/fdbcli.actor.cpp
View File

File diff suppressed because it is too large Load Diff

1604
fdbcli/linenoise/linenoise.c
View File

File diff suppressed because it is too large Load Diff

28
fdbcli/linenoise/linenoise.h
View File

@ -46,24 +46,24 @@ extern "C" {
#endif
typedef struct linenoiseCompletions {
size_t len;
char **cvec;
size_t len;
char** cvec;
} linenoiseCompletions;
typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *);
typedef char*(linenoiseHintsCallback)(const char *, int *color, int *bold);
typedef void(linenoiseFreeHintsCallback)(void *);
void linenoiseSetCompletionCallback(linenoiseCompletionCallback *);
void linenoiseSetHintsCallback(linenoiseHintsCallback *);
void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *);
void linenoiseAddCompletion(linenoiseCompletions *, const char *);
typedef void(linenoiseCompletionCallback)(const char*, linenoiseCompletions*);
typedef char*(linenoiseHintsCallback)(const char*, int* color, int* bold);
typedef void(linenoiseFreeHintsCallback)(void*);
void linenoiseSetCompletionCallback(linenoiseCompletionCallback*);
void linenoiseSetHintsCallback(linenoiseHintsCallback*);
void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback*);
void linenoiseAddCompletion(linenoiseCompletions*, const char*);
char *linenoise(const char *prompt);
void linenoiseFree(void *ptr);
int linenoiseHistoryAdd(const char *line);
char* linenoise(const char* prompt);
void linenoiseFree(void* ptr);
int linenoiseHistoryAdd(const char* line);
int linenoiseHistorySetMaxLen(int len);
int linenoiseHistorySave(const char *filename);
int linenoiseHistoryLoad(const char *filename);
int linenoiseHistorySave(const char* filename);
int linenoiseHistoryLoad(const char* filename);
void linenoiseClearScreen(void);
void linenoiseSetMultiLine(int ml);
void linenoisePrintKeyCodes(void);

6
fdbclient/AsyncFileS3BlobStore.actor.cpp
View File

@ -24,7 +24,8 @@
#include "flow/actorcompiler.h" // has to be last include
Future<int64_t> AsyncFileS3BlobStoreRead::size() const {
if (!m_size.isValid()) m_size = m_bstore->objectSize(m_bucket, m_object);
if (!m_size.isValid())
m_size = m_bstore->objectSize(m_bucket, m_object);
return m_size;
}
@ -48,7 +49,8 @@ ACTOR Future<Void> sendStuff(int id, Reference<IRateControl> t, int bytes) {
TEST_CASE("/backup/throttling") {
// Test will not work in simulation.
if (g_network->isSimulated()) return Void();
if (g_network->isSimulated())
return Void();
state int limit = 100000;
state Reference<IRateControl> t(new SpeedLimit(limit, 1));

23
fdbclient/AsyncFileS3BlobStore.actor.h
View File

@ -46,7 +46,8 @@ static Future<T> joinErrorGroup(Future<T> f, Promise<Void> p) {
wait(success(f) || p.getFuture());
return f.get();
} catch (Error& e) {
if (p.canBeSet()) p.sendError(e);
if (p.canBeSet())
p.sendError(e);
throw;
}
}
@ -117,7 +118,8 @@ public:
}
Future<Void> write(void const* data, int length, int64_t offset) override {
if (offset != m_cursor) throw non_sequential_op();
if (offset != m_cursor)
throw non_sequential_op();
m_cursor += length;
return m_error.getFuture() ||
@ -125,15 +127,16 @@ public:
}
Future<Void> truncate(int64_t size) override {
if (size != m_cursor) return non_sequential_op();
if (size != m_cursor)
return non_sequential_op();
return Void();
}
ACTOR static Future<std::string> doPartUpload(AsyncFileS3BlobStoreWrite* f, Part* p) {
p->finalizeMD5();
std::string upload_id = wait(f->getUploadID());
std::string etag = wait(f->m_bstore->uploadPart(f->m_bucket, f->m_object, upload_id, p->number, &p->content,
p->length, p->md5string));
std::string etag = wait(f->m_bstore->uploadPart(
f->m_bucket, f->m_object, upload_id, p->number, &p->content, p->length, p->md5string));
return etag;
}
@ -142,8 +145,8 @@ public:
if (f->m_parts.size() == 1) {
Reference<Part> part = f->m_parts.back();
part->finalizeMD5();
wait(f->m_bstore->writeEntireFileFromBuffer(f->m_bucket, f->m_object, &part->content, part->length,
part->md5string));
wait(f->m_bstore->writeEntireFileFromBuffer(
f->m_bucket, f->m_object, &part->content, part->length, part->md5string));
return Void();
}
@ -221,7 +224,8 @@ private:
// End the current part and start uploading it, but also wait for a part to finish if too many are in transit.
ACTOR static Future<Void> endCurrentPart(AsyncFileS3BlobStoreWrite* f, bool startNew = false) {
if (f->m_parts.back()->length == 0) return Void();
if (f->m_parts.back()->length == 0)
return Void();
// Wait for an upload slot to be available
wait(f->m_concurrentUploads.take());
@ -241,7 +245,8 @@ private:
}
Future<std::string> getUploadID() {
if (!m_upload_id.isValid()) m_upload_id = m_bstore->beginMultiPartUpload(m_bucket, m_object);
if (!m_upload_id.isValid())
m_upload_id = m_bstore->beginMultiPartUpload(m_bucket, m_object);
return m_upload_id;
}

140
fdbclient/Atomic.h
View File

@ -24,21 +24,25 @@
#include "fdbclient/CommitTransaction.h"
inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand,
Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
int carry = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++) {
for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++) {
int sum = existingValue[i] + otherOperand[i] + carry;
buf[i] = sum;
carry = sum >> 8;
}
for (; i<otherOperand.size(); i++) {
for (; i < otherOperand.size(); i++) {
int sum = otherOperand[i] + carry;
buf[i] = sum;
carry = sum >> 8;
@ -49,14 +53,15 @@ inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptiona
inline ValueRef doAnd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!otherOperand.size()) return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++)
for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] & otherOperand[i];
for(; i<otherOperand.size(); i++)
for (; i < otherOperand.size(); i++)
buf[i] = 0x0;
return StringRef(buf, i);
@ -71,15 +76,17 @@ inline ValueRef doAndV2(const Optional<ValueRef>& existingValueOptional, const V
inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++)
for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] | otherOperand[i];
for(; i<otherOperand.size(); i++)
for (; i < otherOperand.size(); i++)
buf[i] = otherOperand[i];
return StringRef(buf, i);
@ -87,48 +94,56 @@ inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const Valu
inline ValueRef doXor(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++)
for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] ^ otherOperand[i];
for(; i<otherOperand.size(); i++)
for (; i < otherOperand.size(); i++)
buf[i] = otherOperand[i];
return StringRef(buf, i);
}
inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand,
Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return existingValue;
if(existingValue.size() + otherOperand.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) {
TEST( true ) //AppendIfFIts resulted in truncation
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return existingValue;
if (existingValue.size() + otherOperand.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) {
TEST(true) // AppendIfFIts resulted in truncation
return existingValue;
}
uint8_t* buf = new (ar) uint8_t [existingValue.size() + otherOperand.size()];
int i,j;
uint8_t* buf = new (ar) uint8_t[existingValue.size() + otherOperand.size()];
int i, j;
for(i = 0; i<existingValue.size(); i++)
for (i = 0; i < existingValue.size(); i++)
buf[i] = existingValue[i];
for(j = 0; j<otherOperand.size(); j++)
buf[i+j] = otherOperand[j];
for (j = 0; j < otherOperand.size(); j++)
buf[i + j] = otherOperand[j];
return StringRef(buf, i+j);
return StringRef(buf, i + j);
}
inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if (!existingValue.size()) return otherOperand;
if (!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
int i,j;
int i, j;
for (i = otherOperand.size() - 1; i >= existingValue.size(); i--) {
if (otherOperand[i] != 0) {
@ -139,9 +154,8 @@ inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const Val
for (; i >= 0; i--) {
if (otherOperand[i] > existingValue[i]) {
return otherOperand;
}
else if (otherOperand[i] < existingValue[i]) {
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
} else if (otherOperand[i] < existingValue[i]) {
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) {
buf[j] = existingValue[j];
}
@ -156,7 +170,8 @@ inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const Val
}
inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present()) return otherOperand;
if (!existingValueOptional.present())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.get();
if (existingValue > otherOperand)
@ -166,14 +181,15 @@ inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const
}
inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!otherOperand.size()) return otherOperand;
if (!otherOperand.size())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
int i,j;
int i, j;
for (i = otherOperand.size() - 1; i >= existingValue.size(); i--) {
if (otherOperand[i] != 0) {
uint8_t* buf = new (ar)uint8_t[otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) {
buf[j] = existingValue[j];
}
@ -186,7 +202,7 @@ inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const Val
for (; i >= 0; i--) {
if (otherOperand[i] > existingValue[i]) {
uint8_t* buf = new (ar)uint8_t[otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) {
buf[j] = existingValue[j];
}
@ -194,8 +210,7 @@ inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const Val
buf[j] = 0x0;
}
return StringRef(buf, j);
}
else if (otherOperand[i] < existingValue[i]) {
} else if (otherOperand[i] < existingValue[i]) {
return otherOperand;
}
}
@ -211,7 +226,8 @@ inline ValueRef doMinV2(const Optional<ValueRef>& existingValueOptional, const V
}
inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present()) return otherOperand;
if (!existingValueOptional.present())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.get();
if (existingValue < otherOperand)
@ -221,7 +237,8 @@ inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const
}
inline Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand, Arena& ar) {
const ValueRef& otherOperand,
Arena& ar) {
if (!existingValueOptional.present() || existingValueOptional.get() == otherOperand) {
// Clear the value.
return Optional<ValueRef>();
@ -229,19 +246,19 @@ inline Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingVa
return existingValueOptional; // No change required.
}
static void placeVersionstamp( uint8_t* destination, Version version, uint16_t transactionNumber ) {
static void placeVersionstamp(uint8_t* destination, Version version, uint16_t transactionNumber) {
version = bigEndian64(version);
transactionNumber = bigEndian16(transactionNumber);
static_assert( sizeof(version) == 8, "version size mismatch" );
memcpy( destination, &version, sizeof(version) );
static_assert( sizeof(transactionNumber) == 2, "txn num size mismatch");
memcpy( destination + sizeof(version), &transactionNumber, sizeof(transactionNumber) );
static_assert(sizeof(version) == 8, "version size mismatch");
memcpy(destination, &version, sizeof(version));
static_assert(sizeof(transactionNumber) == 2, "txn num size mismatch");
memcpy(destination + sizeof(version), &transactionNumber, sizeof(transactionNumber));
}
/*
* Returns the range corresponding to the specified versionstamp key.
*/
inline KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef &key, Version minVersion, const KeyRef &maxKey) {
* Returns the range corresponding to the specified versionstamp key.
*/
inline KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef& key, Version minVersion, const KeyRef& maxKey) {
KeyRef begin(arena, key);
KeyRef end(arena, key);
@ -253,18 +270,18 @@ inline KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef &key, Vers
pos = littleEndian32(pos);
begin = begin.substr(0, begin.size() - 4);
end = end.substr(0, end.size() - 3);
mutateString(end)[end.size()-1] = 0;
mutateString(end)[end.size() - 1] = 0;
if (pos < 0 || pos + 10 > begin.size())
throw client_invalid_operation();
placeVersionstamp(mutateString(begin)+pos, minVersion, 0);
placeVersionstamp(mutateString(begin) + pos, minVersion, 0);
memset(mutateString(end) + pos, '\xff', 10);
return KeyRangeRef(begin, std::min(end, maxKey));
}
inline void transformVersionstampKey( StringRef& key, Version version, uint16_t transactionNumber ) {
inline void transformVersionstampKey(StringRef& key, Version version, uint16_t transactionNumber) {
if (key.size() < 4)
throw client_invalid_operation();
@ -274,10 +291,13 @@ inline void transformVersionstampKey( StringRef& key, Version version, uint16_t
if (pos < 0 || pos + 10 > key.size())
throw client_invalid_operation();
placeVersionstamp( mutateString(key) + pos, version, transactionNumber );
placeVersionstamp(mutateString(key) + pos, version, transactionNumber);
}
inline void transformVersionstampMutation( MutationRef& mutation, StringRef MutationRef::* param, Version version, uint16_t transactionNumber ) {
inline void transformVersionstampMutation(MutationRef& mutation,
StringRef MutationRef::*param,
Version version,
uint16_t transactionNumber) {
if ((mutation.*param).size() >= 4) {
int32_t pos;
memcpy(&pos, (mutation.*param).end() - sizeof(int32_t), sizeof(int32_t));
@ -285,7 +305,7 @@ inline void transformVersionstampMutation( MutationRef& mutation, StringRef Muta
mutation.*param = (mutation.*param).substr(0, (mutation.*param).size() - 4);
if (pos >= 0 && pos + 10 <= (mutation.*param).size()) {
placeVersionstamp( mutateString(mutation.*param) + pos, version, transactionNumber );
placeVersionstamp(mutateString(mutation.*param) + pos, version, transactionNumber);
}
}

3
fdbclient/AutoPublicAddress.cpp
View File

@ -46,8 +46,7 @@ IPAddress determinePublicIPAutomatically(ClusterConnectionString const& ccs) {
socket.close();
return ip;
}
catch(boost::system::system_error e) {
} catch (boost::system::system_error e) {
fprintf(stderr, "Error determining public address: %s\n", e.what());
throw bind_failed();
}

475
fdbclient/BackupAgent.actor.h
View File

@ -20,10 +20,10 @@
#pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_BACKUP_AGENT_ACTOR_G_H)
#define FDBCLIENT_BACKUP_AGENT_ACTOR_G_H
#include "fdbclient/BackupAgent.actor.g.h"
#define FDBCLIENT_BACKUP_AGENT_ACTOR_G_H
#include "fdbclient/BackupAgent.actor.g.h"
#elif !defined(FDBCLIENT_BACKUP_AGENT_ACTOR_H)
#define FDBCLIENT_BACKUP_AGENT_ACTOR_H
#define FDBCLIENT_BACKUP_AGENT_ACTOR_H
#include "flow/flow.h"
#include "fdbclient/NativeAPI.actor.h"
@ -42,9 +42,7 @@ public:
static std::string formatTime(int64_t epochs);
static int64_t parseTime(std::string timestamp);
static std::string timeFormat() {
return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM";
}
static std::string timeFormat() { return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM"; }
enum class EnumState {
STATE_ERRORED = 0,
@ -125,8 +123,7 @@ public:
static const char* getStateText(EnumState enState) {
const char* stateText;
switch (enState)
{
switch (enState) {
case EnumState::STATE_ERRORED:
stateText = "has errored";
break;
@ -163,8 +160,7 @@ public:
static const char* getStateName(EnumState enState) {
const char* s;
switch (enState)
{
switch (enState) {
case EnumState::STATE_ERRORED:
s = "Errored";
break;
@ -201,8 +197,7 @@ public:
static bool isRunnable(EnumState enState) {
bool isRunnable = false;
switch (enState)
{
switch (enState) {
case EnumState::STATE_SUBMITTED:
case EnumState::STATE_RUNNING:
case EnumState::STATE_RUNNING_DIFFERENTIAL:
@ -216,13 +211,9 @@ public:
return isRunnable;
}
static const KeyRef getDefaultTag() {
return StringRef(defaultTagName);
}
static const KeyRef getDefaultTag() { return StringRef(defaultTagName); }
static const std::string getDefaultTagName() {
return defaultTagName;
}
static const std::string getDefaultTagName() { return defaultTagName; }
// This is only used for automatic backup name generation
static Standalone<StringRef> getCurrentTime() {
@ -234,7 +225,7 @@ public:
strftime(buffer, 128, "%Y-%m-%d-%H-%M-%S", timeinfo);
std::string time(buffer);
return StringRef(time + format(".%06d", (int)(1e6*(t - curTime))));
return StringRef(time + format(".%06d", (int)(1e6 * (t - curTime))));
}
protected:
@ -252,16 +243,13 @@ public:
void operator=(FileBackupAgent&& r) noexcept {
subspace = std::move(r.subspace);
config = std::move(r.config);
lastRestorable = std::move(r.lastRestorable),
taskBucket = std::move(r.taskBucket);
lastRestorable = std::move(r.lastRestorable), taskBucket = std::move(r.taskBucket);
futureBucket = std::move(r.futureBucket);
}
KeyBackedProperty<Key> lastBackupTimestamp() {
return config.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> lastBackupTimestamp() { return config.pack(LiteralStringRef(__FUNCTION__)); }
Future<Void> run(Database cx, double *pollDelay, int maxConcurrentTasks) {
Future<Void> run(Database cx, double* pollDelay, int maxConcurrentTasks) {
return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
}
@ -273,34 +261,80 @@ public:
// parallel restore
Future<Void> parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true);
Future<Void> submitParallelRestore(Database cx, Key backupTag, Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key bcUrl, Version targetVersion, bool lockDB, UID randomUID, Key addPrefix,
Future<Void> submitParallelRestore(Database cx,
Key backupTag,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key bcUrl,
Version targetVersion,
bool lockDB,
UID randomUID,
Key addPrefix,
Key removePrefix);
Future<Void> atomicParallelRestore(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix,
Key removePrefix);
Future<Void> atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix, Key removePrefix);
// restore() will
// - make sure that url is readable and appears to be a complete backup
// - make sure the requested TargetVersion is valid
// - submit a restore on the given tagName
// - Optionally wait for the restore's completion. Will restore_error if restore fails or is aborted.
// restore() will return the targetVersion which will be either the valid version passed in or the max restorable version for the given url.
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url,
Standalone<VectorRef<KeyRangeRef>> ranges, bool waitForComplete = true,
Version targetVersion = -1, bool verbose = true, Key addPrefix = Key(),
Key removePrefix = Key(), bool lockDB = true, bool incrementalBackupOnly = false,
// restore() will return the targetVersion which will be either the valid version passed in or the max restorable
// version for the given url.
Future<Version> restore(Database cx,
Optional<Database> cxOrig,
Key tagName,
Key url,
Standalone<VectorRef<KeyRangeRef>> ranges,
bool waitForComplete = true,
Version targetVersion = -1,
bool verbose = true,
Key addPrefix = Key(),
Key removePrefix = Key(),
bool lockDB = true,
bool incrementalBackupOnly = false,
Version beginVersion = -1);
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, bool waitForComplete = true,
Version targetVersion = -1, bool verbose = true, KeyRange range = normalKeys,
Key addPrefix = Key(), Key removePrefix = Key(), bool lockDB = true,
bool incrementalBackupOnly = false, Version beginVersion = -1) {
Future<Version> restore(Database cx,
Optional<Database> cxOrig,
Key tagName,
Key url,
bool waitForComplete = true,
Version targetVersion = -1,
bool verbose = true,
KeyRange range = normalKeys,
Key addPrefix = Key(),
Key removePrefix = Key(),
bool lockDB = true,
bool incrementalBackupOnly = false,
Version beginVersion = -1) {
Standalone<VectorRef<KeyRangeRef>> rangeRef;
rangeRef.push_back_deep(rangeRef.arena(), range);
return restore(cx, cxOrig, tagName, url, rangeRef, waitForComplete, targetVersion, verbose, addPrefix,
removePrefix, lockDB, incrementalBackupOnly, beginVersion);
return restore(cx,
cxOrig,
tagName,
url,
rangeRef,
waitForComplete,
targetVersion,
verbose,
addPrefix,
removePrefix,
lockDB,
incrementalBackupOnly,
beginVersion);
}
Future<Version> atomicRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix = Key(), Key removePrefix = Key());
Future<Version> atomicRestore(Database cx, Key tagName, KeyRange range = normalKeys, Key addPrefix = Key(), Key removePrefix = Key()) {
Future<Version> atomicRestore(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix = Key(),
Key removePrefix = Key());
Future<Version> atomicRestore(Database cx,
Key tagName,
KeyRange range = normalKeys,
Key addPrefix = Key(),
Key removePrefix = Key()) {
Standalone<VectorRef<KeyRangeRef>> rangeRef;
rangeRef.push_back_deep(rangeRef.arena(), range);
return atomicRestore(cx, tagName, rangeRef, addPrefix, removePrefix);
@ -315,27 +349,44 @@ public:
// Get a string describing the status of a tag
Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<std::string> restoreStatus(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return restoreStatus(tr, tagName); });
return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return restoreStatus(tr, tagName); });
}
/** BACKUP METHODS **/
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds,
std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true, bool partitionedLog = false,
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr,
Key outContainer,
int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
bool partitionedLog = false,
bool incrementalBackupOnly = false);
Future<Void> submitBackup(Database cx, Key outContainer, int snapshotIntervalSeconds, std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true,
bool partitionedLog = false, bool incrementalBackupOnly = false) {
Future<Void> submitBackup(Database cx,
Key outContainer,
int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
bool partitionedLog = false,
bool incrementalBackupOnly = false) {
return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr) {
return submitBackup(tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone,
partitionedLog, incrementalBackupOnly);
return submitBackup(tr,
outContainer,
snapshotIntervalSeconds,
tagName,
backupRanges,
stopWhenDone,
partitionedLog,
incrementalBackupOnly);
});
}
Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> discontinueBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return discontinueBackup(tr, tagName); });
return runRYWTransaction(
cx, [=](Reference<ReadYourWritesTransaction> tr) { return discontinueBackup(tr, tagName); });
}
// Terminate an ongoing backup, without waiting for the backup to finish.
@ -347,7 +398,7 @@ public:
// logRangesRange and backupLogKeys will be cleared for this backup.
Future<Void> abortBackup(Reference<ReadYourWritesTransaction> tr, std::string tagName);
Future<Void> abortBackup(Database cx, std::string tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return abortBackup(tr, tagName); });
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return abortBackup(tr, tagName); });
}
Future<std::string> getStatus(Database cx, bool showErrors, std::string tagName);
@ -358,8 +409,11 @@ public:
// stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it
// will return when the backup directory is restorable.
Future<EnumState> waitBackup(Database cx, std::string tagName, bool stopWhenDone = true,
Reference<IBackupContainer>* pContainer = nullptr, UID* pUID = nullptr);
Future<EnumState> waitBackup(Database cx,
std::string tagName,
bool stopWhenDone = true,
Reference<IBackupContainer>* pContainer = nullptr,
UID* pUID = nullptr);
static const Key keyLastRestorable;
@ -383,8 +437,14 @@ public:
Reference<FutureBucket> futureBucket;
};
template<> inline Tuple Codec<FileBackupAgent::ERestoreState>::pack(FileBackupAgent::ERestoreState const &val) { return Tuple().append(val); }
template<> inline FileBackupAgent::ERestoreState Codec<FileBackupAgent::ERestoreState>::unpack(Tuple const &val) { return (FileBackupAgent::ERestoreState)val.getInt(0); }
template <>
inline Tuple Codec<FileBackupAgent::ERestoreState>::pack(FileBackupAgent::ERestoreState const& val) {
return Tuple().append(val);
}
template <>
inline FileBackupAgent::ERestoreState Codec<FileBackupAgent::ERestoreState>::unpack(Tuple const& val) {
return (FileBackupAgent::ERestoreState)val.getInt(0);
}
class DatabaseBackupAgent : public BackupAgentBase {
public:
@ -410,45 +470,74 @@ public:
sourceTagNames = std::move(r.sourceTagNames);
}
Future<Void> run(Database cx, double *pollDelay, int maxConcurrentTasks) {
Future<Void> run(Database cx, double* pollDelay, int maxConcurrentTasks) {
return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
}
Future<Void> atomicSwitchover(Database dest, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, Key addPrefix, Key removePrefix, bool forceAction=false);
Future<Void> atomicSwitchover(Database dest,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix,
Key removePrefix,
bool forceAction = false);
Future<Void> unlockBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> unlockBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return unlockBackup(tr, tagName); });
return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return unlockBackup(tr, tagName); });
}
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key addPrefix = StringRef(), Key removePrefix = StringRef(), bool lockDatabase = false, bool databasesInSync=false);
Future<Void> submitBackup(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key addPrefix = StringRef(), Key removePrefix = StringRef(), bool lockDatabase = false, bool databasesInSync=false) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return submitBackup(tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, databasesInSync); });
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
Key addPrefix = StringRef(),
Key removePrefix = StringRef(),
bool lockDatabase = false,
bool databasesInSync = false);
Future<Void> submitBackup(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
Key addPrefix = StringRef(),
Key removePrefix = StringRef(),
bool lockDatabase = false,
bool databasesInSync = false) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
return submitBackup(
tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, databasesInSync);
});
}
Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> discontinueBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return discontinueBackup(tr, tagName); });
return runRYWTransaction(
cx, [=](Reference<ReadYourWritesTransaction> tr) { return discontinueBackup(tr, tagName); });
}
Future<Void> abortBackup(Database cx, Key tagName, bool partial = false, bool abortOldBackup = false,
bool dstOnly = false, bool waitForDestUID = false);
Future<Void> abortBackup(Database cx,
Key tagName,
bool partial = false,
bool abortOldBackup = false,
bool dstOnly = false,
bool waitForDestUID = false);
Future<std::string> getStatus(Database cx, int errorLimit, Key tagName);
Future<EnumState> getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
Future<EnumState> getStateValue(Database cx, UID logUid) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getStateValue(tr, logUid); });
return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return getStateValue(tr, logUid); });
}
Future<UID> getDestUid(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
Future<UID> getDestUid(Database cx, UID logUid) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getDestUid(tr, logUid); });
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return getDestUid(tr, logUid); });
}
Future<UID> getLogUid(Reference<ReadYourWritesTransaction> tr, Key tagName, bool snapshot = false);
Future<UID> getLogUid(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getLogUid(tr, tagName); });
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return getLogUid(tr, tagName); });
}
Future<int64_t> getRangeBytesWritten(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
@ -495,7 +584,7 @@ struct RCGroup {
Version version;
uint64_t groupKey;
RCGroup() : version(-1), groupKey(ULLONG_MAX) {};
RCGroup() : version(-1), groupKey(ULLONG_MAX){};
template <class Ar>
void serialize(Ar& ar) {
@ -505,25 +594,48 @@ struct RCGroup {
bool copyParameter(Reference<Task> source, Reference<Task> dest, Key key);
Version getVersionFromString(std::string const& value);
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion,
Version endVersion,
Key destUidValue,
int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version endVersion, Key backupUid);
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion = Optional<Version>(), bool checkBackupUid = false, Version backupUid = 0);
Key getApplyKey( Version version, Key backupUid );
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion = Optional<Version>(),
bool checkBackupUid = false,
Version backupUid = 0);
Key getApplyKey(Version version, Key backupUid);
Version getLogKeyVersion(Key key);
std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key);
Future<Void> logError(Database cx, Key keyErrors, const std::string& message);
Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, const std::string& message);
Future<Void> checkVersion(Reference<ReadYourWritesTransaction> const& tr);
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersion> results, Reference<FlowLock> lock,
KeyRangeRef range, bool terminator = true, bool systemAccess = false,
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RangeResultWithVersion> results,
Reference<FlowLock> lock,
KeyRangeRef range,
bool terminator = true,
bool systemAccess = false,
bool lockAware = false);
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Future<Void> active,
Reference<FlowLock> lock, KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy, bool terminator = true,
bool systemAccess = false, bool lockAware = false);
ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key removePrefix, Version beginVersion,
Version* endVersion, RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion, Reference<KeyRangeMap<Version>> keyVersion);
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RCGroup> results,
Future<Void> active,
Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
bool terminator = true,
bool systemAccess = false,
bool lockAware = false);
ACTOR Future<Void> applyMutations(Database cx,
Key uid,
Key addPrefix,
Key removePrefix,
Version beginVersion,
Version* endVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Reference<KeyRangeMap<Version>> keyVersion);
ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData);
using EBackupState = BackupAgentBase::EnumState;
@ -568,9 +680,11 @@ typedef KeyBackedMap<std::string, UidAndAbortedFlagT> TagMap;
// Map of tagName to {UID, aborted_flag} located in the fileRestorePrefixRange keyspace.
class TagUidMap : public KeyBackedMap<std::string, UidAndAbortedFlagT> {
public:
TagUidMap(const StringRef & prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {}
TagUidMap(const StringRef& prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {}
ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap, Reference<ReadYourWritesTransaction> tr, bool snapshot);
ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap,
Reference<ReadYourWritesTransaction> tr,
bool snapshot);
Future<std::vector<KeyBackedTag>> getAll(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
return getAll_impl(this, tr, snapshot);
@ -587,24 +701,24 @@ static inline KeyBackedTag makeBackupTag(std::string tagName) {
return KeyBackedTag(tagName, fileBackupPrefixRange.begin);
}
static inline Future<std::vector<KeyBackedTag>> getAllRestoreTags(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
static inline Future<std::vector<KeyBackedTag>> getAllRestoreTags(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false) {
return TagUidMap(fileRestorePrefixRange.begin).getAll(tr, snapshot);
}
static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false) {
return TagUidMap(fileBackupPrefixRange.begin).getAll(tr, snapshot);
}
class KeyBackedConfig {
public:
static struct {
static TaskParam<UID> uid() {return LiteralStringRef(__FUNCTION__); }
static TaskParam<UID> uid() { return LiteralStringRef(__FUNCTION__); }
} TaskParams;
KeyBackedConfig(StringRef prefix, UID uid = UID()) :
uid(uid),
prefix(prefix),
configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(prefix), uid)) {}
KeyBackedConfig(StringRef prefix, UID uid = UID())
: uid(uid), prefix(prefix), configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(prefix), uid)) {}
KeyBackedConfig(StringRef prefix, Reference<Task> task) : KeyBackedConfig(prefix, TaskParams.uid().get(task)) {}
@ -616,31 +730,28 @@ public:
return Void();
}
// Set the validation condition for the task which is that the restore uid's tag's uid is the same as the restore uid.
// Get this uid's tag, then get the KEY for the tag's uid but don't read it. That becomes the validation key
// which TaskBucket will check, and its value must be this restore config's uid.
UID u = uid; // 'this' could be invalid in lambda
// Set the validation condition for the task which is that the restore uid's tag's uid is the same as the
// restore uid. Get this uid's tag, then get the KEY for the tag's uid but don't read it. That becomes the
// validation key which TaskBucket will check, and its value must be this restore config's uid.
UID u = uid; // 'this' could be invalid in lambda
Key p = prefix;
return map(tag().get(tr), [u,p,task](Optional<std::string> const &tag) -> Void {
if(!tag.present())
return map(tag().get(tr), [u, p, task](Optional<std::string> const& tag) -> Void {
if (!tag.present())
throw restore_error();
// Validation contition is that the uidPair key must be exactly {u, false}
TaskBucket::setValidationCondition(task, KeyBackedTag(tag.get(), p).key, Codec<UidAndAbortedFlagT>::pack({u, false}).pack());
TaskBucket::setValidationCondition(
task, KeyBackedTag(tag.get(), p).key, Codec<UidAndAbortedFlagT>::pack({ u, false }).pack());
return Void();
});
}
KeyBackedProperty<std::string> tag() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<std::string> tag() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
UID getUid() { return uid; }
Key getUidAsKey() { return BinaryWriter::toValue(uid, Unversioned()); }
void clear(Reference<ReadYourWritesTransaction> tr) {
tr->clear(configSpace.range());
}
void clear(Reference<ReadYourWritesTransaction> tr) { tr->clear(configSpace.range()); }
// lastError is a pair of error message and timestamp expressed as an int64_t
KeyBackedProperty<std::pair<std::string, Version>> lastError() {
@ -654,15 +765,15 @@ public:
// Updates the error per type map and the last error property
Future<Void> updateErrorInfo(Database cx, Error e, std::string message) {
// Avoid capture of this ptr
auto &copy = *this;
auto& copy = *this;
return runRYWTransaction(cx, [=] (Reference<ReadYourWritesTransaction> tr) mutable {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) mutable {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
return map(tr->getReadVersion(), [=] (Version v) mutable {
copy.lastError().set(tr, {message, v});
copy.lastErrorPerType().set(tr, e.code(), {message, v});
return map(tr->getReadVersion(), [=](Version v) mutable {
copy.lastError().set(tr, { message, v });
copy.lastErrorPerType().set(tr, e.code(), { message, v });
return Void();
});
});
@ -674,10 +785,12 @@ protected:
Subspace configSpace;
};
template<> inline Tuple Codec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const &bc) {
template <>
inline Tuple Codec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const& bc) {
return Tuple().append(StringRef(bc->getURL()));
}
template<> inline Reference<IBackupContainer> Codec<Reference<IBackupContainer>>::unpack(Tuple const &val) {
template <>
inline Reference<IBackupContainer> Codec<Reference<IBackupContainer>>::unpack(Tuple const& val) {
return IBackupContainer::openContainer(val.getString(0).toString());
}
@ -695,7 +808,7 @@ public:
Tuple pack() const {
return Tuple().append(begin).append(version).append(StringRef(fileName)).append(fileSize);
}
static RangeSlice unpack(Tuple const &t) {
static RangeSlice unpack(Tuple const& t) {
RangeSlice r;
int i = 0;
r.begin = t.getString(i++);
@ -708,52 +821,34 @@ public:
// Map of range end boundaries to info about the backup file written for that range.
typedef KeyBackedMap<Key, RangeSlice> RangeFileMapT;
RangeFileMapT snapshotRangeFileMap() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
RangeFileMapT snapshotRangeFileMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Number of kv range files that were both committed to persistent storage AND inserted into
// the snapshotRangeFileMap. Note that since insertions could replace 1 or more existing
// map entries this is not necessarily the number of entries currently in the map.
// This value exists to help with sizing of kv range folders for BackupContainers that
// This value exists to help with sizing of kv range folders for BackupContainers that
// require it.
KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Coalesced set of ranges already dispatched for writing.
typedef KeyBackedMap<Key, bool> RangeDispatchMapT;
RangeDispatchMapT snapshotRangeDispatchMap() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
RangeDispatchMapT snapshotRangeDispatchMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Interval to use for determining the target end version for new snapshots
KeyBackedProperty<int64_t> snapshotIntervalSeconds() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<int64_t> snapshotIntervalSeconds() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// When the current snapshot began
KeyBackedProperty<Version> snapshotBeginVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> snapshotBeginVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// When the current snapshot is desired to end.
// This can be changed at runtime to speed up or slow down a snapshot
KeyBackedProperty<Version> snapshotTargetEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// When the current snapshot is desired to end.
// This can be changed at runtime to speed up or slow down a snapshot
KeyBackedProperty<Version> snapshotTargetEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<int64_t> snapshotBatchSize() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<int64_t> snapshotBatchSize() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Key> snapshotBatchFuture() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> snapshotBatchFuture() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<int64_t> snapshotDispatchLastShardsBehind() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
@ -764,14 +859,15 @@ public:
}
Future<Void> initNewSnapshot(Reference<ReadYourWritesTransaction> tr, int64_t intervalSeconds = -1) {
BackupConfig &copy = *this; // Capture this by value instead of this ptr
BackupConfig& copy = *this; // Capture this by value instead of this ptr
Future<Version> beginVersion = tr->getReadVersion();
Future<int64_t> defaultInterval = 0;
if(intervalSeconds < 0)
if (intervalSeconds < 0)
defaultInterval = copy.snapshotIntervalSeconds().getOrThrow(tr);
// Make sure read version and possibly the snapshot interval value are ready, then clear/init the snapshot config members
// Make sure read version and possibly the snapshot interval value are ready, then clear/init the snapshot
// config members
return map(success(beginVersion) && success(defaultInterval), [=](Void) mutable {
copy.snapshotRangeFileMap().clear(tr);
copy.snapshotRangeDispatchMap().clear(tr);
@ -779,10 +875,10 @@ public:
copy.snapshotBatchFuture().clear(tr);
copy.snapshotBatchDispatchDoneKey().clear(tr);
if(intervalSeconds < 0)
if (intervalSeconds < 0)
intervalSeconds = defaultInterval.get();
Version endVersion = beginVersion.get() + intervalSeconds * CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
copy.snapshotBeginVersion().set(tr, beginVersion.get());
copy.snapshotTargetEndVersion().set(tr, endVersion);
copy.snapshotRangeFileCount().set(tr, 0);
@ -793,26 +889,18 @@ public:
});
}
KeyBackedBinaryValue<int64_t> rangeBytesWritten() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedBinaryValue<int64_t> rangeBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedBinaryValue<int64_t> logBytesWritten() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedBinaryValue<int64_t> logBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<EBackupState> stateEnum() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<EBackupState> stateEnum() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Reference<IBackupContainer>> backupContainer() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Set to true when all backup workers for saving mutation logs have been started.
KeyBackedProperty<bool> allWorkerStarted() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> allWorkerStarted() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Each backup worker adds its (epoch, tag.id) to this property.
KeyBackedProperty<std::vector<std::pair<int64_t, int64_t>>> startedBackupWorkers() {
@ -820,19 +908,13 @@ public:
}
// Set to true if backup worker is enabled.
KeyBackedProperty<bool> backupWorkerEnabled() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> backupWorkerEnabled() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Set to true if partitioned log is enabled (only useful if backup worker is also enabled).
KeyBackedProperty<bool> partitionedLogEnabled() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> partitionedLogEnabled() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Set to true if only requesting incremental backup without base snapshot.
KeyBackedProperty<bool> incrementalBackupOnly() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> incrementalBackupOnly() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Latest version for which all prior versions have saved by backup workers.
KeyBackedProperty<Version> latestBackupWorkerSavedVersion() {
@ -840,28 +922,18 @@ public:
}
// Stop differntial logging if already started or don't start after completing KV ranges
KeyBackedProperty<bool> stopWhenDone() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> stopWhenDone() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Latest version for which all prior versions have had their log copy tasks completed
KeyBackedProperty<Version> latestLogEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> latestLogEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// The end version of the last complete snapshot
KeyBackedProperty<Version> latestSnapshotEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> latestSnapshotEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// The end version of the first complete snapshot
KeyBackedProperty<Version> firstSnapshotEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> firstSnapshotEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Key> destUidValue() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> destUidValue() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
Future<Optional<Version>> getLatestRestorableVersion(Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
@ -892,24 +964,26 @@ public:
});
}
KeyBackedProperty<std::vector<KeyRange>> backupRanges() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<std::vector<KeyRange>> backupRanges() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
void startMutationLogs(Reference<ReadYourWritesTransaction> tr, KeyRangeRef backupRange, Key destUidValue) {
Key mutationLogsDestKey = destUidValue.withPrefix(backupLogKeys.begin);
tr->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef<UID>(destUidValue, Unversioned())), logRangesEncodeValue(backupRange.end, mutationLogsDestKey));
tr->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef<UID>(destUidValue, Unversioned())),
logRangesEncodeValue(backupRange.end, mutationLogsDestKey));
}
Future<Void> logError(Database cx, Error e, std::string details, void *taskInstance = nullptr) {
if(!uid.isValid()) {
Future<Void> logError(Database cx, Error e, std::string details, void* taskInstance = nullptr) {
if (!uid.isValid()) {
TraceEvent(SevError, "FileBackupErrorNoUID").error(e).detail("Description", details);
return Void();
}
TraceEvent t(SevWarn, "FileBackupError");
t.error(e).detail("BackupUID", uid).detail("Description", details).detail("TaskInstance", (uint64_t)taskInstance);
t.error(e)
.detail("BackupUID", uid)
.detail("Description", details)
.detail("TaskInstance", (uint64_t)taskInstance);
// key_not_found could happen
if(e.code() == error_code_key_not_found)
if (e.code() == error_code_key_not_found)
t.backtrace();
return updateErrorInfo(cx, e, details);
@ -925,10 +999,12 @@ struct StringRefReader {
// Return a pointer to len bytes at the current read position and advance read pos
const uint8_t* consume(unsigned int len) {
if (rptr == end && len != 0) throw end_of_stream();
if (rptr == end && len != 0)
throw end_of_stream();
const uint8_t* p = rptr;
rptr += len;
if (rptr > end) throw failure_error;
if (rptr > end)
throw failure_error;
return p;
}
@ -954,19 +1030,22 @@ struct StringRefReader {
};
namespace fileBackup {
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file, int64_t offset,
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
int64_t offset,
int len);
// Return a block of contiguous padding bytes "\0xff" for backup files, growing if needed.
Value makePadding(int size);
}
} // namespace fileBackup
// For fast restore simulation test
// For testing addPrefix feature in fast restore.
// Transform db content in restoreRanges by removePrefix and then addPrefix.
// Assume: DB is locked
ACTOR Future<Void> transformRestoredDatabase(Database cx, Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix, Key removePrefix);
ACTOR Future<Void> transformRestoredDatabase(Database cx,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix,
Key removePrefix);
void simulateBlobFailure();

461
fdbclient/BackupAgentBase.actor.cpp
View File

@ -53,7 +53,7 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
return -1;
}
#else
if(strptime(timeOnly.c_str(), "%Y/%m/%d.%H:%M:%S", &out) == nullptr) {
if (strptime(timeOnly.c_str(), "%Y/%m/%d.%H:%M:%S", &out) == nullptr) {
return -1;
}
#endif
@ -61,25 +61,25 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
// Read timezone offset in +/-HHMM format then convert to seconds
int tzHH;
int tzMM;
if(sscanf(timestamp.substr(19, 5).c_str(), "%3d%2d", &tzHH, &tzMM) != 2) {
if (sscanf(timestamp.substr(19, 5).c_str(), "%3d%2d", &tzHH, &tzMM) != 2) {
return -1;
}
if(tzHH < 0) {
if (tzHH < 0) {
tzMM = -tzMM;
}
// tzOffset is the number of seconds EAST of GMT
int tzOffset = tzHH * 60 * 60 + tzMM * 60;
// The goal is to convert the timestamp string to epoch seconds assuming the date/time was expressed in the timezone at the end of the string.
// However, mktime() will ONLY return epoch seconds assuming the date/time is expressed in local time (based on locale / environment)
// mktime() will set out.tm_gmtoff when available
// The goal is to convert the timestamp string to epoch seconds assuming the date/time was expressed in the timezone
// at the end of the string. However, mktime() will ONLY return epoch seconds assuming the date/time is expressed in
// local time (based on locale / environment) mktime() will set out.tm_gmtoff when available
int64_t ts = mktime(&out);
// localTZOffset is the number of seconds EAST of GMT
long localTZOffset;
#ifdef _WIN32
// _get_timezone() returns the number of seconds WEST of GMT
if(_get_timezone(&localTZOffset) != 0) {
if (_get_timezone(&localTZOffset) != 0) {
return -1;
}
// Negate offset to match the orientation of tzOffset
@ -89,7 +89,8 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
localTZOffset = out.tm_gmtoff;
#endif
// Add back the difference between the local timezone assumed by mktime() and the intended timezone from the input string
// Add back the difference between the local timezone assumed by mktime() and the intended timezone from the input
// string
ts += (localTZOffset - tzOffset);
return ts;
}
@ -145,7 +146,10 @@ Version getVersionFromString(std::string const& value) {
// Return the ranges of keys that contain the data for the given range
// of versions.
// assert CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE % blocksize = 0. Otherwise calculation of hash will be incorrect
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize) {
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion,
Version endVersion,
Key destUidValue,
int blockSize) {
Standalone<VectorRef<KeyRangeRef>> ret;
Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin);
@ -161,8 +165,9 @@ Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version en
Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
ret.push_back_deep(ret.arena(), KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
ret.push_back_deep(ret.arena(),
KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
}
return ret;
@ -175,7 +180,9 @@ Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version
//TraceEvent("GetLogRanges").detail("BackupUid", backupUid).detail("Prefix", baLogRangePrefix);
for (int64_t vblock = beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE; vblock < (endVersion + CLIENT_KNOBS->APPLY_BLOCK_SIZE - 1) / CLIENT_KNOBS->APPLY_BLOCK_SIZE; ++vblock) {
for (int64_t vblock = beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE;
vblock < (endVersion + CLIENT_KNOBS->APPLY_BLOCK_SIZE - 1) / CLIENT_KNOBS->APPLY_BLOCK_SIZE;
++vblock) {
int64_t tb = vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
uint64_t bv = bigEndian64(std::max(beginVersion, vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE));
uint64_t ev = bigEndian64(std::min(endVersion, (vblock + 1) * CLIENT_KNOBS->APPLY_BLOCK_SIZE));
@ -184,15 +191,16 @@ Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version
Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
ret.push_back_deep(ret.arena(), KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
ret.push_back_deep(ret.arena(),
KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
}
return ret;
}
Key getApplyKey( Version version, Key backupUid ) {
int64_t vblock = (version-1) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
Key getApplyKey(Version version, Key backupUid) {
int64_t vblock = (version - 1) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
uint64_t v = bigEndian64(version);
uint32_t data = vblock & 0xffffffff;
uint8_t hash = (uint8_t)hashlittle(&data, sizeof(uint32_t), 0);
@ -205,24 +213,34 @@ Version getLogKeyVersion(Key key) {
return bigEndian64(*(int64_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t)));
}
//Given a key from one of the ranges returned by get_log_ranges,
//returns(version, part) where version is the database version number of
//the transaction log data in the value, and part is 0 for the first such
//data for a given version, 1 for the second block of data, etc.
// Given a key from one of the ranges returned by get_log_ranges,
// returns(version, part) where version is the database version number of
// the transaction log data in the value, and part is 0 for the first such
// data for a given version, 1 for the second block of data, etc.
std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key) {
return std::make_pair(getLogKeyVersion(key),
bigEndian32(*(int32_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t) + sizeof(int64_t))));
return std::make_pair(
getLogKeyVersion(key),
bigEndian32(*(int32_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t) + sizeof(int64_t))));
}
void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mutationSize, StringRef value, StringRef addPrefix, StringRef removePrefix, Version version, Reference<KeyRangeMap<Version>> key_version) {
void decodeBackupLogValue(Arena& arena,
VectorRef<MutationRef>& result,
int& mutationSize,
StringRef value,
StringRef addPrefix,
StringRef removePrefix,
Version version,
Reference<KeyRangeMap<Version>> key_version) {
try {
uint64_t offset(0);
uint64_t protocolVersion = 0;
memcpy(&protocolVersion, value.begin(), sizeof(uint64_t));
offset += sizeof(uint64_t);
if (protocolVersion <= 0x0FDB00A200090001){
TraceEvent(SevError, "DecodeBackupLogValue").detail("IncompatibleProtocolVersion", protocolVersion)
.detail("ValueSize", value.size()).detail("Value", value);
if (protocolVersion <= 0x0FDB00A200090001) {
TraceEvent(SevError, "DecodeBackupLogValue")
.detail("IncompatibleProtocolVersion", protocolVersion)
.detail("ValueSize", value.size())
.detail("Value", value);
throw incompatible_protocol_version();
}
@ -231,12 +249,12 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
offset += sizeof(uint32_t);
uint32_t consumed = 0;
if(totalBytes + offset > value.size())
if (totalBytes + offset > value.size())
throw restore_missing_data();
int originalOffset = offset;
while (consumed < totalBytes){
while (consumed < totalBytes) {
uint32_t type = 0;
memcpy(&type, value.begin() + offset, sizeof(uint32_t));
offset += sizeof(uint32_t);
@ -247,7 +265,7 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
memcpy(&len2, value.begin() + offset, sizeof(uint32_t));
offset += sizeof(uint32_t);
ASSERT(offset+len1+len2<=value.size() && isValidMutationType(type));
ASSERT(offset + len1 + len2 <= value.size() && isValidMutationType(type));
MutationRef logValue;
Arena tempArena;
@ -265,24 +283,23 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
KeyRef minKey = std::min(r.range().end, range.end);
if (minKey == (removePrefix == StringRef() ? normalKeys.end : strinc(removePrefix))) {
logValue.param1 = std::max(r.range().begin, range.begin);
if(removePrefix.size()) {
if (removePrefix.size()) {
logValue.param1 = logValue.param1.removePrefix(removePrefix);
}
if(addPrefix.size()) {
if (addPrefix.size()) {
logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
}
logValue.param2 = addPrefix == StringRef() ? normalKeys.end : strinc(addPrefix, tempArena);
result.push_back_deep(arena, logValue);
mutationSize += logValue.expectedSize();
}
else {
} else {
logValue.param1 = std::max(r.range().begin, range.begin);
logValue.param2 = minKey;
if(removePrefix.size()) {
if (removePrefix.size()) {
logValue.param1 = logValue.param1.removePrefix(removePrefix);
logValue.param2 = logValue.param2.removePrefix(removePrefix);
}
if(addPrefix.size()) {
if (addPrefix.size()) {
logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
logValue.param2 = logValue.param2.withPrefix(addPrefix, tempArena);
}
@ -291,15 +308,14 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
}
}
}
}
else {
} else {
Version ver = key_version->rangeContaining(logValue.param1).value();
//TraceEvent("ApplyMutation").detail("LogValue", logValue.toString()).detail("Version", version).detail("Ver", ver).detail("Apply", version > ver && ver != invalidVersion);
if (version > ver && ver != invalidVersion) {
if(removePrefix.size()) {
if (removePrefix.size()) {
logValue.param1 = logValue.param1.removePrefix(removePrefix);
}
if(addPrefix.size()) {
if (addPrefix.size()) {
logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
}
result.push_back_deep(arena, logValue);
@ -312,12 +328,20 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
ASSERT(consumed == totalBytes);
if (value.size() != offset) {
TraceEvent(SevError, "BA_DecodeBackupLogValue").detail("UnexpectedExtraDataSize", value.size()).detail("Offset", offset).detail("TotalBytes", totalBytes).detail("Consumed", consumed).detail("OriginalOffset", originalOffset);
TraceEvent(SevError, "BA_DecodeBackupLogValue")
.detail("UnexpectedExtraDataSize", value.size())
.detail("Offset", offset)
.detail("TotalBytes", totalBytes)
.detail("Consumed", consumed)
.detail("OriginalOffset", originalOffset);
throw restore_corrupted_data();
}
}
catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_DecodeBackupLogValue").error(e).GetLastError().detail("ValueSize", value.size()).detail("Value", value);
} catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_DecodeBackupLogValue")
.error(e)
.GetLastError()
.detail("ValueSize", value.size())
.detail("Value", value);
throw;
}
}
@ -327,7 +351,7 @@ static double lastErrorTime = 0;
void logErrorWorker(Reference<ReadYourWritesTransaction> tr, Key keyErrors, std::string message) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
if(now() - lastErrorTime > CLIENT_KNOBS->BACKUP_ERROR_DELAY) {
if (now() - lastErrorTime > CLIENT_KNOBS->BACKUP_ERROR_DELAY) {
TraceEvent("BA_LogError").detail("Key", keyErrors).detail("Message", message);
lastErrorTime = now();
}
@ -335,8 +359,8 @@ void logErrorWorker(Reference<ReadYourWritesTransaction> tr, Key keyErrors, std:
}
Future<Void> logError(Database cx, Key keyErrors, const std::string& message) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
logErrorWorker(tr, keyErrors, message);
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
logErrorWorker(tr, keyErrors, message);
return Future<Void>(Void());
});
}
@ -345,14 +369,19 @@ Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, co
return logError(tr->getDatabase(), keyErrors, message);
}
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersion> results, Reference<FlowLock> lock,
KeyRangeRef range, bool terminator, bool systemAccess, bool lockAware) {
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RangeResultWithVersion> results,
Reference<FlowLock> lock,
KeyRangeRef range,
bool terminator,
bool systemAccess,
bool lockAware) {
state KeySelector begin = firstGreaterOrEqual(range.begin);
state KeySelector end = firstGreaterOrEqual(range.end);
state Transaction tr(cx);
state FlowLock::Releaser releaser;
loop{
loop {
try {
state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
(g_network->isSimulated() && !g_simulator.speedUpSimulation)
@ -364,23 +393,26 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
if (lockAware)
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
//add lock
// add lock
releaser.release();
wait(lock->take(TaskPriority::DefaultYield, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT));
releaser = FlowLock::Releaser(*lock, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
wait(lock->take(TaskPriority::DefaultYield,
limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT));
releaser = FlowLock::Releaser(
*lock, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
state Standalone<RangeResultRef> values = wait(tr.getRange(begin, end, limits));
// When this buggify line is enabled, if there are more than 1 result then use half of the results
if(values.size() > 1 && BUGGIFY) {
if (values.size() > 1 && BUGGIFY) {
values.resize(values.arena(), values.size() / 2);
values.more = true;
// Half of the time wait for this tr to expire so that the next read is at a different version
if(deterministicRandom()->random01() < 0.5)
if (deterministicRandom()->random01() < 0.5)
wait(delay(6.0));
}
releaser.remaining -= values.expectedSize(); //its the responsibility of the caller to release after this point
releaser.remaining -=
values.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0);
results.send(RangeResultWithVersion(values, tr.getReadVersion().get()));
@ -389,28 +421,31 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
begin = firstGreaterThan(values.end()[-1].key);
if (!values.more && !limits.isReached()) {
if(terminator)
if (terminator)
results.sendError(end_of_stream());
return Void();
}
}
catch (Error &e) {
} catch (Error& e) {
if (e.code() == error_code_transaction_too_old) {
// We are using this transaction until it's too old and then resetting to a fresh one,
// so we don't need to delay.
tr.fullReset();
}
else {
} else {
wait(tr.onError(e));
}
}
}
}
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Future<Void> active, Reference<FlowLock> lock,
KeyRangeRef range, std::function< std::pair<uint64_t, uint32_t>(Key key) > groupBy,
bool terminator, bool systemAccess, bool lockAware)
{
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RCGroup> results,
Future<Void> active,
Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
bool terminator,
bool systemAccess,
bool lockAware) {
state KeySelector nextKey = firstGreaterOrEqual(range.begin);
state KeySelector end = firstGreaterOrEqual(range.end);
@ -419,7 +454,7 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
state Transaction tr(cx);
state FlowLock::Releaser releaser;
loop{
loop {
try {
state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
(g_network->isSimulated() && !g_simulator.speedUpSimulation)
@ -434,36 +469,37 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
state Standalone<RangeResultRef> rangevalue = wait(tr.getRange(nextKey, end, limits));
// When this buggify line is enabled, if there are more than 1 result then use half of the results
if(rangevalue.size() > 1 && BUGGIFY) {
if (rangevalue.size() > 1 && BUGGIFY) {
rangevalue.resize(rangevalue.arena(), rangevalue.size() / 2);
rangevalue.more = true;
// Half of the time wait for this tr to expire so that the next read is at a different version
if(deterministicRandom()->random01() < 0.5)
if (deterministicRandom()->random01() < 0.5)
wait(delay(6.0));
}
//add lock
// add lock
wait(active);
releaser.release();
wait(lock->take(TaskPriority::DefaultYield, rangevalue.expectedSize() + rcGroup.items.expectedSize()));
releaser = FlowLock::Releaser(*lock, rangevalue.expectedSize() + rcGroup.items.expectedSize());
for (auto & s : rangevalue){
for (auto& s : rangevalue) {
uint64_t groupKey = groupBy(s.key).first;
//TraceEvent("Log_ReadCommitted").detail("GroupKey", groupKey).detail("SkipGroup", skipGroup).detail("NextKey", nextKey.key).detail("End", end.key).detail("Valuesize", value.size()).detail("Index",index++).detail("Size",s.value.size());
if (groupKey != skipGroup){
if (rcGroup.version == -1){
if (groupKey != skipGroup) {
if (rcGroup.version == -1) {
rcGroup.version = tr.getReadVersion().get();
rcGroup.groupKey = groupKey;
}
else if (rcGroup.groupKey != groupKey) {
} else if (rcGroup.groupKey != groupKey) {
//TraceEvent("Log_ReadCommitted").detail("SendGroup0", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength",rcGroup.items[0].value.size());
//state uint32_t len(0);
//for (size_t j = 0; j < rcGroup.items.size(); ++j) {
// state uint32_t len(0);
// for (size_t j = 0; j < rcGroup.items.size(); ++j) {
// len += rcGroup.items[j].value.size();
//}
//TraceEvent("SendGroup").detail("GroupKey", rcGroup.groupKey).detail("Version", rcGroup.version).detail("Length", len).detail("Releaser.remaining", releaser.remaining);
releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point
releaser.remaining -=
rcGroup.items
.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0);
results.send(rcGroup);
nextKey = firstGreaterThan(rcGroup.items.end()[-1].key);
@ -478,39 +514,54 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
}
if (!rangevalue.more) {
if (rcGroup.version != -1){
releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point
if (rcGroup.version != -1) {
releaser.remaining -=
rcGroup.items
.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0);
//TraceEvent("Log_ReadCommitted").detail("SendGroup1", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength", rcGroup.items[0].value.size());
results.send(rcGroup);
}
if(terminator)
if (terminator)
results.sendError(end_of_stream());
return Void();
}
nextKey = firstGreaterThan(rangevalue.end()[-1].key);
}
catch (Error &e) {
} catch (Error& e) {
if (e.code() == error_code_transaction_too_old) {
// We are using this transaction until it's too old and then resetting to a fresh one,
// so we don't need to delay.
tr.fullReset();
}
else {
} else {
wait(tr.onError(e));
}
}
}
}
Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Reference<FlowLock> lock, KeyRangeRef range, std::function< std::pair<uint64_t, uint32_t>(Key key) > groupBy) {
Future<Void> readCommitted(Database cx,
PromiseStream<RCGroup> results,
Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy) {
return readCommitted(cx, results, Void(), lock, range, groupBy, true, true, true);
}
ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Reference<FlowLock> lock, Key uid, Key addPrefix, Key removePrefix, RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion, Optional<Version> endVersion, Key rangeBegin, PromiseStream<Future<Void>> addActor, FlowLock* commitLock, Reference<KeyRangeMap<Version>> keyVersion ) {
ACTOR Future<int> dumpData(Database cx,
PromiseStream<RCGroup> results,
Reference<FlowLock> lock,
Key uid,
Key addPrefix,
Key removePrefix,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Optional<Version> endVersion,
Key rangeBegin,
PromiseStream<Future<Void>> addActor,
FlowLock* commitLock,
Reference<KeyRangeMap<Version>> keyVersion) {
state Version lastVersion = invalidVersion;
state bool endOfStream = false;
state int totalBytes = 0;
@ -524,21 +575,27 @@ ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Referenc
lock->release(group.items.expectedSize());
BinaryWriter bw(Unversioned());
for(int i = 0; i < group.items.size(); ++i) {
for (int i = 0; i < group.items.size(); ++i) {
bw.serializeBytes(group.items[i].value);
}
decodeBackupLogValue(req.arena, req.transaction.mutations, mutationSize, bw.toValue(), addPrefix, removePrefix, group.groupKey, keyVersion);
decodeBackupLogValue(req.arena,
req.transaction.mutations,
mutationSize,
bw.toValue(),
addPrefix,
removePrefix,
group.groupKey,
keyVersion);
newBeginVersion = group.groupKey + 1;
if(mutationSize >= CLIENT_KNOBS->BACKUP_LOG_WRITE_BATCH_MAX_SIZE) {
if (mutationSize >= CLIENT_KNOBS->BACKUP_LOG_WRITE_BATCH_MAX_SIZE) {
break;
}
}
catch (Error &e) {
} catch (Error& e) {
if (e.code() == error_code_end_of_stream) {
if(endVersion.present() && endVersion.get() > lastVersion && endVersion.get() > newBeginVersion) {
if (endVersion.present() && endVersion.get() > lastVersion && endVersion.get() > newBeginVersion) {
newBeginVersion = endVersion.get();
}
if(newBeginVersion == invalidVersion)
if (newBeginVersion == invalidVersion)
return totalBytes;
endOfStream = true;
break;
@ -563,31 +620,39 @@ ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Referenc
req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE;
totalBytes += mutationSize;
wait( commitLock->take(TaskPriority::DefaultYield, mutationSize) );
addActor.send( commitLock->releaseWhen( success(commit.getReply(req)), mutationSize ) );
wait(commitLock->take(TaskPriority::DefaultYield, mutationSize));
addActor.send(commitLock->releaseWhen(success(commit.getReply(req)), mutationSize));
if(endOfStream) {
if (endOfStream) {
return totalBytes;
}
}
}
ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Reference<KeyRangeMap<Version>> keyVersion, RequestStream<CommitTransactionRequest> commit, NotifiedVersion* committedVersion, PromiseStream<Future<Void>> addActor, FlowLock* commitLock) {
ACTOR Future<Void> coalesceKeyVersionCache(Key uid,
Version endVersion,
Reference<KeyRangeMap<Version>> keyVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
PromiseStream<Future<Void>> addActor,
FlowLock* commitLock) {
Version lastVersion = -1000;
int64_t removed = 0;
state CommitTransactionRequest req;
state int64_t mutationSize = 0;
Key mapPrefix = uid.withPrefix(applyMutationsKeyVersionMapRange.begin);
for(auto it : keyVersion->ranges()) {
if( lastVersion == -1000 ) {
for (auto it : keyVersion->ranges()) {
if (lastVersion == -1000) {
lastVersion = it.value();
} else {
Version ver = it.value();
if(ver < endVersion && lastVersion < endVersion && ver != invalidVersion && lastVersion != invalidVersion) {
if (ver < endVersion && lastVersion < endVersion && ver != invalidVersion &&
lastVersion != invalidVersion) {
Key removeKey = it.range().begin.withPrefix(mapPrefix);
Key removeEnd = keyAfter(removeKey);
req.transaction.mutations.push_back_deep(req.arena, MutationRef(MutationRef::ClearRange, removeKey, removeEnd));
req.transaction.mutations.push_back_deep(req.arena,
MutationRef(MutationRef::ClearRange, removeKey, removeEnd));
mutationSize += removeKey.size() + removeEnd.size();
removed--;
} else {
@ -596,40 +661,51 @@ ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Referenc
}
}
if(removed != 0) {
if (removed != 0) {
Key countKey = uid.withPrefix(applyMutationsKeyVersionCountRange.begin);
req.transaction.write_conflict_ranges.push_back_deep(req.arena, singleKeyRange(countKey));
req.transaction.mutations.push_back_deep(req.arena, MutationRef(MutationRef::AddValue, countKey, StringRef((uint8_t*)&removed, 8)));
req.transaction.mutations.push_back_deep(
req.arena, MutationRef(MutationRef::AddValue, countKey, StringRef((uint8_t*)&removed, 8)));
req.transaction.read_snapshot = committedVersion->get();
req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE;
wait( commitLock->take(TaskPriority::DefaultYield, mutationSize) );
addActor.send( commitLock->releaseWhen( success(commit.getReply(req)), mutationSize ) );
wait(commitLock->take(TaskPriority::DefaultYield, mutationSize));
addActor.send(commitLock->releaseWhen(success(commit.getReply(req)), mutationSize));
}
return Void();
}
ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key removePrefix, Version beginVersion, Version* endVersion, RequestStream<CommitTransactionRequest> commit, NotifiedVersion* committedVersion, Reference<KeyRangeMap<Version>> keyVersion ) {
ACTOR Future<Void> applyMutations(Database cx,
Key uid,
Key addPrefix,
Key removePrefix,
Version beginVersion,
Version* endVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Reference<KeyRangeMap<Version>> keyVersion) {
state FlowLock commitLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
state PromiseStream<Future<Void>> addActor;
state Future<Void> error = actorCollection( addActor.getFuture() );
state Future<Void> error = actorCollection(addActor.getFuture());
state int maxBytes = CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES;
keyVersion->insert(metadataVersionKey, 0);
try {
loop {
if(beginVersion >= *endVersion) {
wait( commitLock.take(TaskPriority::DefaultYield, CLIENT_KNOBS->BACKUP_LOCK_BYTES) );
if (beginVersion >= *endVersion) {
wait(commitLock.take(TaskPriority::DefaultYield, CLIENT_KNOBS->BACKUP_LOCK_BYTES));
commitLock.release(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
if(beginVersion >= *endVersion) {
if (beginVersion >= *endVersion) {
return Void();
}
}
int rangeCount = std::max(1, CLIENT_KNOBS->APPLY_MAX_LOCK_BYTES / maxBytes);
state Version newEndVersion = std::min(*endVersion, ((beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE) + rangeCount) * CLIENT_KNOBS->APPLY_BLOCK_SIZE);
state Version newEndVersion = std::min(*endVersion,
((beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE) + rangeCount) *
CLIENT_KNOBS->APPLY_BLOCK_SIZE);
state Standalone<VectorRef<KeyRangeRef>> ranges = getApplyRanges(beginVersion, newEndVersion, uid);
state size_t idx;
state std::vector<PromiseStream<RCGroup>> results;
@ -643,23 +719,43 @@ ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key remov
rc.push_back(readCommitted(cx, results[i], locks[i], ranges[i], decodeBKMutationLogKey));
}
maxBytes = std::max<int>(maxBytes*CLIENT_KNOBS->APPLY_MAX_DECAY_RATE, CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES);
maxBytes = std::max<int>(maxBytes * CLIENT_KNOBS->APPLY_MAX_DECAY_RATE, CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES);
for (idx = 0; idx < ranges.size(); ++idx) {
int bytes = wait(dumpData(cx, results[idx], locks[idx], uid, addPrefix, removePrefix, commit, committedVersion, idx==ranges.size()-1 ? newEndVersion : Optional<Version>(), ranges[idx].begin, addActor, &commitLock, keyVersion));
maxBytes = std::max<int>(CLIENT_KNOBS->APPLY_MAX_INCREASE_FACTOR*bytes, maxBytes);
if(error.isError()) throw error.getError();
int bytes = wait(dumpData(cx,
results[idx],
locks[idx],
uid,
addPrefix,
removePrefix,
commit,
committedVersion,
idx == ranges.size() - 1 ? newEndVersion : Optional<Version>(),
ranges[idx].begin,
addActor,
&commitLock,
keyVersion));
maxBytes = std::max<int>(CLIENT_KNOBS->APPLY_MAX_INCREASE_FACTOR * bytes, maxBytes);
if (error.isError())
throw error.getError();
}
wait(coalesceKeyVersionCache(uid, newEndVersion, keyVersion, commit, committedVersion, addActor, &commitLock));
wait(coalesceKeyVersionCache(
uid, newEndVersion, keyVersion, commit, committedVersion, addActor, &commitLock));
beginVersion = newEndVersion;
}
} catch( Error &e ) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarnAlways : SevError, "ApplyMutationsError").error(e);
throw;
} catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarnAlways : SevError, "ApplyMutationsError")
.error(e);
throw;
}
}
ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) {
ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion,
bool checkBackupUid,
Version backupUid) {
state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
state Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath);
@ -671,13 +767,15 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (checkBackupUid) {
Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
Optional<Value> v = wait( tr->get( sourceStates.pack(DatabaseBackupAgent::keyFolderId) ) );
if(v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) > backupUid)
Subspace sourceStates =
Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
Optional<Value> v = wait(tr->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId)));
if (v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) > backupUid)
return Void();
}
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
state Standalone<RangeResultRef> backupVersions = wait(
tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
// Make sure version history key does exist and lower the beginVersion if needed
state Version currBeginVersion = invalidVersion;
@ -696,7 +794,7 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
}
state Version currEndVersion = std::numeric_limits<Version>::max();
if(endVersion.present()) {
if (endVersion.present()) {
currEndVersion = std::min(currEndVersion, endVersion.get());
}
@ -725,7 +823,7 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
if (!endVersion.present()) {
// Clear current backup version history
tr->clear(backupLatestVersionsKey);
if(backupVersions.size() == 1) {
if (backupVersions.size() == 1) {
tr->clear(prefixRange(destUidValue.withPrefix(logRangesRange.begin)));
}
} else {
@ -735,19 +833,22 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
// Clear log ranges if needed
if (clearLogRangesRequired) {
if((nextSmallestVersion - currBeginVersion) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE >= std::numeric_limits<uint8_t>::max() || BUGGIFY) {
if ((nextSmallestVersion - currBeginVersion) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE >=
std::numeric_limits<uint8_t>::max() ||
BUGGIFY) {
Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin);
for(int h = 0; h <= std::numeric_limits<uint8_t>::max(); h++) {
for (int h = 0; h <= std::numeric_limits<uint8_t>::max(); h++) {
uint64_t bv = bigEndian64(Version(0));
uint64_t ev = bigEndian64(nextSmallestVersion);
uint8_t h1 = h;
Key vblockPrefix = StringRef(&h1, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
tr->clear(KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
}
} else {
Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(currBeginVersion, nextSmallestVersion, destUidValue);
Standalone<VectorRef<KeyRangeRef>> ranges =
getLogRanges(currBeginVersion, nextSmallestVersion, destUidValue);
for (auto& range : ranges) {
tr->clear(range);
}
@ -763,11 +864,12 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
// Disable committing mutations into blog
tr->clear(prefixRange(destUidValue.withPrefix(logRangesRange.begin)));
}
if(!endVersion.present() && backupVersions.size() == 1) {
Standalone<RangeResultRef> existingDestUidValues = wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
for(auto it : existingDestUidValues) {
if( it.value == destUidValue ) {
if (!endVersion.present() && backupVersions.size() == 1) {
Standalone<RangeResultRef> existingDestUidValues =
wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
for (auto it : existingDestUidValues) {
if (it.value == destUidValue) {
tr->clear(it.key);
}
}
@ -776,7 +878,12 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
return Void();
}
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) {
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion,
bool checkBackupUid,
Version backupUid) {
return _eraseLogData(tr, logUidValue, destUidValue, endVersion, checkBackupUid, backupUid);
}
@ -792,58 +899,81 @@ ACTOR Future<Void> cleanupLogMutations(Database cx, Value destUidValue, bool del
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(
KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
state Version readVer = tr->getReadVersion().get();
state Version minVersion = std::numeric_limits<Version>::max();
state Key minVersionLogUid;
state int backupIdx = 0;
for (; backupIdx < backupVersions.size(); backupIdx++) {
state Version currVersion = BinaryReader::fromStringRef<Version>(backupVersions[backupIdx].value, Unversioned());
state Key currLogUid = backupVersions[backupIdx].key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
if( currVersion < minVersion ) {
state Version currVersion =
BinaryReader::fromStringRef<Version>(backupVersions[backupIdx].value, Unversioned());
state Key currLogUid =
backupVersions[backupIdx].key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
if (currVersion < minVersion) {
minVersionLogUid = currLogUid;
minVersion = currVersion;
}
if(!loggedLogUids.count(currLogUid)) {
state Future<Optional<Value>> foundDRKey = tr->get(Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(currLogUid).pack(DatabaseBackupAgent::keyStateStatus));
state Future<Optional<Value>> foundBackupKey = tr->get(Subspace(currLogUid.withPrefix(LiteralStringRef("uid->config/")).withPrefix(fileBackupPrefixRange.begin)).pack(LiteralStringRef("stateEnum")));
if (!loggedLogUids.count(currLogUid)) {
state Future<Optional<Value>> foundDRKey = tr->get(Subspace(databaseBackupPrefixRange.begin)
.get(BackupAgentBase::keySourceStates)
.get(currLogUid)
.pack(DatabaseBackupAgent::keyStateStatus));
state Future<Optional<Value>> foundBackupKey =
tr->get(Subspace(currLogUid.withPrefix(LiteralStringRef("uid->config/"))
.withPrefix(fileBackupPrefixRange.begin))
.pack(LiteralStringRef("stateEnum")));
wait(success(foundDRKey) && success(foundBackupKey));
if(foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("WARNING: Found a tag that looks like both a backup and a DR. This tag is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if(foundDRKey.get().present() && !foundBackupKey.get().present()) {
printf("Found a DR that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if(!foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("Found a Backup that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
if (foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("WARNING: Found a tag that looks like both a backup and a DR. This tag is %.4f hours "
"behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (foundDRKey.get().present() && !foundBackupKey.get().present()) {
printf("Found a DR that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (!foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("Found a Backup that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else {
printf("WARNING: Found an unknown tag that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
printf("WARNING: Found an unknown tag that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
}
loggedLogUids.insert(currLogUid);
}
}
if(deleteData) {
if(readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS*CLIENT_KNOBS->CORE_VERSIONSPERSECOND && (!removingLogUid.present() || minVersionLogUid == removingLogUid.get())) {
if (deleteData) {
if (readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS * CLIENT_KNOBS->CORE_VERSIONSPERSECOND &&
(!removingLogUid.present() || minVersionLogUid == removingLogUid.get())) {
removingLogUid = minVersionLogUid;
wait(eraseLogData(tr, minVersionLogUid, destUidValue));
wait(tr->commit());
printf("\nSuccessfully removed the tag that was %.4f hours behind.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if(removingLogUid.present() && minVersionLogUid != removingLogUid.get()) {
printf("\nWARNING: The oldest tag was possibly removed, run again without `--delete_data' to check.\n\n");
printf("\nSuccessfully removed the tag that was %.4f hours behind.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (removingLogUid.present() && minVersionLogUid != removingLogUid.get()) {
printf("\nWARNING: The oldest tag was possibly removed, run again without `--delete_data' to "
"check.\n\n");
} else {
printf("\nWARNING: Did not delete data because the tag is not at least %.4f hours behind. Change `--min_cleanup_seconds' to adjust this threshold.\n\n", CLIENT_KNOBS->MIN_CLEANUP_SECONDS/3600.0);
printf("\nWARNING: Did not delete data because the tag is not at least %.4f hours behind. Change "
"`--min_cleanup_seconds' to adjust this threshold.\n\n",
CLIENT_KNOBS->MIN_CLEANUP_SECONDS / 3600.0);
}
} else if(readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS*CLIENT_KNOBS->CORE_VERSIONSPERSECOND) {
printf("\nPassing `--delete_data' would delete the tag that is %.4f hours behind.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (readVer - minVersion >
CLIENT_KNOBS->MIN_CLEANUP_SECONDS * CLIENT_KNOBS->CORE_VERSIONSPERSECOND) {
printf("\nPassing `--delete_data' would delete the tag that is %.4f hours behind.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else {
printf("\nPassing `--delete_data' would not delete the tag that is %.4f hours behind. Change `--min_cleanup_seconds' to adjust the cleanup threshold.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
printf("\nPassing `--delete_data' would not delete the tag that is %.4f hours behind. Change "
"`--min_cleanup_seconds' to adjust the cleanup threshold.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
}
return Void();
} catch( Error& e) {
} catch (Error& e) {
wait(tr->onError(e));
}
}
@ -856,13 +986,14 @@ ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData) {
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Standalone<RangeResultRef> destUids = wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
state Standalone<RangeResultRef> destUids = wait(
tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
for(auto destUid : destUids) {
for (auto destUid : destUids) {
wait(cleanupLogMutations(cx, destUid.value, deleteData));
}
return Void();
} catch( Error& e) {
} catch (Error& e) {
wait(tr->onError(e));
}
}

118
fdbclient/BackupContainer.actor.cpp
View File

@ -67,31 +67,31 @@ Future<Void> IBackupFile::appendStringRefWithLen(Standalone<StringRef> s) {
std::string IBackupContainer::ExpireProgress::toString() const {
std::string s = step + "...";
if(total > 0) {
if (total > 0) {
s += format("%d/%d (%.2f%%)", done, total, double(done) / total * 100);
}
return s;
}
void BackupFileList::toStream(FILE *fout) const {
for(const RangeFile &f : ranges) {
void BackupFileList::toStream(FILE* fout) const {
for (const RangeFile& f : ranges) {
fprintf(fout, "range %" PRId64 " %s\n", f.fileSize, f.fileName.c_str());
}
for(const LogFile &f : logs) {
for (const LogFile& f : logs) {
fprintf(fout, "log %" PRId64 " %s\n", f.fileSize, f.fileName.c_str());
}
for(const KeyspaceSnapshotFile &f : snapshots) {
for (const KeyspaceSnapshotFile& f : snapshots) {
fprintf(fout, "snapshotManifest %" PRId64 " %s\n", f.totalSize, f.fileName.c_str());
}
}
Future<Void> fetchTimes(Reference<ReadYourWritesTransaction> tr, std::map<Version, int64_t> *pVersionTimeMap) {
Future<Void> fetchTimes(Reference<ReadYourWritesTransaction> tr, std::map<Version, int64_t>* pVersionTimeMap) {
std::vector<Future<Void>> futures;
// Resolve each version in the map,
for(auto &p : *pVersionTimeMap) {
for (auto& p : *pVersionTimeMap) {
futures.push_back(map(timeKeeperEpochsFromVersion(p.first, tr), [=](Optional<int64_t> t) {
if(t.present())
if (t.present())
pVersionTimeMap->at(p.first) = t.get();
else
pVersionTimeMap->erase(p.first);
@ -106,22 +106,23 @@ Future<Void> BackupDescription::resolveVersionTimes(Database cx) {
// Populate map with versions needed
versionTimeMap.clear();
for(const KeyspaceSnapshotFile &m : snapshots) {
for (const KeyspaceSnapshotFile& m : snapshots) {
versionTimeMap[m.beginVersion];
versionTimeMap[m.endVersion];
}
if(minLogBegin.present())
if (minLogBegin.present())
versionTimeMap[minLogBegin.get()];
if(maxLogEnd.present())
if (maxLogEnd.present())
versionTimeMap[maxLogEnd.get()];
if(contiguousLogEnd.present())
if (contiguousLogEnd.present())
versionTimeMap[contiguousLogEnd.get()];
if(minRestorableVersion.present())
if (minRestorableVersion.present())
versionTimeMap[minRestorableVersion.get()];
if(maxRestorableVersion.present())
if (maxRestorableVersion.present())
versionTimeMap[maxRestorableVersion.get()];
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return fetchTimes(tr, &versionTimeMap); });
return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return fetchTimes(tr, &versionTimeMap); });
};
std::string BackupDescription::toString() const {
@ -133,46 +134,49 @@ std::string BackupDescription::toString() const {
auto formatVersion = [&](Version v) {
std::string s;
if(!versionTimeMap.empty()) {
if (!versionTimeMap.empty()) {
auto i = versionTimeMap.find(v);
if(i != versionTimeMap.end())
if (i != versionTimeMap.end())
s = format("%lld (%s)", v, BackupAgentBase::formatTime(i->second).c_str());
else
s = format("%lld (unknown)", v);
}
else if(maxLogEnd.present()) {
} else if (maxLogEnd.present()) {
double days = double(maxLogEnd.get() - v) / (CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 60 * 60);
s = format("%lld (maxLogEnd %s%.2f days)", v, days < 0 ? "+" : "-", days);
}
else {
} else {
s = format("%lld", v);
}
return s;
};
for(const KeyspaceSnapshotFile &m : snapshots) {
info.append(format("Snapshot: startVersion=%s endVersion=%s totalBytes=%lld restorable=%s expiredPct=%.2f\n",
formatVersion(m.beginVersion).c_str(), formatVersion(m.endVersion).c_str(), m.totalSize, m.restorable.orDefault(false) ? "true" : "false", m.expiredPct(expiredEndVersion)));
for (const KeyspaceSnapshotFile& m : snapshots) {
info.append(
format("Snapshot: startVersion=%s endVersion=%s totalBytes=%lld restorable=%s expiredPct=%.2f\n",
formatVersion(m.beginVersion).c_str(),
formatVersion(m.endVersion).c_str(),
m.totalSize,
m.restorable.orDefault(false) ? "true" : "false",
m.expiredPct(expiredEndVersion)));
}
info.append(format("SnapshotBytes: %lld\n", snapshotBytes));
if(expiredEndVersion.present())
if (expiredEndVersion.present())
info.append(format("ExpiredEndVersion: %s\n", formatVersion(expiredEndVersion.get()).c_str()));
if(unreliableEndVersion.present())
if (unreliableEndVersion.present())
info.append(format("UnreliableEndVersion: %s\n", formatVersion(unreliableEndVersion.get()).c_str()));
if(minLogBegin.present())
if (minLogBegin.present())
info.append(format("MinLogBeginVersion: %s\n", formatVersion(minLogBegin.get()).c_str()));
if(contiguousLogEnd.present())
if (contiguousLogEnd.present())
info.append(format("ContiguousLogEndVersion: %s\n", formatVersion(contiguousLogEnd.get()).c_str()));
if(maxLogEnd.present())
if (maxLogEnd.present())
info.append(format("MaxLogEndVersion: %s\n", formatVersion(maxLogEnd.get()).c_str()));
if(minRestorableVersion.present())
if (minRestorableVersion.present())
info.append(format("MinRestorableVersion: %s\n", formatVersion(minRestorableVersion.get()).c_str()));
if(maxRestorableVersion.present())
if (maxRestorableVersion.present())
info.append(format("MaxRestorableVersion: %s\n", formatVersion(maxRestorableVersion.get()).c_str()));
if(!extendedDetail.empty())
if (!extendedDetail.empty())
info.append("ExtendedDetail: ").append(extendedDetail);
return info;
@ -189,14 +193,13 @@ std::string BackupDescription::toJSON() const {
auto formatVersion = [&](Version v) {
JsonBuilderObject doc;
doc.setKey("Version", v);
if(!versionTimeMap.empty()) {
if (!versionTimeMap.empty()) {
auto i = versionTimeMap.find(v);
if(i != versionTimeMap.end()) {
if (i != versionTimeMap.end()) {
doc.setKey("Timestamp", BackupAgentBase::formatTime(i->second));
doc.setKey("EpochSeconds", i->second);
}
}
else if(maxLogEnd.present()) {
} else if (maxLogEnd.present()) {
double days = double(v - maxLogEnd.get()) / (CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 60 * 60);
doc.setKey("RelativeDays", days);
}
@ -204,7 +207,7 @@ std::string BackupDescription::toJSON() const {
};
JsonBuilderArray snapshotsArray;
for(const KeyspaceSnapshotFile &m : snapshots) {
for (const KeyspaceSnapshotFile& m : snapshots) {
JsonBuilderObject snapshotDoc;
snapshotDoc.setKey("Start", formatVersion(m.beginVersion));
snapshotDoc.setKey("End", formatVersion(m.endVersion));
@ -217,22 +220,22 @@ std::string BackupDescription::toJSON() const {
doc.setKey("TotalSnapshotBytes", snapshotBytes);
if(expiredEndVersion.present())
if (expiredEndVersion.present())
doc.setKey("ExpiredEnd", formatVersion(expiredEndVersion.get()));
if(unreliableEndVersion.present())
if (unreliableEndVersion.present())
doc.setKey("UnreliableEnd", formatVersion(unreliableEndVersion.get()));
if(minLogBegin.present())
if (minLogBegin.present())
doc.setKey("MinLogBegin", formatVersion(minLogBegin.get()));
if(contiguousLogEnd.present())
if (contiguousLogEnd.present())
doc.setKey("ContiguousLogEnd", formatVersion(contiguousLogEnd.get()));
if(maxLogEnd.present())
if (maxLogEnd.present())
doc.setKey("MaxLogEnd", formatVersion(maxLogEnd.get()));
if(minRestorableVersion.present())
if (minRestorableVersion.present())
doc.setKey("MinRestorablePoint", formatVersion(minRestorableVersion.get()));
if(maxRestorableVersion.present())
if (maxRestorableVersion.present())
doc.setKey("MaxRestorablePoint", formatVersion(maxRestorableVersion.get()));
if(!extendedDetail.empty())
if (!extendedDetail.empty())
doc.setKey("ExtendedDetail", extendedDetail);
return doc.getJson();
@ -255,7 +258,8 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
static std::map<std::string, Reference<IBackupContainer>> m_cache;
Reference<IBackupContainer>& r = m_cache[url];
if (r) return r;
if (r)
return r;
try {
StringRef u(url);
@ -269,9 +273,11 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
Reference<S3BlobStoreEndpoint> bstore =
S3BlobStoreEndpoint::fromString(url, &resource, &lastOpenError, &backupParams);
if (resource.empty()) throw backup_invalid_url();
if (resource.empty())
throw backup_invalid_url();
for (auto c : resource)
if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/') throw backup_invalid_url();
if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/')
throw backup_invalid_url();
r = Reference<IBackupContainer>(new BackupContainerS3BlobStore(bstore, resource, backupParams));
}
#ifdef BUILD_AZURE_BACKUP
@ -291,13 +297,15 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
r->URL = url;
return r;
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw;
if (e.code() == error_code_actor_cancelled)
throw;
TraceEvent m(SevWarn, "BackupContainer");
m.detail("Description", "Invalid container specification. See help.");
m.detail("URL", url);
m.error(e);
if (e.code() == error_code_backup_invalid_url) m.detail("LastOpenError", lastOpenError);
if (e.code() == error_code_backup_invalid_url)
m.detail("LastOpenError", lastOpenError);
throw;
}
@ -344,14 +352,16 @@ ACTOR Future<std::vector<std::string>> listContainers_impl(std::string baseURL)
}
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw;
if (e.code() == error_code_actor_cancelled)
throw;
TraceEvent m(SevWarn, "BackupContainer");
m.detail("Description", "Invalid backup container URL prefix. See help.");
m.detail("URL", baseURL);
m.error(e);
if (e.code() == error_code_backup_invalid_url) m.detail("LastOpenError", IBackupContainer::lastOpenError);
if (e.code() == error_code_backup_invalid_url)
m.detail("LastOpenError", IBackupContainer::lastOpenError);
throw;
}
@ -367,8 +377,8 @@ ACTOR Future<Version> timeKeeperVersionFromDatetime(std::string datetime, Databa
state int64_t time = BackupAgentBase::parseTime(datetime);
if (time < 0) {
fprintf(stderr, "ERROR: Incorrect date/time or format. Format is %s.\n",
BackupAgentBase::timeFormat().c_str());
fprintf(
stderr, "ERROR: Incorrect date/time or format. Format is %s.\n", BackupAgentBase::timeFormat().c_str());
throw backup_error();
}

81
fdbclient/BackupContainer.h
View File

@ -31,8 +31,8 @@
class ReadYourWritesTransaction;
Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version const &v, Reference<ReadYourWritesTransaction> const &tr);
Future<Version> timeKeeperVersionFromDatetime(std::string const &datetime, Database const &db);
Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version const& v, Reference<ReadYourWritesTransaction> const& tr);
Future<Version> timeKeeperVersionFromDatetime(std::string const& datetime, Database const& db);
// Append-only file interface for writing backup data
// Once finish() is called the file cannot be further written to.
@ -43,16 +43,15 @@ public:
IBackupFile(const std::string& fileName) : m_fileName(fileName) {}
virtual ~IBackupFile() {}
// Backup files are append-only and cannot have more than 1 append outstanding at once.
virtual Future<Void> append(const void *data, int len) = 0;
virtual Future<Void> append(const void* data, int len) = 0;
virtual Future<Void> finish() = 0;
inline std::string getFileName() const {
return m_fileName;
}
inline std::string getFileName() const { return m_fileName; }
virtual int64_t size() const = 0;
virtual void addref() = 0;
virtual void delref() = 0;
Future<Void> appendStringRefWithLen(Standalone<StringRef> s);
protected:
std::string m_fileName;
};
@ -78,7 +77,7 @@ struct LogFile {
int totalTags = -1; // Total number of log router tags.
// Order by beginVersion, break ties with endVersion
bool operator< (const LogFile &rhs) const {
bool operator<(const LogFile& rhs) const {
return beginVersion == rhs.beginVersion ? endVersion < rhs.endVersion : beginVersion < rhs.beginVersion;
}
@ -88,9 +87,7 @@ struct LogFile {
return beginVersion >= rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId;
}
bool isPartitionedLog() const {
return tagId >= 0 && tagId < totalTags;
}
bool isPartitionedLog() const { return tagId >= 0 && tagId < totalTags; }
std::string toString() const {
std::stringstream ss;
@ -109,14 +106,14 @@ struct RangeFile {
int64_t fileSize;
// Order by version, break ties with name
bool operator< (const RangeFile &rhs) const {
bool operator<(const RangeFile& rhs) const {
return version == rhs.version ? fileName < rhs.fileName : version < rhs.version;
}
std::string toString() const {
std::stringstream ss;
ss << "version:" << std::to_string(version) << " blockSize:" << std::to_string(blockSize) <<
" fileName:" << fileName << " fileSize:" << std::to_string(fileSize);
ss << "version:" << std::to_string(version) << " blockSize:" << std::to_string(blockSize)
<< " fileName:" << fileName << " fileSize:" << std::to_string(fileSize);
return ss.str();
}
};
@ -126,25 +123,23 @@ struct KeyspaceSnapshotFile {
Version endVersion;
std::string fileName;
int64_t totalSize;
Optional<bool> restorable; // Whether or not the snapshot can be used in a restore, if known
bool isSingleVersion() const {
return beginVersion == endVersion;
}
Optional<bool> restorable; // Whether or not the snapshot can be used in a restore, if known
bool isSingleVersion() const { return beginVersion == endVersion; }
double expiredPct(Optional<Version> expiredEnd) const {
double pctExpired = 0;
if(expiredEnd.present() && expiredEnd.get() > beginVersion) {
if(isSingleVersion()) {
if (expiredEnd.present() && expiredEnd.get() > beginVersion) {
if (isSingleVersion()) {
pctExpired = 1;
}
else {
pctExpired = double(std::min(endVersion, expiredEnd.get()) - beginVersion) / (endVersion - beginVersion);
} else {
pctExpired =
double(std::min(endVersion, expiredEnd.get()) - beginVersion) / (endVersion - beginVersion);
}
}
return pctExpired * 100;
}
// Order by beginVersion, break ties with endVersion
bool operator< (const KeyspaceSnapshotFile &rhs) const {
bool operator<(const KeyspaceSnapshotFile& rhs) const {
return beginVersion == rhs.beginVersion ? endVersion < rhs.endVersion : beginVersion < rhs.beginVersion;
}
};
@ -154,7 +149,7 @@ struct BackupFileList {
std::vector<LogFile> logs;
std::vector<KeyspaceSnapshotFile> snapshots;
void toStream(FILE *fout) const;
void toStream(FILE* fout) const;
};
// The byte counts here only include usable log files and byte counts from kvrange manifests
@ -177,7 +172,7 @@ struct BackupDescription {
Optional<Version> maxRestorableVersion;
// The minimum version which this backup can be used to restore to
Optional<Version> minRestorableVersion;
std::string extendedDetail; // Freeform container-specific info.
std::string extendedDetail; // Freeform container-specific info.
bool partitioned; // If this backup contains partitioned mutation logs.
// Resolves the versions above to timestamps using a given database's TimeKeeper data.
@ -231,11 +226,17 @@ public:
// Open a log file or range file for writing
virtual Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) = 0;
virtual Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Version fileVersion, int blockSize) = 0;
virtual Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion,
int snapshotFileCount,
Version fileVersion,
int blockSize) = 0;
// Open a tagged log file for writing, where tagId is the log router tag's id.
virtual Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize,
uint16_t tagId, int totalTags) = 0;
virtual Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion,
Version endVersion,
int blockSize,
uint16_t tagId,
int totalTags) = 0;
// Write a KeyspaceSnapshotFile of range file names representing a full non overlapping
// snapshot of the key ranges this backup is targeting.
@ -260,22 +261,27 @@ public:
// If force is false, then nothing will be deleted unless there is a restorable snapshot which
// - begins at or after expireEndVersion
// - ends at or before restorableBeginVersion
// If force is true, data is deleted unconditionally which could leave the backup in an unusable state. This is not recommended.
// Returns true if expiration was done.
virtual Future<Void> expireData(Version expireEndVersion, bool force = false, ExpireProgress *progress = nullptr, Version restorableBeginVersion = std::numeric_limits<Version>::max()) = 0;
// If force is true, data is deleted unconditionally which could leave the backup in an unusable state. This is not
// recommended. Returns true if expiration was done.
virtual Future<Void> expireData(Version expireEndVersion,
bool force = false,
ExpireProgress* progress = nullptr,
Version restorableBeginVersion = std::numeric_limits<Version>::max()) = 0;
// Delete entire container. During the process, if pNumDeleted is not null it will be
// updated with the count of deleted files so that progress can be seen.
virtual Future<Void> deleteContainer(int *pNumDeleted = nullptr) = 0;
virtual Future<Void> deleteContainer(int* pNumDeleted = nullptr) = 0;
// Return key details about a backup's contents.
// Unless deepScan is true, use cached metadata, if present, as initial contiguous available log range.
// If logStartVersionOverride is given, log data prior to that version will be ignored for the purposes
// of this describe operation. This can be used to calculate what the restorability of a backup would
// be after deleting all data prior to logStartVersionOverride.
virtual Future<BackupDescription> describeBackup(bool deepScan = false, Version logStartVersionOverride = invalidVersion) = 0;
virtual Future<BackupDescription> describeBackup(bool deepScan = false,
Version logStartVersionOverride = invalidVersion) = 0;
virtual Future<BackupFileList> dumpFileList(Version begin = 0, Version end = std::numeric_limits<Version>::max()) = 0;
virtual Future<BackupFileList> dumpFileList(Version begin = 0,
Version end = std::numeric_limits<Version>::max()) = 0;
// Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion.
// If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot.
@ -283,16 +289,15 @@ public:
// Returns non-present if restoring to the given version is not possible.
virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion,
VectorRef<KeyRangeRef> keyRangesFilter = {},
bool logsOnly = false, Version beginVersion = -1) = 0;
bool logsOnly = false,
Version beginVersion = -1) = 0;
// Get an IBackupContainer based on a container spec string
static Reference<IBackupContainer> openContainer(const std::string& url);
static std::vector<std::string> getURLFormats();
static Future<std::vector<std::string>> listContainers(const std::string& baseURL);
std::string getURL() const {
return URL;
}
std::string getURL() const { return URL; }
static std::string lastOpenError;

41
fdbclient/BackupContainerAzureBlobStore.actor.cpp
View File

@ -33,15 +33,21 @@ public:
AzureClient* client;
public:
ReadFile(AsyncTaskThread& asyncTaskThread, const std::string& containerName, const std::string& blobName,
ReadFile(AsyncTaskThread& asyncTaskThread,
const std::string& containerName,
const std::string& blobName,
AzureClient* client)
: asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {}
void addref() override { ReferenceCounted<ReadFile>::addref(); }
void delref() override { ReferenceCounted<ReadFile>::delref(); }
Future<int> read(void* data, int length, int64_t offset) {
return asyncTaskThread.execAsync([client = this->client, containerName = this->containerName,
blobName = this->blobName, data, length, offset] {
return asyncTaskThread.execAsync([client = this->client,
containerName = this->containerName,
blobName = this->blobName,
data,
length,
offset] {
std::ostringstream oss(std::ios::out | std::ios::binary);
client->download_blob_to_stream(containerName, blobName, offset, length, oss);
auto str = std::move(oss).str();
@ -54,7 +60,8 @@ public:
Future<Void> truncate(int64_t size) override { throw file_not_writable(); }
Future<Void> sync() override { throw file_not_writable(); }
Future<int64_t> size() const override {
return asyncTaskThread.execAsync([client = this->client, containerName = this->containerName,
return asyncTaskThread.execAsync([client = this->client,
containerName = this->containerName,
blobName = this->blobName] {
return static_cast<int64_t>(client->get_blob_properties(containerName, blobName).get().response().size);
});
@ -77,7 +84,9 @@ public:
static constexpr size_t bufferLimit = 1 << 20;
public:
WriteFile(AsyncTaskThread& asyncTaskThread, const std::string& containerName, const std::string& blobName,
WriteFile(AsyncTaskThread& asyncTaskThread,
const std::string& containerName,
const std::string& blobName,
AzureClient* client)
: asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {}
@ -106,8 +115,10 @@ public:
Future<Void> sync() override {
auto movedBuffer = std::move(buffer);
buffer.clear();
return asyncTaskThread.execAsync([client = this->client, containerName = this->containerName,
blobName = this->blobName, buffer = std::move(movedBuffer)] {
return asyncTaskThread.execAsync([client = this->client,
containerName = this->containerName,
blobName = this->blobName,
buffer = std::move(movedBuffer)] {
std::istringstream iss(std::move(buffer));
auto resp = client->append_block_from_stream(containerName, blobName, iss).get();
return Void();
@ -167,11 +178,14 @@ public:
return Void();
}));
return Reference<IBackupFile>(
new BackupFile(fileName, Reference<IAsyncFile>(new WriteFile(self->asyncTaskThread, self->containerName,
fileName, self->client.get()))));
new BackupFile(fileName,
Reference<IAsyncFile>(new WriteFile(
self->asyncTaskThread, self->containerName, fileName, self->client.get()))));
}
static void listFiles(AzureClient* client, const std::string& containerName, const std::string& path,
static void listFiles(AzureClient* client,
const std::string& containerName,
const std::string& path,
std::function<bool(std::string const&)> folderPathFilter,
BackupContainerFileSystem::FilesAndSizesT& result) {
auto resp = client->list_blobs_segmented(containerName, "/", "", path).get().response();
@ -251,8 +265,11 @@ Future<Reference<IBackupFile>> BackupContainerAzureBlobStore::writeFile(const st
}
Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerAzureBlobStore::listFiles(
const std::string& path, std::function<bool(std::string const&)> folderPathFilter) {
return asyncTaskThread.execAsync([client = this->client.get(), containerName = this->containerName, path = path,
const std::string& path,
std::function<bool(std::string const&)> folderPathFilter) {
return asyncTaskThread.execAsync([client = this->client.get(),
containerName = this->containerName,
path = path,
folderPathFilter = folderPathFilter] {
FilesAndSizesT result;
BackupContainerAzureBlobStoreImpl::listFiles(client, containerName, path, folderPathFilter, result);

3
fdbclient/BackupContainerAzureBlobStore.h
View File

@ -42,7 +42,8 @@ class BackupContainerAzureBlobStore final : public BackupContainerFileSystem,
friend class BackupContainerAzureBlobStoreImpl;
public:
BackupContainerAzureBlobStore(const NetworkAddress& address, const std::string& accountName,
BackupContainerAzureBlobStore(const NetworkAddress& address,
const std::string& accountName,
const std::string& containerName);
void addref() override;

217
fdbclient/BackupContainerFileSystem.actor.cpp
View File

@ -35,13 +35,15 @@ public:
// TODO: Do this more efficiently, as the range file list for a snapshot could potentially be hundreds of
// megabytes.
ACTOR static Future<std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>>> readKeyspaceSnapshot(
Reference<BackupContainerFileSystem> bc, KeyspaceSnapshotFile snapshot) {
Reference<BackupContainerFileSystem> bc,
KeyspaceSnapshotFile snapshot) {
// Read the range file list for the specified version range, and then index them by fileName.
// This is so we can verify that each of the files listed in the manifest file are also in the container at this
// time.
std::vector<RangeFile> files = wait(bc->listRangeFiles(snapshot.beginVersion, snapshot.endVersion));
state std::map<std::string, RangeFile> rangeIndex;
for (auto& f : files) rangeIndex[f.fileName] = std::move(f);
for (auto& f : files)
rangeIndex[f.fileName] = std::move(f);
// Read the snapshot file, verify the version range, then find each of the range files by name in the index and
// return them.
@ -54,17 +56,21 @@ public:
JSONDoc doc(json);
Version v;
if (!doc.tryGet("beginVersion", v) || v != snapshot.beginVersion) throw restore_corrupted_data();
if (!doc.tryGet("endVersion", v) || v != snapshot.endVersion) throw restore_corrupted_data();
if (!doc.tryGet("beginVersion", v) || v != snapshot.beginVersion)
throw restore_corrupted_data();
if (!doc.tryGet("endVersion", v) || v != snapshot.endVersion)
throw restore_corrupted_data();
json_spirit::mValue& filesArray = doc.create("files");
if (filesArray.type() != json_spirit::array_type) throw restore_corrupted_data();
if (filesArray.type() != json_spirit::array_type)
throw restore_corrupted_data();
std::vector<RangeFile> results;
int missing = 0;
for (auto const& fileValue : filesArray.get_array()) {
if (fileValue.type() != json_spirit::str_type) throw restore_corrupted_data();
if (fileValue.type() != json_spirit::str_type)
throw restore_corrupted_data();
// If the file is not in the index then log the error but don't throw yet, keep checking the whole list.
auto i = rangeIndex.find(fileValue.get_str());
@ -162,8 +168,10 @@ public:
for (const auto& f : fileNames) {
if (pathToRangeFile(rf, f, 0)) {
fileArray.push_back(f);
if (rf.version < minVer) minVer = rf.version;
if (rf.version > maxVer) maxVer = rf.version;
if (rf.version < minVer)
minVer = rf.version;
if (rf.version > maxVer)
maxVer = rf.version;
} else
throw restore_unknown_file_type();
wait(yield());
@ -195,7 +203,8 @@ public:
return Void();
}
ACTOR static Future<BackupFileList> dumpFileList(Reference<BackupContainerFileSystem> bc, Version begin,
ACTOR static Future<BackupFileList> dumpFileList(Reference<BackupContainerFileSystem> bc,
Version begin,
Version end) {
state Future<std::vector<RangeFile>> fRanges = bc->listRangeFiles(begin, end);
state Future<std::vector<KeyspaceSnapshotFile>> fSnapshots = bc->listKeyspaceSnapshots(begin, end);
@ -229,8 +238,11 @@ public:
// nullptr, then it will be populated with [begin, end] -> tags, where next
// pair's begin <= previous pair's end + 1. On return, the last pair's end
// version (inclusive) gives the continuous range from begin.
static bool isContinuous(const std::vector<LogFile>& files, const std::vector<int>& indices, Version begin,
Version end, std::map<std::pair<Version, Version>, int>* tags) {
static bool isContinuous(const std::vector<LogFile>& files,
const std::vector<int>& indices,
Version begin,
Version end,
std::map<std::pair<Version, Version>, int>* tags) {
Version lastBegin = invalidVersion;
Version lastEnd = invalidVersion;
int lastTags = -1;
@ -239,7 +251,8 @@ public:
for (int idx : indices) {
const LogFile& file = files[idx];
if (lastEnd == invalidVersion) {
if (file.beginVersion > begin) return false;
if (file.beginVersion > begin)
return false;
if (file.endVersion > begin) {
lastBegin = begin;
lastTags = file.totalTags;
@ -261,7 +274,8 @@ public:
lastTags = file.totalTags;
}
lastEnd = file.endVersion;
if (lastEnd > end) break;
if (lastEnd > end)
break;
}
if (tags != nullptr && lastBegin != invalidVersion) {
tags->emplace(std::make_pair(lastBegin, std::min(end, lastEnd - 1)), lastTags);
@ -297,7 +311,8 @@ public:
// check partition 0 is continuous in [begin, end] and create a map of ranges to partitions
std::map<std::pair<Version, Version>, int> tags; // range [start, end] -> partitions
isContinuous(logs, tagIndices[0], begin, end, &tags);
if (tags.empty() || end <= begin) return 0;
if (tags.empty() || end <= begin)
return 0;
end = std::min(end, tags.rbegin()->first.second);
TraceEvent("ContinuousLogEnd").detail("Partition", 0).detail("EndVersion", end).detail("Begin", begin);
@ -314,7 +329,8 @@ public:
.detail("EndVersion", tagEnd)
.detail("RangeBegin", beginEnd.first)
.detail("RangeEnd", beginEnd.second);
if (tagEnd == 0) return lastEnd == begin ? 0 : lastEnd;
if (tagEnd == 0)
return lastEnd == begin ? 0 : lastEnd;
}
if (tagEnd < beginEnd.second) {
return tagEnd;
@ -327,9 +343,12 @@ public:
// Analyze partitioned logs and set contiguousLogEnd for "desc" if larger
// than the "scanBegin" version.
static void updatePartitionedLogsContinuousEnd(BackupDescription* desc, const std::vector<LogFile>& logs,
const Version scanBegin, const Version scanEnd) {
if (logs.empty()) return;
static void updatePartitionedLogsContinuousEnd(BackupDescription* desc,
const std::vector<LogFile>& logs,
const Version scanBegin,
const Version scanEnd) {
if (logs.empty())
return;
Version snapshotBeginVersion = desc->snapshots.size() > 0 ? desc->snapshots[0].beginVersion : invalidVersion;
Version begin = std::max(scanBegin, desc->minLogBegin.get());
@ -340,7 +359,8 @@ public:
.detail("ContiguousLogEnd", desc->contiguousLogEnd.get());
for (const auto& file : logs) {
if (file.beginVersion > begin) {
if (scanBegin > 0) return;
if (scanBegin > 0)
return;
// scanBegin is 0
desc->minLogBegin = file.beginVersion;
@ -352,7 +372,8 @@ public:
// contiguousLogEnd is not inclusive, so +1 here.
desc->contiguousLogEnd.get() = ver + 1;
TraceEvent("UpdateContinuousLogEnd").detail("Version", ver + 1);
if (ver > snapshotBeginVersion) return;
if (ver > snapshotBeginVersion)
return;
}
}
}
@ -360,24 +381,30 @@ public:
// Computes the continuous end version for non-partitioned mutation logs up to
// the "targetVersion". If "outLogs" is not nullptr, it will be updated with
// continuous log files. "*end" is updated with the continuous end version.
static void computeRestoreEndVersion(const std::vector<LogFile>& logs, std::vector<LogFile>* outLogs, Version* end,
static void computeRestoreEndVersion(const std::vector<LogFile>& logs,
std::vector<LogFile>* outLogs,
Version* end,
Version targetVersion) {
auto i = logs.begin();
if (outLogs != nullptr) outLogs->push_back(*i);
if (outLogs != nullptr)
outLogs->push_back(*i);
// Add logs to restorable logs set until continuity is broken OR we reach targetVersion
while (++i != logs.end()) {
if (i->beginVersion > *end || i->beginVersion > targetVersion) break;
if (i->beginVersion > *end || i->beginVersion > targetVersion)
break;
// If the next link in the log chain is found, update the end
if (i->beginVersion == *end) {
if (outLogs != nullptr) outLogs->push_back(*i);
if (outLogs != nullptr)
outLogs->push_back(*i);
*end = i->endVersion;
}
}
}
ACTOR static Future<BackupDescription> describeBackup(Reference<BackupContainerFileSystem> bc, bool deepScan,
ACTOR static Future<BackupDescription> describeBackup(Reference<BackupContainerFileSystem> bc,
bool deepScan,
Version logStartVersionOverride) {
state BackupDescription desc;
desc.url = bc->getURL();
@ -397,8 +424,8 @@ public:
// This could be handled more efficiently without recursion but it's tricky, this will do for now.
if (logStartVersionOverride != invalidVersion && logStartVersionOverride < 0) {
BackupDescription tmp = wait(bc->describeBackup(false, invalidVersion));
logStartVersionOverride = resolveRelativeVersion(tmp.maxLogEnd, logStartVersionOverride,
"LogStartVersionOverride", invalid_option_value());
logStartVersionOverride = resolveRelativeVersion(
tmp.maxLogEnd, logStartVersionOverride, "LogStartVersionOverride", invalid_option_value());
}
// Get metadata versions
@ -562,7 +589,8 @@ public:
wait(updates);
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw;
if (e.code() == error_code_actor_cancelled)
throw;
TraceEvent(SevWarn, "BackupContainerMetadataUpdateFailure").error(e).detail("URL", bc->getURL());
}
}
@ -572,7 +600,8 @@ public:
s.restorable = true;
// If this is not a single-version snapshot then see if the available contiguous logs cover its range
if (s.beginVersion != s.endVersion) {
if (!desc.minLogBegin.present() || desc.minLogBegin.get() > s.beginVersion) s.restorable = false;
if (!desc.minLogBegin.present() || desc.minLogBegin.get() > s.beginVersion)
s.restorable = false;
if (!desc.contiguousLogEnd.present() || desc.contiguousLogEnd.get() <= s.endVersion)
s.restorable = false;
}
@ -604,8 +633,11 @@ public:
return desc;
}
ACTOR static Future<Void> expireData(Reference<BackupContainerFileSystem> bc, Version expireEndVersion, bool force,
IBackupContainer::ExpireProgress* progress, Version restorableBeginVersion) {
ACTOR static Future<Void> expireData(Reference<BackupContainerFileSystem> bc,
Version expireEndVersion,
bool force,
IBackupContainer::ExpireProgress* progress,
Version restorableBeginVersion) {
if (progress != nullptr) {
progress->step = "Describing backup";
progress->total = 0;
@ -622,11 +654,12 @@ public:
// Resolve relative versions using max log version
expireEndVersion =
resolveRelativeVersion(desc.maxLogEnd, expireEndVersion, "ExpireEndVersion", invalid_option_value());
restorableBeginVersion = resolveRelativeVersion(desc.maxLogEnd, restorableBeginVersion,
"RestorableBeginVersion", invalid_option_value());
restorableBeginVersion = resolveRelativeVersion(
desc.maxLogEnd, restorableBeginVersion, "RestorableBeginVersion", invalid_option_value());
// It would be impossible to have restorability to any version < expireEndVersion after expiring to that version
if (restorableBeginVersion < expireEndVersion) throw backup_cannot_expire();
if (restorableBeginVersion < expireEndVersion)
throw backup_cannot_expire();
// If the expire request is to a version at or before the previous version to which data was already deleted
// then do nothing and just return
@ -651,7 +684,8 @@ public:
// Note that it is possible for there to be no actual files in the backup prior to expireEndVersion,
// if they were externally deleted or an expire operation deleted them but was terminated before
// updating expireEndVersion
if (forceNeeded && !force) throw backup_cannot_expire();
if (forceNeeded && !force)
throw backup_cannot_expire();
// Start scan for files to delete at the last completed expire operation's end or 0.
state Version scanBegin = desc.expiredEndVersion.orDefault(0);
@ -723,7 +757,8 @@ public:
ranges.clear();
for (auto const& f : desc.snapshots) {
if (f.endVersion < expireEndVersion) toDelete.push_back(std::move(f.fileName));
if (f.endVersion < expireEndVersion)
toDelete.push_back(std::move(f.fileName));
}
desc = BackupDescription();
@ -838,11 +873,13 @@ public:
}
i = j;
}
if (i < logs.size()) filtered.push_back(logs[i]);
if (i < logs.size())
filtered.push_back(logs[i]);
return filtered;
}
static Optional<RestorableFileSet> getRestoreSetFromLogs(const std::vector<LogFile>& logs, Version targetVersion,
static Optional<RestorableFileSet> getRestoreSetFromLogs(const std::vector<LogFile>& logs,
Version targetVersion,
RestorableFileSet restorable) {
Version end = logs.begin()->endVersion;
computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion);
@ -923,7 +960,8 @@ public:
// 'latestVersion' represents using the minimum restorable version in a snapshot.
restorable.targetVersion = targetVersion == latestVersion ? maxKeyRangeVersion : targetVersion;
// Any version < maxKeyRangeVersion is not restorable.
if (restorable.targetVersion < maxKeyRangeVersion) continue;
if (restorable.targetVersion < maxKeyRangeVersion)
continue;
restorable.snapshot = snapshots[i];
// TODO: Reenable the sanity check after TooManyFiles error is resolved
@ -931,7 +969,8 @@ public:
// Sanity check key ranges
state std::map<std::string, KeyRange>::iterator rit;
for (rit = restorable.keyRanges.begin(); rit != restorable.keyRanges.end(); rit++) {
auto it = std::find_if(restorable.ranges.begin(), restorable.ranges.end(),
auto it = std::find_if(restorable.ranges.begin(),
restorable.ranges.end(),
[file = rit->first](const RangeFile f) { return f.fileName == file; });
ASSERT(it != restorable.ranges.end());
KeyRange result = wait(bc->getSnapshotFileKeyRange(*it));
@ -1040,13 +1079,23 @@ public:
f.fileName = path;
f.fileSize = size;
int len;
if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u%n", &f.beginVersion, &f.endVersion,
&f.blockSize, &len) == 3 &&
if (sscanf(name.c_str(),
"log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u%n",
&f.beginVersion,
&f.endVersion,
&f.blockSize,
&len) == 3 &&
len == name.size()) {
out = f;
return true;
} else if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%d-of-%d,%u%n", &f.beginVersion,
&f.endVersion, &f.tagId, &f.totalTags, &f.blockSize, &len) == 5 &&
} else if (sscanf(name.c_str(),
"log,%" SCNd64 ",%" SCNd64 ",%*[^,],%d-of-%d,%u%n",
&f.beginVersion,
&f.endVersion,
&f.tagId,
&f.totalTags,
&f.blockSize,
&len) == 5 &&
len == name.size() && f.tagId >= 0) {
out = f;
return true;
@ -1059,8 +1108,12 @@ public:
KeyspaceSnapshotFile f;
f.fileName = path;
int len;
if (sscanf(name.c_str(), "snapshot,%" SCNd64 ",%" SCNd64 ",%" SCNd64 "%n", &f.beginVersion, &f.endVersion,
&f.totalSize, &len) == 3 &&
if (sscanf(name.c_str(),
"snapshot,%" SCNd64 ",%" SCNd64 ",%" SCNd64 "%n",
&f.beginVersion,
&f.endVersion,
&f.totalSize,
&len) == 3 &&
len == name.size()) {
out = f;
return true;
@ -1070,26 +1123,38 @@ public:
}; // class BackupContainerFileSystemImpl
Future<Reference<IBackupFile>> BackupContainerFileSystem::writeLogFile(Version beginVersion, Version endVersion,
Future<Reference<IBackupFile>> BackupContainerFileSystem::writeLogFile(Version beginVersion,
Version endVersion,
int blockSize) {
return writeFile(BackupContainerFileSystemImpl::logVersionFolderString(beginVersion, false) +
format("log,%lld,%lld,%s,%d", beginVersion, endVersion,
deterministicRandom()->randomUniqueID().toString().c_str(), blockSize));
format("log,%lld,%lld,%s,%d",
beginVersion,
endVersion,
deterministicRandom()->randomUniqueID().toString().c_str(),
blockSize));
}
Future<Reference<IBackupFile>> BackupContainerFileSystem::writeTaggedLogFile(Version beginVersion, Version endVersion,
int blockSize, uint16_t tagId,
Future<Reference<IBackupFile>> BackupContainerFileSystem::writeTaggedLogFile(Version beginVersion,
Version endVersion,
int blockSize,
uint16_t tagId,
int totalTags) {
return writeFile(BackupContainerFileSystemImpl::logVersionFolderString(beginVersion, true) +
format("log,%lld,%lld,%s,%d-of-%d,%d", beginVersion, endVersion,
deterministicRandom()->randomUniqueID().toString().c_str(), tagId, totalTags, blockSize));
format("log,%lld,%lld,%s,%d-of-%d,%d",
beginVersion,
endVersion,
deterministicRandom()->randomUniqueID().toString().c_str(),
tagId,
totalTags,
blockSize));
}
Future<Reference<IBackupFile>> BackupContainerFileSystem::writeRangeFile(Version snapshotBeginVersion,
int snapshotFileCount, Version fileVersion,
int snapshotFileCount,
Version fileVersion,
int blockSize) {
std::string fileName = format("range,%" PRId64 ",%s,%d", fileVersion,
deterministicRandom()->randomUniqueID().toString().c_str(), blockSize);
std::string fileName = format(
"range,%" PRId64 ",%s,%d", fileVersion, deterministicRandom()->randomUniqueID().toString().c_str(), blockSize);
// In order to test backward compatibility in simulation, sometimes write to the old path format
if (g_network->isSimulated() && deterministicRandom()->coinflip()) {
@ -1109,11 +1174,12 @@ BackupContainerFileSystem::readKeyspaceSnapshot(KeyspaceSnapshotFile snapshot) {
Future<Void> BackupContainerFileSystem::writeKeyspaceSnapshotFile(const std::vector<std::string>& fileNames,
const std::vector<std::pair<Key, Key>>& beginEndKeys,
int64_t totalBytes) {
return BackupContainerFileSystemImpl::writeKeyspaceSnapshotFile(Reference<BackupContainerFileSystem>::addRef(this),
fileNames, beginEndKeys, totalBytes);
return BackupContainerFileSystemImpl::writeKeyspaceSnapshotFile(
Reference<BackupContainerFileSystem>::addRef(this), fileNames, beginEndKeys, totalBytes);
};
Future<std::vector<LogFile>> BackupContainerFileSystem::listLogFiles(Version beginVersion, Version targetVersion,
Future<std::vector<LogFile>> BackupContainerFileSystem::listLogFiles(Version beginVersion,
Version targetVersion,
bool partitioned) {
// The first relevant log file could have a begin version less than beginVersion based on the knobs which
// determine log file range size, so start at an earlier version adjusted by how many versions a file could
@ -1201,8 +1267,8 @@ Future<std::vector<RangeFile>> BackupContainerFileSystem::listRangeFiles(Version
return map(success(oldFiles) && success(newFiles), [=](Void _) {
std::vector<RangeFile> results = std::move(newFiles.get());
std::vector<RangeFile> oldResults = std::move(oldFiles.get());
results.insert(results.end(), std::make_move_iterator(oldResults.begin()),
std::make_move_iterator(oldResults.end()));
results.insert(
results.end(), std::make_move_iterator(oldResults.begin()), std::make_move_iterator(oldResults.end()));
return results;
});
}
@ -1226,14 +1292,16 @@ Future<BackupFileList> BackupContainerFileSystem::dumpFileList(Version begin, Ve
}
Future<BackupDescription> BackupContainerFileSystem::describeBackup(bool deepScan, Version logStartVersionOverride) {
return BackupContainerFileSystemImpl::describeBackup(Reference<BackupContainerFileSystem>::addRef(this), deepScan,
logStartVersionOverride);
return BackupContainerFileSystemImpl::describeBackup(
Reference<BackupContainerFileSystem>::addRef(this), deepScan, logStartVersionOverride);
}
Future<Void> BackupContainerFileSystem::expireData(Version expireEndVersion, bool force, ExpireProgress* progress,
Future<Void> BackupContainerFileSystem::expireData(Version expireEndVersion,
bool force,
ExpireProgress* progress,
Version restorableBeginVersion) {
return BackupContainerFileSystemImpl::expireData(Reference<BackupContainerFileSystem>::addRef(this),
expireEndVersion, force, progress, restorableBeginVersion);
return BackupContainerFileSystemImpl::expireData(
Reference<BackupContainerFileSystem>::addRef(this), expireEndVersion, force, progress, restorableBeginVersion);
}
ACTOR static Future<KeyRange> getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem> bc, RangeFile file) {
@ -1302,13 +1370,15 @@ ACTOR static Future<Optional<Version>> readVersionProperty(Reference<BackupConta
int rs = wait(f->read((uint8_t*)s.data(), size, 0));
Version v;
int len;
if (rs == size && sscanf(s.c_str(), "%" SCNd64 "%n", &v, &len) == 1 && len == size) return v;
if (rs == size && sscanf(s.c_str(), "%" SCNd64 "%n", &v, &len) == 1 && len == size)
return v;
TraceEvent(SevWarn, "BackupContainerInvalidProperty").detail("URL", bc->getURL()).detail("Path", path);
throw backup_invalid_info();
} catch (Error& e) {
if (e.code() == error_code_file_not_found) return Optional<Version>();
if (e.code() == error_code_file_not_found)
return Optional<Version>();
TraceEvent(SevWarn, "BackupContainerReadPropertyFailed")
.error(e)
@ -1324,12 +1394,12 @@ Future<KeyRange> BackupContainerFileSystem::getSnapshotFileKeyRange(const RangeF
return getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem>::addRef(this), file);
}
Future<Optional<RestorableFileSet>> BackupContainerFileSystem::getRestoreSet(Version targetVersion,
VectorRef<KeyRangeRef> keyRangesFilter,
bool logsOnly, Version beginVersion) {
return BackupContainerFileSystemImpl::getRestoreSet(Reference<BackupContainerFileSystem>::addRef(this),
targetVersion, keyRangesFilter, logsOnly, beginVersion);
bool logsOnly,
Version beginVersion) {
return BackupContainerFileSystemImpl::getRestoreSet(
Reference<BackupContainerFileSystem>::addRef(this), targetVersion, keyRangesFilter, logsOnly, beginVersion);
}
Future<Optional<Version>> BackupContainerFileSystem::VersionProperty::get() {
@ -1429,7 +1499,8 @@ ACTOR static Future<Void> testBackupContainer(std::string url) {
try {
wait(c->deleteContainer());
} catch (Error& e) {
if (e.code() != error_code_backup_invalid_url && e.code() != error_code_backup_does_not_exist) throw;
if (e.code() != error_code_backup_invalid_url && e.code() != error_code_backup_does_not_exist)
throw;
}
wait(c->create());

23
fdbclient/BackupContainerFileSystem.h
View File

@ -101,11 +101,16 @@ public:
Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) final;
Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize,
uint16_t tagId, int totalTags) final;
Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion,
Version endVersion,
int blockSize,
uint16_t tagId,
int totalTags) final;
Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount,
Version fileVersion, int blockSize) override;
Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion,
int snapshotFileCount,
Version fileVersion,
int blockSize) override;
Future<std::pair<std::vector<RangeFile>, std::map<std::string, KeyRange>>> readKeyspaceSnapshot(
KeyspaceSnapshotFile snapshot);
@ -136,13 +141,17 @@ public:
Future<BackupDescription> describeBackup(bool deepScan, Version logStartVersionOverride) final;
// Delete all data up to (but not including endVersion)
Future<Void> expireData(Version expireEndVersion, bool force, ExpireProgress* progress,
Future<Void> expireData(Version expireEndVersion,
bool force,
ExpireProgress* progress,
Version restorableBeginVersion) final;
Future<KeyRange> getSnapshotFileKeyRange(const RangeFile& file) final;
Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion, VectorRef<KeyRangeRef> keyRangesFilter,
bool logsOnly, Version beginVersion) final;
Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion,
VectorRef<KeyRangeRef> keyRangesFilter,
bool logsOnly,
Version beginVersion) final;
private:
struct VersionProperty {

15
fdbclient/BackupContainerLocalDirectory.actor.cpp
View File

@ -94,7 +94,8 @@ ACTOR static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles_impl(st
// openFile() above for more info on why they are created.
if (g_network->isSimulated())
files.erase(
std::remove_if(files.begin(), files.end(),
std::remove_if(files.begin(),
files.end(),
[](std::string const& f) { return StringRef(f).endsWith(LiteralStringRef(".lnk")); }),
files.end());
@ -174,7 +175,8 @@ Future<std::vector<std::string>> BackupContainerLocalDirectory::listURLs(const s
std::vector<std::string> results;
for (const auto& r : dirs) {
if (r == "." || r == "..") continue;
if (r == "." || r == "..")
continue;
results.push_back(std::string("file://") + joinPath(path, r));
}
@ -262,7 +264,8 @@ Future<Void> BackupContainerLocalDirectory::deleteFile(const std::string& path)
}
Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerLocalDirectory::listFiles(
const std::string& path, std::function<bool(std::string const&)>) {
const std::string& path,
std::function<bool(std::string const&)>) {
return listFiles_impl(path, m_path);
}
@ -271,10 +274,12 @@ Future<Void> BackupContainerLocalDirectory::deleteContainer(int* pNumDeleted) {
// and make sure it has something in it.
return map(describeBackup(false, invalidVersion), [=](BackupDescription const& desc) {
// If the backup has no snapshots and no logs then it's probably not a valid backup
if (desc.snapshots.size() == 0 && !desc.minLogBegin.present()) throw backup_invalid_url();
if (desc.snapshots.size() == 0 && !desc.minLogBegin.present())
throw backup_invalid_url();
int count = platform::eraseDirectoryRecursive(m_path);
if (pNumDeleted != nullptr) *pNumDeleted = count;
if (pNumDeleted != nullptr)
*pNumDeleted = count;
return Void();
});

17
fdbclient/BackupContainerS3BlobStore.actor.cpp
View File

@ -73,7 +73,8 @@ public:
};
ACTOR static Future<BackupContainerFileSystem::FilesAndSizesT> listFiles(
Reference<BackupContainerS3BlobStore> bc, std::string path,
Reference<BackupContainerS3BlobStore> bc,
std::string path,
std::function<bool(std::string const&)> pathFilter) {
// pathFilter expects container based paths, so create a wrapper which converts a raw path
// to a container path by removing the known backup name prefix.
@ -134,7 +135,8 @@ std::string BackupContainerS3BlobStore::indexEntry() {
return BackupContainerS3BlobStoreImpl::INDEXFOLDER + "/" + m_name;
}
BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore, const std::string& name,
BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore,
const std::string& name,
const S3BlobStoreEndpoint::ParametersT& params)
: m_bstore(bstore), m_name(name), m_bucket("FDB_BACKUPS_V2") {
@ -164,7 +166,9 @@ std::string BackupContainerS3BlobStore::getURLFormat() {
Future<Reference<IAsyncFile>> BackupContainerS3BlobStore::readFile(const std::string& path) {
return Reference<IAsyncFile>(new AsyncFileReadAheadCache(
Reference<IAsyncFile>(new AsyncFileS3BlobStoreRead(m_bstore, m_bucket, dataPath(path))),
m_bstore->knobs.read_block_size, m_bstore->knobs.read_ahead_blocks, m_bstore->knobs.concurrent_reads_per_file,
m_bstore->knobs.read_block_size,
m_bstore->knobs.read_ahead_blocks,
m_bstore->knobs.concurrent_reads_per_file,
m_bstore->knobs.read_cache_blocks_per_file));
}
@ -183,9 +187,10 @@ Future<Void> BackupContainerS3BlobStore::deleteFile(const std::string& path) {
}
Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerS3BlobStore::listFiles(
const std::string& path, std::function<bool(std::string const&)> pathFilter) {
return BackupContainerS3BlobStoreImpl::listFiles(Reference<BackupContainerS3BlobStore>::addRef(this), path,
pathFilter);
const std::string& path,
std::function<bool(std::string const&)> pathFilter) {
return BackupContainerS3BlobStoreImpl::listFiles(
Reference<BackupContainerS3BlobStore>::addRef(this), path, pathFilter);
}
Future<Void> BackupContainerS3BlobStore::create() {

3
fdbclient/BackupContainerS3BlobStore.h
View File

@ -41,7 +41,8 @@ class BackupContainerS3BlobStore final : public BackupContainerFileSystem,
friend class BackupContainerS3BlobStoreImpl;
public:
BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore, const std::string& name,
BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore,
const std::string& name,
const S3BlobStoreEndpoint::ParametersT& params);
void addref() override;

677
fdbclient/ClientLogEvents.h
View File

@ -38,7 +38,8 @@ enum class TransactionPriorityType { PRIORITY_DEFAULT = 0, PRIORITY_BATCH = 1, P
struct Event {
Event(EventType t, double ts, const Optional<Standalone<StringRef>>& dc) : type(t), startTs(ts) {
if (dc.present()) dcId = dc.get();
if (dc.present())
dcId = dc.get();
}
Event() {}
@ -56,357 +57,385 @@ struct Event {
Key dcId{};
void logEvent(std::string id, int maxFieldLength) const {}
};
};
struct EventGetVersion : public Event {
EventGetVersion() { }
struct EventGetVersion : public Event {
EventGetVersion() {}
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency);
else
return serializer(ar, latency);
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency);
else
return serializer(ar, latency);
}
double latency;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion").detail("TransactionID", id).detail("Latency", latency);
}
};
// Version V2 of EventGetVersion starting at 6.2
struct EventGetVersion_V2 : public Event {
EventGetVersion_V2() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType);
else
return serializer(ar, latency, priorityType);
}
double latency;
TransactionPriorityType priorityType{ TransactionPriorityType::UNSET };
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion")
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("PriorityType", priorityType);
}
};
// Version V3 of EventGetVersion starting at 6.3
struct EventGetVersion_V3 : public Event {
EventGetVersion_V3(double ts,
const Optional<Standalone<StringRef>>& dcId,
double lat,
TransactionPriority priority,
Version version)
: Event(EventType::GET_VERSION_LATENCY, ts, dcId), latency(lat), readVersion(version) {
switch (priority) {
// Unfortunately, the enum serialized here disagrees with the enum used elsewhere for the values used by each
// priority
case TransactionPriority::IMMEDIATE:
priorityType = TransactionPriorityType::PRIORITY_IMMEDIATE;
break;
case TransactionPriority::DEFAULT:
priorityType = TransactionPriorityType::PRIORITY_DEFAULT;
break;
case TransactionPriority::BATCH:
priorityType = TransactionPriorityType::PRIORITY_BATCH;
break;
default:
ASSERT(false);
}
}
EventGetVersion_V3() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType, readVersion);
else
return serializer(ar, latency, priorityType, readVersion);
}
double latency;
TransactionPriorityType priorityType{ TransactionPriorityType::UNSET };
Version readVersion;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion")
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("PriorityType", priorityType)
.detail("ReadVersion", readVersion);
}
};
struct EventGet : public Event {
EventGet(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int size, const KeyRef& in_key)
: Event(EventType::GET_LATENCY, ts, dcId), latency(lat), valueSize(size), key(in_key) {}
EventGet() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, valueSize, key);
else
return serializer(ar, latency, valueSize, key);
}
double latency;
int valueSize;
Key key;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_Get")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("ValueSizeBytes", valueSize)
.setMaxFieldLength(maxFieldLength)
.detail("Key", key);
}
};
struct EventGetRange : public Event {
EventGetRange(double ts,
const Optional<Standalone<StringRef>>& dcId,
double lat,
int size,
const KeyRef& start_key,
const KeyRef& end_key)
: Event(EventType::GET_RANGE_LATENCY, ts, dcId), latency(lat), rangeSize(size), startKey(start_key),
endKey(end_key) {}
EventGetRange() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, rangeSize, startKey, endKey);
else
return serializer(ar, latency, rangeSize, startKey, endKey);
}
double latency;
int rangeSize;
Key startKey;
Key endKey;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("RangeSizeBytes", rangeSize)
.setMaxFieldLength(maxFieldLength)
.detail("StartKey", startKey)
.detail("EndKey", endKey);
}
};
struct EventCommit : public Event {
EventCommit() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, numMutations, commitBytes, req.transaction, req.arena);
else
return serializer(ar, latency, numMutations, commitBytes, req.transaction, req.arena);
}
double latency;
int numMutations;
int commitBytes;
CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
double latency;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion")
.detail("TransactionID", id)
.detail("Latency", latency);
}
};
// Version V2 of EventGetVersion starting at 6.2
struct EventGetVersion_V2 : public Event {
EventGetVersion_V2() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType);
else
return serializer(ar, latency, priorityType);
for (auto& write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_WriteConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
.detail("End", write_range.end);
}
double latency;
TransactionPriorityType priorityType{ TransactionPriorityType::UNSET };
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion")
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("PriorityType", priorityType);
}
};
// Version V3 of EventGetVersion starting at 6.3
struct EventGetVersion_V3 : public Event {
EventGetVersion_V3(double ts, const Optional<Standalone<StringRef>>& dcId, double lat,
TransactionPriority priority, Version version)
: Event(EventType::GET_VERSION_LATENCY, ts, dcId), latency(lat), readVersion(version) {
switch(priority) {
// Unfortunately, the enum serialized here disagrees with the enum used elsewhere for the values used by each priority
case TransactionPriority::IMMEDIATE:
priorityType = TransactionPriorityType::PRIORITY_IMMEDIATE;
break;
case TransactionPriority::DEFAULT:
priorityType = TransactionPriorityType::PRIORITY_DEFAULT;
break;
case TransactionPriority::BATCH:
priorityType = TransactionPriorityType::PRIORITY_BATCH;
break;
default:
ASSERT(false);
}
}
EventGetVersion_V3() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, priorityType, readVersion);
else
return serializer(ar, latency, priorityType, readVersion);
for (auto& mutation : req.transaction.mutations) {
TraceEvent("TransactionTrace_Commit_Mutation")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
double latency;
TransactionPriorityType priorityType{ TransactionPriorityType::UNSET };
Version readVersion;
TraceEvent("TransactionTrace_Commit")
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("NumMutations", numMutations)
.detail("CommitSizeBytes", commitBytes);
}
};
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetVersion")
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("PriorityType", priorityType)
.detail("ReadVersion", readVersion);
}
};
// Version V2 of EventGetVersion starting at 6.3
struct EventCommit_V2 : public Event {
EventCommit_V2(double ts,
const Optional<Standalone<StringRef>>& dcId,
double lat,
int mut,
int bytes,
Version version,
const CommitTransactionRequest& commit_req)
: Event(EventType::COMMIT_LATENCY, ts, dcId), latency(lat), numMutations(mut), commitBytes(bytes),
commitVersion(version), req(commit_req) {}
EventCommit_V2() {}
struct EventGet : public Event {
EventGet(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int size, const KeyRef& in_key)
: Event(EventType::GET_LATENCY, ts, dcId), latency(lat), valueSize(size), key(in_key) {}
EventGet() { }
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(
Event::serialize(ar), latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
else
return serializer(ar, latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
}
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, valueSize, key);
else
return serializer(ar, latency, valueSize, key);
double latency;
int numMutations;
int commitBytes;
Version commitVersion;
CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
double latency;
int valueSize;
Key key;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_Get")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("ValueSizeBytes", valueSize)
.setMaxFieldLength(maxFieldLength)
.detail("Key", key);
}
};
struct EventGetRange : public Event {
EventGetRange(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int size,
const KeyRef& start_key, const KeyRef& end_key)
: Event(EventType::GET_RANGE_LATENCY, ts, dcId), latency(lat), rangeSize(size), startKey(start_key),
endKey(end_key) {}
EventGetRange() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, rangeSize, startKey, endKey);
else
return serializer(ar, latency, rangeSize, startKey, endKey);
for (auto& write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_WriteConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
.detail("End", write_range.end);
}
double latency;
int rangeSize;
Key startKey;
Key endKey;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("RangeSizeBytes", rangeSize)
.setMaxFieldLength(maxFieldLength)
.detail("StartKey", startKey)
.detail("EndKey", endKey);
}
};
struct EventCommit : public Event {
EventCommit() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, numMutations, commitBytes, req.transaction, req.arena);
else
return serializer(ar, latency, numMutations, commitBytes, req.transaction, req.arena);
for (auto& mutation : req.transaction.mutations) {
TraceEvent("TransactionTrace_Commit_Mutation")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
double latency;
int numMutations;
int commitBytes;
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
TraceEvent("TransactionTrace_Commit")
.detail("TransactionID", id)
.detail("CommitVersion", commitVersion)
.detail("Latency", latency)
.detail("NumMutations", numMutations)
.detail("CommitSizeBytes", commitBytes);
}
};
void logEvent(std::string id, int maxFieldLength) const {
for (auto &read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
struct EventGetError : public Event {
EventGetError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code, const KeyRef& in_key)
: Event(EventType::ERROR_GET, ts, dcId), errCode(err_code), key(in_key) {}
EventGetError() {}
for (auto &write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_WriteConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
.detail("End", write_range.end);
}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, key);
else
return serializer(ar, errCode, key);
}
for (auto &mutation : req.transaction.mutations) {
TraceEvent("TransactionTrace_Commit_Mutation")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
int errCode;
Key key;
TraceEvent("TransactionTrace_Commit")
.detail("TransactionID", id)
.detail("Latency", latency)
.detail("NumMutations", numMutations)
.detail("CommitSizeBytes", commitBytes);
}
};
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetError")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("ErrCode", errCode)
.setMaxFieldLength(maxFieldLength)
.detail("Key", key);
}
};
// Version V2 of EventGetVersion starting at 6.3
struct EventCommit_V2 : public Event {
EventCommit_V2(double ts, const Optional<Standalone<StringRef>>& dcId, double lat, int mut, int bytes,
Version version, const CommitTransactionRequest& commit_req)
: Event(EventType::COMMIT_LATENCY, ts, dcId), latency(lat), numMutations(mut), commitBytes(bytes),
commitVersion(version), req(commit_req) {}
EventCommit_V2() { }
struct EventGetRangeError : public Event {
EventGetRangeError(double ts,
const Optional<Standalone<StringRef>>& dcId,
int err_code,
const KeyRef& start_key,
const KeyRef& end_key)
: Event(EventType::ERROR_GET_RANGE, ts, dcId), errCode(err_code), startKey(start_key), endKey(end_key) {}
EventGetRangeError() {}
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
else
return serializer(ar, latency, numMutations, commitBytes, commitVersion, req.transaction, req.arena);
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, startKey, endKey);
else
return serializer(ar, errCode, startKey, endKey);
}
int errCode;
Key startKey;
Key endKey;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetRangeError")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("ErrCode", errCode)
.setMaxFieldLength(maxFieldLength)
.detail("StartKey", startKey)
.detail("EndKey", endKey);
}
};
struct EventCommitError : public Event {
EventCommitError(double ts,
const Optional<Standalone<StringRef>>& dcId,
int err_code,
const CommitTransactionRequest& commit_req)
: Event(EventType::ERROR_COMMIT, ts, dcId), errCode(err_code), req(commit_req) {}
EventCommitError() {}
template <typename Ar>
Ar& serialize(Ar& ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, req.transaction, req.arena);
else
return serializer(ar, errCode, req.transaction, req.arena);
}
int errCode;
CommitTransactionRequest
req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto& read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_CommitError_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
double latency;
int numMutations;
int commitBytes;
Version commitVersion;
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto &read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
for (auto &write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_Commit_WriteConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
.detail("End", write_range.end);
}
for (auto &mutation : req.transaction.mutations) {
TraceEvent("TransactionTrace_Commit_Mutation")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
TraceEvent("TransactionTrace_Commit")
.detail("TransactionID", id)
.detail("CommitVersion", commitVersion)
.detail("Latency", latency)
.detail("NumMutations", numMutations)
.detail("CommitSizeBytes", commitBytes);
}
};
struct EventGetError : public Event {
EventGetError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code, const KeyRef& in_key)
: Event(EventType::ERROR_GET, ts, dcId), errCode(err_code), key(in_key) {}
EventGetError() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, key);
else
return serializer(ar, errCode, key);
for (auto& write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_CommitError_WriteConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
.detail("End", write_range.end);
}
int errCode;
Key key;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetError")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("ErrCode", errCode)
.setMaxFieldLength(maxFieldLength)
.detail("Key", key);
}
};
struct EventGetRangeError : public Event {
EventGetRangeError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code,
const KeyRef& start_key, const KeyRef& end_key)
: Event(EventType::ERROR_GET_RANGE, ts, dcId), errCode(err_code), startKey(start_key), endKey(end_key) {}
EventGetRangeError() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, startKey, endKey);
else
return serializer(ar, errCode, startKey, endKey);
for (auto& mutation : req.transaction.mutations) {
TraceEvent("TransactionTrace_CommitError_Mutation")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
int errCode;
Key startKey;
Key endKey;
void logEvent(std::string id, int maxFieldLength) const {
TraceEvent("TransactionTrace_GetRangeError")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.detail("ErrCode", errCode)
.setMaxFieldLength(maxFieldLength)
.detail("StartKey", startKey)
.detail("EndKey", endKey);
}
};
struct EventCommitError : public Event {
EventCommitError(double ts, const Optional<Standalone<StringRef>>& dcId, int err_code,
const CommitTransactionRequest& commit_req)
: Event(EventType::ERROR_COMMIT, ts, dcId), errCode(err_code), req(commit_req) {}
EventCommitError() { }
template <typename Ar> Ar& serialize(Ar &ar) {
if (!ar.isDeserializing)
return serializer(Event::serialize(ar), errCode, req.transaction, req.arena);
else
return serializer(ar, errCode, req.transaction, req.arena);
}
int errCode;
CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized
void logEvent(std::string id, int maxFieldLength) const {
for (auto &read_range : req.transaction.read_conflict_ranges) {
TraceEvent("TransactionTrace_CommitError_ReadConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", read_range.begin)
.detail("End", read_range.end);
}
for (auto &write_range : req.transaction.write_conflict_ranges) {
TraceEvent("TransactionTrace_CommitError_WriteConflictRange")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Begin", write_range.begin)
.detail("End", write_range.end);
}
for (auto &mutation : req.transaction.mutations) {
TraceEvent("TransactionTrace_CommitError_Mutation")
.setMaxEventLength(-1)
.detail("TransactionID", id)
.setMaxFieldLength(maxFieldLength)
.detail("Mutation", mutation.toString());
}
TraceEvent("TransactionTrace_CommitError")
.detail("TransactionID", id)
.detail("ErrCode", errCode);
}
};
}
TraceEvent("TransactionTrace_CommitError").detail("TransactionID", id).detail("ErrCode", errCode);
}
};
} // namespace FdbClientLogEvents
#endif

24
fdbclient/ClientWorkerInterface.h
View File

@ -31,20 +31,18 @@
// A ClientWorkerInterface is embedded as the first element of a WorkerInterface.
struct ClientWorkerInterface {
constexpr static FileIdentifier file_identifier = 12418152;
RequestStream< struct RebootRequest > reboot;
RequestStream< struct ProfilerRequest > profiler;
RequestStream<struct RebootRequest> reboot;
RequestStream<struct ProfilerRequest> profiler;
bool operator == (ClientWorkerInterface const& r) const { return id() == r.id(); }
bool operator != (ClientWorkerInterface const& r) const { return id() != r.id(); }
bool operator==(ClientWorkerInterface const& r) const { return id() == r.id(); }
bool operator!=(ClientWorkerInterface const& r) const { return id() != r.id(); }
UID id() const { return reboot.getEndpoint().token; }
NetworkAddress address() const { return reboot.getEndpoint().getPrimaryAddress(); }
void initEndpoints() {
reboot.getEndpoint( TaskPriority::ReadSocket );
}
void initEndpoints() { reboot.getEndpoint(TaskPriority::ReadSocket); }
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
serializer(ar, reboot, profiler);
}
};
@ -74,11 +72,7 @@ struct ProfilerRequest {
GPROF_HEAP = 3,
};
enum class Action : std::int8_t {
DISABLE = 0,
ENABLE = 1,
RUN = 2
};
enum class Action : std::int8_t { DISABLE = 0, ENABLE = 1, RUN = 2 };
Type type;
Action action;
@ -88,8 +82,8 @@ struct ProfilerRequest {
ProfilerRequest() = default;
explicit ProfilerRequest(Type t, Action a, int d) : type(t), action(a), duration(d) {}
template<class Ar>
void serialize( Ar& ar ) {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, reply, type, action, duration, outputFile);
}
};

103
fdbclient/ClusterInterface.h
View File

@ -29,39 +29,36 @@
#include "fdbclient/ClientWorkerInterface.h"
struct ClusterInterface {
constexpr static FileIdentifier file_identifier = 15888863;
RequestStream< struct OpenDatabaseRequest > openDatabase;
RequestStream< struct FailureMonitoringRequest > failureMonitoring;
RequestStream< struct StatusRequest > databaseStatus;
RequestStream< ReplyPromise<Void> > ping;
RequestStream< struct GetClientWorkersRequest > getClientWorkers;
RequestStream< struct ForceRecoveryRequest > forceRecovery;
constexpr static FileIdentifier file_identifier = 15888863;
RequestStream<struct OpenDatabaseRequest> openDatabase;
RequestStream<struct FailureMonitoringRequest> failureMonitoring;
RequestStream<struct StatusRequest> databaseStatus;
RequestStream<ReplyPromise<Void>> ping;
RequestStream<struct GetClientWorkersRequest> getClientWorkers;
RequestStream<struct ForceRecoveryRequest> forceRecovery;
bool operator == (ClusterInterface const& r) const { return id() == r.id(); }
bool operator != (ClusterInterface const& r) const { return id() != r.id(); }
bool operator==(ClusterInterface const& r) const { return id() == r.id(); }
bool operator!=(ClusterInterface const& r) const { return id() != r.id(); }
UID id() const { return openDatabase.getEndpoint().token; }
NetworkAddress address() const { return openDatabase.getEndpoint().getPrimaryAddress(); }
bool hasMessage() {
return openDatabase.getFuture().isReady() ||
failureMonitoring.getFuture().isReady() ||
databaseStatus.getFuture().isReady() ||
ping.getFuture().isReady() ||
getClientWorkers.getFuture().isReady() ||
forceRecovery.getFuture().isReady();
return openDatabase.getFuture().isReady() || failureMonitoring.getFuture().isReady() ||
databaseStatus.getFuture().isReady() || ping.getFuture().isReady() ||
getClientWorkers.getFuture().isReady() || forceRecovery.getFuture().isReady();
}
void initEndpoints() {
openDatabase.getEndpoint( TaskPriority::ClusterController );
failureMonitoring.getEndpoint( TaskPriority::FailureMonitor );
databaseStatus.getEndpoint( TaskPriority::ClusterController );
ping.getEndpoint( TaskPriority::ClusterController );
getClientWorkers.getEndpoint( TaskPriority::ClusterController );
forceRecovery.getEndpoint( TaskPriority::ClusterController );
openDatabase.getEndpoint(TaskPriority::ClusterController);
failureMonitoring.getEndpoint(TaskPriority::FailureMonitor);
databaseStatus.getEndpoint(TaskPriority::ClusterController);
ping.getEndpoint(TaskPriority::ClusterController);
getClientWorkers.getEndpoint(TaskPriority::ClusterController);
forceRecovery.getEndpoint(TaskPriority::ClusterController);
}
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
serializer(ar, openDatabase, failureMonitoring, databaseStatus, ping, getClientWorkers, forceRecovery);
}
};
@ -88,12 +85,13 @@ struct ClientVersionRef {
StringRef sourceVersion;
StringRef protocolVersion;
ClientVersionRef() {
initUnknown();
}
ClientVersionRef() { initUnknown(); }
ClientVersionRef(Arena &arena, ClientVersionRef const& cv) : clientVersion(arena, cv.clientVersion), sourceVersion(arena, cv.sourceVersion), protocolVersion(arena, cv.protocolVersion) {}
ClientVersionRef(StringRef clientVersion, StringRef sourceVersion, StringRef protocolVersion) : clientVersion(clientVersion), sourceVersion(sourceVersion), protocolVersion(protocolVersion) {}
ClientVersionRef(Arena& arena, ClientVersionRef const& cv)
: clientVersion(arena, cv.clientVersion), sourceVersion(arena, cv.sourceVersion),
protocolVersion(arena, cv.protocolVersion) {}
ClientVersionRef(StringRef clientVersion, StringRef sourceVersion, StringRef protocolVersion)
: clientVersion(clientVersion), sourceVersion(sourceVersion), protocolVersion(protocolVersion) {}
ClientVersionRef(StringRef versionString) {
std::vector<StringRef> parts = versionString.splitAny(LiteralStringRef(","));
if (parts.size() != 3) {
@ -119,12 +117,12 @@ struct ClientVersionRef {
size_t expectedSize() const { return clientVersion.size() + sourceVersion.size() + protocolVersion.size(); }
bool operator<(const ClientVersionRef& rhs) const {
if(protocolVersion != rhs.protocolVersion) {
if (protocolVersion != rhs.protocolVersion) {
return protocolVersion < rhs.protocolVersion;
}
// These comparisons are arbitrary because they aren't ordered
if(clientVersion != rhs.clientVersion) {
if (clientVersion != rhs.clientVersion) {
return clientVersion < rhs.clientVersion;
}
@ -136,10 +134,11 @@ template <class T>
struct ItemWithExamples {
T item;
int count;
std::vector<std::pair<NetworkAddress,Key>> examples;
std::vector<std::pair<NetworkAddress, Key>> examples;
ItemWithExamples() : item{}, count(0) {}
ItemWithExamples(T const& item, int count, std::vector<std::pair<NetworkAddress,Key>> const& examples) : item(item), count(count), examples(examples) {}
ItemWithExamples(T const& item, int count, std::vector<std::pair<NetworkAddress, Key>> const& examples)
: item(item), count(count), examples(examples) {}
template <class Ar>
void serialize(Ar& ar) {
@ -157,9 +156,9 @@ struct OpenDatabaseRequest {
std::vector<ItemWithExamples<Key>> issues;
std::vector<ItemWithExamples<Standalone<ClientVersionRef>>> supportedVersions;
std::vector<ItemWithExamples<Key>> maxProtocolSupported;
UID knownClientInfoID;
ReplyPromise< struct ClientDBInfo > reply;
ReplyPromise<struct ClientDBInfo> reply;
template <class Ar>
void serialize(Ar& ar) {
@ -176,7 +175,7 @@ struct SystemFailureStatus {
FailureStatus status;
SystemFailureStatus() {}
SystemFailureStatus( NetworkAddressList const& a, FailureStatus const& s ) : addresses(a), status(s) {}
SystemFailureStatus(NetworkAddressList const& a, FailureStatus const& s) : addresses(a), status(s) {}
template <class Ar>
void serialize(Ar& ar) {
@ -186,16 +185,24 @@ struct SystemFailureStatus {
struct FailureMonitoringReply {
constexpr static FileIdentifier file_identifier = 6820325;
VectorRef< SystemFailureStatus > changes;
VectorRef<SystemFailureStatus> changes;
Version failureInformationVersion;
bool allOthersFailed; // If true, changes are relative to all servers being failed, otherwise to the version given in the request
int clientRequestIntervalMS, // after this many milliseconds, send another request
considerServerFailedTimeoutMS; // after this many additional milliseconds, consider the ClusterController itself to be failed
bool allOthersFailed; // If true, changes are relative to all servers being failed, otherwise to the version given
// in the request
int clientRequestIntervalMS, // after this many milliseconds, send another request
considerServerFailedTimeoutMS; // after this many additional milliseconds, consider the ClusterController itself
// to be failed
Arena arena;
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, changes, failureInformationVersion, allOthersFailed, clientRequestIntervalMS, considerServerFailedTimeoutMS, arena);
serializer(ar,
changes,
failureInformationVersion,
allOthersFailed,
clientRequestIntervalMS,
considerServerFailedTimeoutMS,
arena);
}
};
@ -214,7 +221,7 @@ struct FailureMonitoringRequest {
Optional<FailureStatus> senderStatus;
Version failureInformationVersion;
NetworkAddressList addresses;
ReplyPromise< struct FailureMonitoringReply > reply;
ReplyPromise<struct FailureMonitoringReply> reply;
template <class Ar>
void serialize(Ar& ar) {
@ -228,20 +235,20 @@ struct StatusReply {
std::string statusStr;
StatusReply() {}
explicit StatusReply(StatusObject obj) : statusObj(obj), statusStr(json_spirit::write_string(json_spirit::mValue(obj))) {}
explicit StatusReply(std::string &&text) : statusStr(text) {}
explicit StatusReply(StatusObject obj)
: statusObj(obj), statusStr(json_spirit::write_string(json_spirit::mValue(obj))) {}
explicit StatusReply(std::string&& text) : statusStr(text) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, statusStr);
if( ar.isDeserializing ) {
if (ar.isDeserializing) {
json_spirit::mValue mv;
if(g_network->isSimulated()) {
if (g_network->isSimulated()) {
mv = readJSONStrictly(statusStr);
}
else {
} else {
// In non-simulation allow errors because some status data is better than no status data
json_spirit::read_string( statusStr, mv );
json_spirit::read_string(statusStr, mv);
}
statusObj = std::move(mv.get_obj());
}
@ -250,7 +257,7 @@ struct StatusReply {
struct StatusRequest {
constexpr static FileIdentifier file_identifier = 14419140;
ReplyPromise< struct StatusReply > reply;
ReplyPromise<struct StatusReply> reply;
template <class Ar>
void serialize(Ar& ar) {

227
fdbclient/CommitProxyInterface.h
View File

@ -43,19 +43,22 @@ struct CommitProxyInterface {
Optional<Key> processId;
bool provisional;
RequestStream< struct CommitTransactionRequest > commit;
RequestStream< struct GetReadVersionRequest > getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported committed (by a commit response) when this request was sent
// (at some point between when this request is sent and when its response is received, the latest version reported committed)
RequestStream< struct GetKeyServerLocationsRequest > getKeyServersLocations;
RequestStream< struct GetStorageServerRejoinInfoRequest > getStorageServerRejoinInfo;
RequestStream<struct CommitTransactionRequest> commit;
RequestStream<struct GetReadVersionRequest>
getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported
// committed (by a commit response) when this request was sent
// (at some point between when this request is sent and when its response is
// received, the latest version reported committed)
RequestStream<struct GetKeyServerLocationsRequest> getKeyServersLocations;
RequestStream<struct GetStorageServerRejoinInfoRequest> getStorageServerRejoinInfo;
RequestStream<ReplyPromise<Void>> waitFailure;
RequestStream< struct TxnStateRequest > txnState;
RequestStream< struct GetHealthMetricsRequest > getHealthMetrics;
RequestStream< struct ProxySnapRequest > proxySnapReq;
RequestStream< struct ExclusionSafetyCheckRequest > exclusionSafetyCheckReq;
RequestStream< struct GetDDMetricsRequest > getDDMetrics;
RequestStream<struct TxnStateRequest> txnState;
RequestStream<struct GetHealthMetricsRequest> getHealthMetrics;
RequestStream<struct ProxySnapRequest> proxySnapReq;
RequestStream<struct ExclusionSafetyCheckRequest> exclusionSafetyCheckReq;
RequestStream<struct GetDDMetricsRequest> getDDMetrics;
UID id() const { return commit.getEndpoint().token; }
std::string toString() const { return id().shortString(); }
@ -66,16 +69,21 @@ struct CommitProxyInterface {
template <class Archive>
void serialize(Archive& ar) {
serializer(ar, processId, provisional, commit);
if( Archive::isDeserializing ) {
getConsistentReadVersion = RequestStream< struct GetReadVersionRequest >( commit.getEndpoint().getAdjustedEndpoint(1) );
getKeyServersLocations = RequestStream< struct GetKeyServerLocationsRequest >( commit.getEndpoint().getAdjustedEndpoint(2) );
getStorageServerRejoinInfo = RequestStream< struct GetStorageServerRejoinInfoRequest >( commit.getEndpoint().getAdjustedEndpoint(3) );
waitFailure = RequestStream<ReplyPromise<Void>>( commit.getEndpoint().getAdjustedEndpoint(4) );
txnState = RequestStream< struct TxnStateRequest >( commit.getEndpoint().getAdjustedEndpoint(5) );
getHealthMetrics = RequestStream< struct GetHealthMetricsRequest >( commit.getEndpoint().getAdjustedEndpoint(6) );
proxySnapReq = RequestStream< struct ProxySnapRequest >( commit.getEndpoint().getAdjustedEndpoint(7) );
exclusionSafetyCheckReq = RequestStream< struct ExclusionSafetyCheckRequest >( commit.getEndpoint().getAdjustedEndpoint(8) );
getDDMetrics = RequestStream< struct GetDDMetricsRequest >( commit.getEndpoint().getAdjustedEndpoint(9) );
if (Archive::isDeserializing) {
getConsistentReadVersion =
RequestStream<struct GetReadVersionRequest>(commit.getEndpoint().getAdjustedEndpoint(1));
getKeyServersLocations =
RequestStream<struct GetKeyServerLocationsRequest>(commit.getEndpoint().getAdjustedEndpoint(2));
getStorageServerRejoinInfo =
RequestStream<struct GetStorageServerRejoinInfoRequest>(commit.getEndpoint().getAdjustedEndpoint(3));
waitFailure = RequestStream<ReplyPromise<Void>>(commit.getEndpoint().getAdjustedEndpoint(4));
txnState = RequestStream<struct TxnStateRequest>(commit.getEndpoint().getAdjustedEndpoint(5));
getHealthMetrics =
RequestStream<struct GetHealthMetricsRequest>(commit.getEndpoint().getAdjustedEndpoint(6));
proxySnapReq = RequestStream<struct ProxySnapRequest>(commit.getEndpoint().getAdjustedEndpoint(7));
exclusionSafetyCheckReq =
RequestStream<struct ExclusionSafetyCheckRequest>(commit.getEndpoint().getAdjustedEndpoint(8));
getDDMetrics = RequestStream<struct GetDDMetricsRequest>(commit.getEndpoint().getAdjustedEndpoint(9));
}
}
@ -83,7 +91,8 @@ struct CommitProxyInterface {
std::vector<std::pair<FlowReceiver*, TaskPriority>> streams;
streams.push_back(commit.getReceiver(TaskPriority::ReadSocket));
streams.push_back(getConsistentReadVersion.getReceiver(TaskPriority::ReadSocket));
streams.push_back(getKeyServersLocations.getReceiver(TaskPriority::ReadSocket)); //priority lowered to TaskPriority::DefaultEndpoint on the proxy
streams.push_back(getKeyServersLocations.getReceiver(
TaskPriority::ReadSocket)); // priority lowered to TaskPriority::DefaultEndpoint on the proxy
streams.push_back(getStorageServerRejoinInfo.getReceiver(TaskPriority::ProxyStorageRejoin));
streams.push_back(waitFailure.getReceiver());
streams.push_back(txnState.getReceiver());
@ -99,7 +108,7 @@ struct CommitProxyInterface {
// It is returned (and kept up to date) by the OpenDatabaseRequest interface of ClusterInterface
struct ClientDBInfo {
constexpr static FileIdentifier file_identifier = 5355080;
UID id; // Changes each time anything else changes
UID id; // Changes each time anything else changes
vector<GrvProxyInterface> grvProxies;
vector<CommitProxyInterface> commitProxies;
Optional<CommitProxyInterface>
@ -115,22 +124,29 @@ struct ClientDBInfo {
transactionTagSampleRate(CLIENT_KNOBS->READ_TAG_SAMPLE_RATE),
transactionTagSampleCost(CLIENT_KNOBS->COMMIT_SAMPLE_COST) {}
bool operator == (ClientDBInfo const& r) const { return id == r.id; }
bool operator != (ClientDBInfo const& r) const { return id != r.id; }
bool operator==(ClientDBInfo const& r) const { return id == r.id; }
bool operator!=(ClientDBInfo const& r) const { return id != r.id; }
template <class Archive>
void serialize(Archive& ar) {
if constexpr (!is_fb_function<Archive>) {
ASSERT(ar.protocolVersion().isValid());
}
serializer(ar, grvProxies, commitProxies, id, clientTxnInfoSampleRate, clientTxnInfoSizeLimit, forward,
transactionTagSampleRate, transactionTagSampleCost);
serializer(ar,
grvProxies,
commitProxies,
id,
clientTxnInfoSampleRate,
clientTxnInfoSizeLimit,
forward,
transactionTagSampleRate,
transactionTagSampleCost);
}
};
struct CommitID {
constexpr static FileIdentifier file_identifier = 14254927;
Version version; // returns invalidVersion if transaction conflicts
Version version; // returns invalidVersion if transaction conflicts
uint16_t txnBatchId;
Optional<Value> metadataVersion;
Optional<Standalone<VectorRef<int>>> conflictingKRIndices;
@ -141,7 +157,9 @@ struct CommitID {
}
CommitID() : version(invalidVersion), txnBatchId(0) {}
CommitID(Version version, uint16_t txnBatchId, const Optional<Value>& metadataVersion,
CommitID(Version version,
uint16_t txnBatchId,
const Optional<Value>& metadataVersion,
const Optional<Standalone<VectorRef<int>>>& conflictingKRIndices = Optional<Standalone<VectorRef<int>>>())
: version(version), txnBatchId(txnBatchId), metadataVersion(metadataVersion),
conflictingKRIndices(conflictingKRIndices) {}
@ -149,14 +167,11 @@ struct CommitID {
struct CommitTransactionRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 93948;
enum {
FLAG_IS_LOCK_AWARE = 0x1,
FLAG_FIRST_IN_BATCH = 0x2
};
enum { FLAG_IS_LOCK_AWARE = 0x1, FLAG_FIRST_IN_BATCH = 0x2 };
bool isLockAware() const { return (flags & FLAG_IS_LOCK_AWARE) != 0; }
bool firstInBatch() const { return (flags & FLAG_FIRST_IN_BATCH) != 0; }
Arena arena;
SpanID spanContext;
CommitTransactionRef transaction;
@ -169,21 +184,21 @@ struct CommitTransactionRequest : TimedRequest {
CommitTransactionRequest() : CommitTransactionRequest(SpanID()) {}
CommitTransactionRequest(SpanID const& context) : spanContext(context), flags(0) {}
template <class Ar>
void serialize(Ar& ar) {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, transaction, reply, arena, flags, debugID, commitCostEstimation, tagSet, spanContext);
}
};
static inline int getBytes( CommitTransactionRequest const& r ) {
static inline int getBytes(CommitTransactionRequest const& r) {
// SOMEDAY: Optimize
//return r.arena.getSize(); // NOT correct because arena can be shared!
// return r.arena.getSize(); // NOT correct because arena can be shared!
int total = sizeof(r);
for(auto m = r.transaction.mutations.begin(); m != r.transaction.mutations.end(); ++m)
for (auto m = r.transaction.mutations.begin(); m != r.transaction.mutations.end(); ++m)
total += m->expectedSize() + CLIENT_KNOBS->PROXY_COMMIT_OVERHEAD_BYTES;
for(auto i = r.transaction.read_conflict_ranges.begin(); i != r.transaction.read_conflict_ranges.end(); ++i)
for (auto i = r.transaction.read_conflict_ranges.begin(); i != r.transaction.read_conflict_ranges.end(); ++i)
total += i->expectedSize();
for(auto i = r.transaction.write_conflict_ranges.begin(); i != r.transaction.write_conflict_ranges.end(); ++i)
for (auto i = r.transaction.write_conflict_ranges.begin(); i != r.transaction.write_conflict_ranges.end(); ++i)
total += i->expectedSize();
return total;
}
@ -201,14 +216,21 @@ struct GetReadVersionReply : public BasicLoadBalancedReply {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, BasicLoadBalancedReply::processBusyTime, version, locked, metadataVersion, tagThrottleInfo, midShardSize);
serializer(ar,
BasicLoadBalancedReply::processBusyTime,
version,
locked,
metadataVersion,
tagThrottleInfo,
midShardSize);
}
};
struct GetReadVersionRequest : TimedRequest {
constexpr static FileIdentifier file_identifier = 838566;
enum {
PRIORITY_SYSTEM_IMMEDIATE = 15 << 24, // Highest possible priority, always executed even if writes are otherwise blocked
enum {
PRIORITY_SYSTEM_IMMEDIATE =
15 << 24, // Highest possible priority, always executed even if writes are otherwise blocked
PRIORITY_DEFAULT = 8 << 24,
PRIORITY_BATCH = 1 << 24
};
@ -230,44 +252,44 @@ struct GetReadVersionRequest : TimedRequest {
ReplyPromise<GetReadVersionReply> reply;
GetReadVersionRequest() : transactionCount(1), flags(0) {}
GetReadVersionRequest(SpanID spanContext, uint32_t transactionCount, TransactionPriority priority,
uint32_t flags = 0, TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(),
GetReadVersionRequest(SpanID spanContext,
uint32_t transactionCount,
TransactionPriority priority,
uint32_t flags = 0,
TransactionTagMap<uint32_t> tags = TransactionTagMap<uint32_t>(),
Optional<UID> debugID = Optional<UID>())
: spanContext(spanContext), transactionCount(transactionCount), priority(priority), flags(flags), tags(tags),
debugID(debugID) {
flags = flags & ~FLAG_PRIORITY_MASK;
switch(priority) {
case TransactionPriority::BATCH:
flags |= PRIORITY_BATCH;
break;
case TransactionPriority::DEFAULT:
flags |= PRIORITY_DEFAULT;
break;
case TransactionPriority::IMMEDIATE:
flags |= PRIORITY_SYSTEM_IMMEDIATE;
break;
default:
ASSERT(false);
switch (priority) {
case TransactionPriority::BATCH:
flags |= PRIORITY_BATCH;
break;
case TransactionPriority::DEFAULT:
flags |= PRIORITY_DEFAULT;
break;
case TransactionPriority::IMMEDIATE:
flags |= PRIORITY_SYSTEM_IMMEDIATE;
break;
default:
ASSERT(false);
}
}
bool operator < (GetReadVersionRequest const& rhs) const { return priority < rhs.priority; }
bool operator<(GetReadVersionRequest const& rhs) const { return priority < rhs.priority; }
template <class Ar>
void serialize(Ar& ar) {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, transactionCount, flags, tags, debugID, reply, spanContext);
if(ar.isDeserializing) {
if((flags & PRIORITY_SYSTEM_IMMEDIATE) == PRIORITY_SYSTEM_IMMEDIATE) {
if (ar.isDeserializing) {
if ((flags & PRIORITY_SYSTEM_IMMEDIATE) == PRIORITY_SYSTEM_IMMEDIATE) {
priority = TransactionPriority::IMMEDIATE;
}
else if((flags & PRIORITY_DEFAULT) == PRIORITY_DEFAULT) {
} else if ((flags & PRIORITY_DEFAULT) == PRIORITY_DEFAULT) {
priority = TransactionPriority::DEFAULT;
}
else if((flags & PRIORITY_BATCH) == PRIORITY_BATCH) {
} else if ((flags & PRIORITY_BATCH) == PRIORITY_BATCH) {
priority = TransactionPriority::BATCH;
}
else {
} else {
priority = TransactionPriority::DEFAULT;
}
}
@ -296,12 +318,16 @@ struct GetKeyServerLocationsRequest {
ReplyPromise<GetKeyServerLocationsReply> reply;
GetKeyServerLocationsRequest() : limit(0), reverse(false) {}
GetKeyServerLocationsRequest(SpanID spanContext, KeyRef const& begin, Optional<KeyRef> const& end, int limit,
bool reverse, Arena const& arena)
GetKeyServerLocationsRequest(SpanID spanContext,
KeyRef const& begin,
Optional<KeyRef> const& end,
int limit,
bool reverse,
Arena const& arena)
: spanContext(spanContext), begin(begin), end(end), limit(limit), reverse(reverse), arena(arena) {}
template <class Ar>
void serialize(Ar& ar) {
void serialize(Ar& ar) {
serializer(ar, begin, end, limit, reverse, reply, spanContext, arena);
}
};
@ -314,10 +340,12 @@ struct GetRawCommittedVersionReply {
Optional<Value> metadataVersion;
Version minKnownCommittedVersion;
GetRawCommittedVersionReply(): debugID(Optional<UID>()), version(invalidVersion), locked(false), metadataVersion(Optional<Value>()), minKnownCommittedVersion(invalidVersion) {}
GetRawCommittedVersionReply()
: debugID(Optional<UID>()), version(invalidVersion), locked(false), metadataVersion(Optional<Value>()),
minKnownCommittedVersion(invalidVersion) {}
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
serializer(ar, debugID, version, locked, metadataVersion, minKnownCommittedVersion);
}
};
@ -328,11 +356,12 @@ struct GetRawCommittedVersionRequest {
Optional<UID> debugID;
ReplyPromise<GetRawCommittedVersionReply> reply;
explicit GetRawCommittedVersionRequest(SpanID spanContext, Optional<UID> const& debugID = Optional<UID>()) : spanContext(spanContext), debugID(debugID) {}
explicit GetRawCommittedVersionRequest(SpanID spanContext, Optional<UID> const& debugID = Optional<UID>())
: spanContext(spanContext), debugID(debugID) {}
explicit GetRawCommittedVersionRequest() : spanContext(), debugID() {}
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
serializer(ar, debugID, reply, spanContext);
}
};
@ -355,13 +384,13 @@ struct GetStorageServerRejoinInfoRequest {
constexpr static FileIdentifier file_identifier = 994279;
UID id;
Optional<Value> dcId;
ReplyPromise< GetStorageServerRejoinInfoReply > reply;
ReplyPromise<GetStorageServerRejoinInfoReply> reply;
GetStorageServerRejoinInfoRequest() {}
explicit GetStorageServerRejoinInfoRequest( UID const& id, Optional<Value> const& dcId ) : id(id), dcId(dcId) {}
explicit GetStorageServerRejoinInfoRequest(UID const& id, Optional<Value> const& dcId) : id(id), dcId(dcId) {}
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
serializer(ar, id, dcId, reply);
}
};
@ -375,26 +404,23 @@ struct TxnStateRequest {
std::vector<Endpoint> broadcastInfo;
ReplyPromise<Void> reply;
template <class Ar>
void serialize(Ar& ar) {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, data, sequence, last, broadcastInfo, reply, arena);
}
};
struct GetHealthMetricsReply
{
struct GetHealthMetricsReply {
constexpr static FileIdentifier file_identifier = 11544290;
Standalone<StringRef> serialized;
HealthMetrics healthMetrics;
explicit GetHealthMetricsReply(const HealthMetrics& healthMetrics = HealthMetrics()) :
healthMetrics(healthMetrics)
{
explicit GetHealthMetricsReply(const HealthMetrics& healthMetrics = HealthMetrics())
: healthMetrics(healthMetrics) {
update(healthMetrics, true, true);
}
void update(const HealthMetrics& healthMetrics, bool detailedInput, bool detailedOutput)
{
void update(const HealthMetrics& healthMetrics, bool detailedInput, bool detailedOutput) {
this->healthMetrics.update(healthMetrics, detailedInput, detailedOutput);
BinaryWriter bw(IncludeVersion());
bw << this->healthMetrics;
@ -411,8 +437,7 @@ struct GetHealthMetricsReply
}
};
struct GetHealthMetricsRequest
{
struct GetHealthMetricsRequest {
constexpr static FileIdentifier file_identifier = 11403900;
ReplyPromise<struct GetHealthMetricsReply> reply;
bool detailed;
@ -420,14 +445,12 @@ struct GetHealthMetricsRequest
explicit GetHealthMetricsRequest(bool detailed = false) : detailed(detailed) {}
template <class Ar>
void serialize(Ar& ar)
{
void serialize(Ar& ar) {
serializer(ar, reply, detailed);
}
};
struct GetDDMetricsReply
{
struct GetDDMetricsReply {
constexpr static FileIdentifier file_identifier = 7277713;
Standalone<VectorRef<DDMetricsRef>> storageMetricsList;
@ -448,14 +471,13 @@ struct GetDDMetricsRequest {
GetDDMetricsRequest() {}
explicit GetDDMetricsRequest(KeyRange const& keys, const int shardLimit) : keys(keys), shardLimit(shardLimit) {}
template<class Ar>
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, keys, shardLimit, reply);
}
}
};
struct ProxySnapRequest
{
struct ProxySnapRequest {
constexpr static FileIdentifier file_identifier = 5427684;
Arena arena;
StringRef snapPayload; // command used to snapshot the data folder
@ -464,7 +486,8 @@ struct ProxySnapRequest
Optional<UID> debugID;
explicit ProxySnapRequest(Optional<UID> const& debugID = Optional<UID>()) : debugID(debugID) {}
explicit ProxySnapRequest(StringRef snap, UID snapUID, Optional<UID> debugID = Optional<UID>()) : snapPayload(snap), snapUID(snapUID), debugID(debugID) {}
explicit ProxySnapRequest(StringRef snap, UID snapUID, Optional<UID> debugID = Optional<UID>())
: snapPayload(snap), snapUID(snapUID), debugID(debugID) {}
template <class Ar>
void serialize(Ar& ar) {
@ -472,8 +495,7 @@ struct ProxySnapRequest
}
};
struct ExclusionSafetyCheckReply
{
struct ExclusionSafetyCheckReply {
constexpr static FileIdentifier file_identifier = 11;
bool safe;
@ -486,8 +508,7 @@ struct ExclusionSafetyCheckReply
}
};
struct ExclusionSafetyCheckRequest
{
struct ExclusionSafetyCheckRequest {
constexpr static FileIdentifier file_identifier = 13852702;
vector<AddressExclusion> exclusions;
ReplyPromise<ExclusionSafetyCheckReply> reply;
@ -496,7 +517,7 @@ struct ExclusionSafetyCheckRequest
explicit ExclusionSafetyCheckRequest(vector<AddressExclusion> exclusions) : exclusions(exclusions) {}
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
serializer(ar, exclusions, reply);
}
};

51
fdbclient/CommitTransaction.h
View File

@ -53,7 +53,7 @@ static const char* typeString[] = { "SetValue",
"MAX_ATOMIC_OP" };
struct MutationRef {
static const int OVERHEAD_BYTES = 12; //12 is the size of Header in MutationList entries
static const int OVERHEAD_BYTES = 12; // 12 is the size of Header in MutationList entries
enum Type : uint8_t {
SetValue = 0,
ClearRange,
@ -84,9 +84,10 @@ struct MutationRef {
StringRef param1, param2;
MutationRef() {}
MutationRef( Type t, StringRef a, StringRef b ) : type(t), param1(a), param2(b) {}
MutationRef( Arena& to, Type t, StringRef a, StringRef b ) : type(t), param1(to, a), param2(to, b) {}
MutationRef( Arena& to, const MutationRef& from ) : type(from.type), param1( to, from.param1 ), param2( to, from.param2 ) {}
MutationRef(Type t, StringRef a, StringRef b) : type(t), param1(a), param2(b) {}
MutationRef(Arena& to, Type t, StringRef a, StringRef b) : type(t), param1(to, a), param2(to, b) {}
MutationRef(Arena& to, const MutationRef& from)
: type(from.type), param1(to, from.param1), param2(to, from.param2) {}
int totalSize() const { return OVERHEAD_BYTES + param1.size() + param2.size(); }
int expectedSize() const { return param1.size() + param2.size(); }
int weightedTotalSize() const {
@ -102,14 +103,15 @@ struct MutationRef {
std::string toString() const {
return format("code: %s param1: %s param2: %s",
type < MutationRef::MAX_ATOMIC_OP ? typeString[(int)type] : "Unset", printable(param1).c_str(),
type < MutationRef::MAX_ATOMIC_OP ? typeString[(int)type] : "Unset",
printable(param1).c_str(),
printable(param2).c_str());
}
bool isAtomicOp() const { return (ATOMIC_MASK & (1 << type)) != 0; }
template <class Ar>
void serialize( Ar& ar ) {
void serialize(Ar& ar) {
if (ar.isSerializing && type == ClearRange && equalsKeyAfter(param1, param2)) {
StringRef empty;
serializer(ar, type, param2, empty);
@ -117,13 +119,14 @@ struct MutationRef {
serializer(ar, type, param1, param2);
}
if (ar.isDeserializing && type == ClearRange && param2 == StringRef() && param1 != StringRef()) {
ASSERT(param1[param1.size()-1] == '\x00');
ASSERT(param1[param1.size() - 1] == '\x00');
param2 = param1;
param1 = param2.substr(0, param2.size()-1);
param1 = param2.substr(0, param2.size() - 1);
}
}
// These masks define which mutation types have particular properties (they are used to implement isSingleKeyMutation() etc)
// These masks define which mutation types have particular properties (they are used to implement
// isSingleKeyMutation() etc)
enum {
ATOMIC_MASK = (1 << AddValue) | (1 << And) | (1 << Or) | (1 << Xor) | (1 << AppendIfFits) | (1 << Max) |
(1 << Min) | (1 << SetVersionstampedKey) | (1 << SetVersionstampedValue) | (1 << ByteMin) |
@ -135,11 +138,9 @@ struct MutationRef {
};
};
template<>
template <>
struct Traceable<MutationRef> : std::true_type {
static std::string toString(MutationRef const& value) {
return value.toString();
}
static std::string toString(MutationRef const& value) { return value.toString(); }
};
static inline std::string getTypeString(MutationRef::Type type) {
@ -152,7 +153,7 @@ static inline std::string getTypeString(uint8_t type) {
// A 'single key mutation' is one which affects exactly the value of the key specified by its param1
static inline bool isSingleKeyMutation(MutationRef::Type type) {
return (MutationRef::SINGLE_KEY_MASK & (1<<type)) != 0;
return (MutationRef::SINGLE_KEY_MASK & (1 << type)) != 0;
}
// Returns true if the given type can be safely cast to MutationRef::Type and used as a parameter to
@ -162,17 +163,17 @@ static inline bool isValidMutationType(uint32_t type) {
return (type < MutationRef::MAX_ATOMIC_OP);
}
// An 'atomic operation' is a single key mutation which sets the key specified by its param1 to a
// nontrivial function of the previous value of the key and param2, and thus requires a
// An 'atomic operation' is a single key mutation which sets the key specified by its param1 to a
// nontrivial function of the previous value of the key and param2, and thus requires a
// read/modify/write to implement. (Basically a single key mutation other than a set)
static inline bool isAtomicOp(MutationRef::Type mutationType) {
return (MutationRef::ATOMIC_MASK & (1<<mutationType)) != 0;
return (MutationRef::ATOMIC_MASK & (1 << mutationType)) != 0;
}
// Returns true for operations which do not obey the associative law (i.e. a*(b*c) == (a*b)*c) in all cases
// unless a, b, and c have equal lengths, in which case even these operations are associative.
static inline bool isNonAssociativeOp(MutationRef::Type mutationType) {
return (MutationRef::NON_ASSOCIATIVE_MASK & (1<<mutationType)) != 0;
return (MutationRef::NON_ASSOCIATIVE_MASK & (1 << mutationType)) != 0;
}
struct CommitTransactionRef {
@ -181,17 +182,17 @@ struct CommitTransactionRef {
: read_conflict_ranges(a, from.read_conflict_ranges), write_conflict_ranges(a, from.write_conflict_ranges),
mutations(a, from.mutations), read_snapshot(from.read_snapshot),
report_conflicting_keys(from.report_conflicting_keys) {}
VectorRef< KeyRangeRef > read_conflict_ranges;
VectorRef< KeyRangeRef > write_conflict_ranges;
VectorRef< MutationRef > mutations;
VectorRef<KeyRangeRef> read_conflict_ranges;
VectorRef<KeyRangeRef> write_conflict_ranges;
VectorRef<MutationRef> mutations;
Version read_snapshot;
bool report_conflicting_keys;
template <class Ar>
force_inline void serialize(Ar& ar) {
if constexpr (is_fb_function<Ar>) {
serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot,
report_conflicting_keys);
serializer(
ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot, report_conflicting_keys);
} else {
serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot);
if (ar.protocolVersion().hasReportConflictingKeys()) {
@ -201,12 +202,12 @@ struct CommitTransactionRef {
}
// Convenience for internal code required to manipulate these without the Native API
void set( Arena& arena, KeyRef const& key, ValueRef const& value ) {
void set(Arena& arena, KeyRef const& key, ValueRef const& value) {
mutations.push_back_deep(arena, MutationRef(MutationRef::SetValue, key, value));
write_conflict_ranges.push_back(arena, singleKeyRange(key, arena));
}
void clear( Arena& arena, KeyRangeRef const& keys ) {
void clear(Arena& arena, KeyRangeRef const& keys) {
mutations.push_back_deep(arena, MutationRef(MutationRef::ClearRange, keys.begin, keys.end));
write_conflict_ranges.push_back_deep(arena, keys);
}

59
fdbclient/CoordinationInterface.h
View File

@ -36,26 +36,29 @@ constexpr UID WLTOKEN_CLIENTLEADERREG_OPENDATABASE(-1, 3);
constexpr UID WLTOKEN_PROTOCOL_INFO(-1, 10);
struct ClientLeaderRegInterface {
RequestStream< struct GetLeaderRequest > getLeader;
RequestStream< struct OpenDatabaseCoordRequest > openDatabase;
RequestStream<struct GetLeaderRequest> getLeader;
RequestStream<struct OpenDatabaseCoordRequest> openDatabase;
ClientLeaderRegInterface() {}
ClientLeaderRegInterface( NetworkAddress remote );
ClientLeaderRegInterface( INetwork* local );
ClientLeaderRegInterface(NetworkAddress remote);
ClientLeaderRegInterface(INetwork* local);
};
class ClusterConnectionString {
public:
ClusterConnectionString() {}
ClusterConnectionString( std::string const& connectionString );
ClusterConnectionString( vector<NetworkAddress>, Key );
ClusterConnectionString(std::string const& connectionString);
ClusterConnectionString(vector<NetworkAddress>, Key);
vector<NetworkAddress> const& coordinators() const { return coord; }
Key clusterKey() const { return key; }
Key clusterKeyName() const { return keyDesc; } // Returns the "name" or "description" part of the clusterKey (the part before the ':')
Key clusterKeyName() const {
return keyDesc;
} // Returns the "name" or "description" part of the clusterKey (the part before the ':')
std::string toString() const;
static std::string getErrorString(std::string const& source, Error const& e);
private:
void parseKey( std::string const& key );
void parseKey(std::string const& key);
vector<NetworkAddress> coord;
Key key, keyDesc;
@ -75,23 +78,27 @@ public:
// - The ID contains only allowed characters (a-z, A-Z, 0-9)
// - At least one address is specified
// - There is no address present more than once
explicit ClusterConnectionFile( std::string const& path );
explicit ClusterConnectionFile(std::string const& path);
explicit ClusterConnectionFile(ClusterConnectionString const& cs) : cs(cs), setConn(false) {}
explicit ClusterConnectionFile(std::string const& filename, ClusterConnectionString const& contents);
// returns <resolved name, was default file>
static std::pair<std::string, bool> lookupClusterFileName( std::string const& filename );
static std::pair<std::string, bool> lookupClusterFileName(std::string const& filename);
// get a human readable error message describing the error returned from the constructor
static std::string getErrorString( std::pair<std::string, bool> const& resolvedFile, Error const& e );
static std::string getErrorString(std::pair<std::string, bool> const& resolvedFile, Error const& e);
ClusterConnectionString const& getConnectionString() const;
bool writeFile();
void setConnectionString( ClusterConnectionString const& );
std::string const& getFilename() const { ASSERT( filename.size() ); return filename; }
void setConnectionString(ClusterConnectionString const&);
std::string const& getFilename() const {
ASSERT(filename.size());
return filename;
}
bool canGetFilename() const { return filename.size() != 0; }
bool fileContentsUpToDate() const;
bool fileContentsUpToDate(ClusterConnectionString &fileConnectionString) const;
bool fileContentsUpToDate(ClusterConnectionString& fileConnectionString) const;
void notifyConnected();
private:
ClusterConnectionString cs;
std::string filename;
@ -103,7 +110,7 @@ struct LeaderInfo {
UID changeID;
static const uint64_t mask = ~(127ll << 57);
Value serializedInfo;
bool forward; // If true, serializedInfo is a connection string instead!
bool forward; // If true, serializedInfo is a connection string instead!
LeaderInfo() : forward(false) {}
LeaderInfo(UID changeID) : changeID(changeID), forward(false) {}
@ -117,19 +124,23 @@ struct LeaderInfo {
// The first 7 bits of ChangeID represent cluster controller process class fitness, the lower the better
void updateChangeID(ClusterControllerPriorityInfo info) {
changeID = UID( ((uint64_t)info.processClassFitness << 57) | ((uint64_t)info.isExcluded << 60) | ((uint64_t)info.dcFitness << 61) | (changeID.first() & mask), changeID.second() );
changeID = UID(((uint64_t)info.processClassFitness << 57) | ((uint64_t)info.isExcluded << 60) |
((uint64_t)info.dcFitness << 61) | (changeID.first() & mask),
changeID.second());
}
// All but the first 7 bits are used to represent process id
bool equalInternalId(LeaderInfo const& leaderInfo) const {
return ((changeID.first() & mask) == (leaderInfo.changeID.first() & mask)) && changeID.second() == leaderInfo.changeID.second();
return ((changeID.first() & mask) == (leaderInfo.changeID.first() & mask)) &&
changeID.second() == leaderInfo.changeID.second();
}
// Change leader only if
// 1. the candidate has better process class fitness and the candidate is not the leader
// 2. the leader process class fitness becomes worse
bool leaderChangeRequired(LeaderInfo const& candidate) const {
return ((changeID.first() & ~mask) > (candidate.changeID.first() & ~mask) && !equalInternalId(candidate)) || ((changeID.first() & ~mask) < (candidate.changeID.first() & ~mask) && equalInternalId(candidate));
return ((changeID.first() & ~mask) > (candidate.changeID.first() & ~mask) && !equalInternalId(candidate)) ||
((changeID.first() & ~mask) < (candidate.changeID.first() & ~mask) && equalInternalId(candidate));
}
ClusterControllerPriorityInfo getPriorityInfo() const {
@ -150,7 +161,7 @@ struct GetLeaderRequest {
constexpr static FileIdentifier file_identifier = 214727;
Key key;
UID knownLeader;
ReplyPromise< Optional<LeaderInfo> > reply;
ReplyPromise<Optional<LeaderInfo>> reply;
GetLeaderRequest() {}
explicit GetLeaderRequest(Key key, UID kl) : key(key), knownLeader(kl) {}
@ -172,7 +183,7 @@ struct OpenDatabaseCoordRequest {
UID knownClientInfoID;
Key clusterKey;
vector<NetworkAddress> coordinators;
ReplyPromise< CachedSerialization<struct ClientDBInfo> > reply;
ReplyPromise<CachedSerialization<struct ClientDBInfo>> reply;
template <class Ar>
void serialize(Ar& ar) {
@ -182,12 +193,12 @@ struct OpenDatabaseCoordRequest {
class ClientCoordinators {
public:
vector< ClientLeaderRegInterface > clientLeaderServers;
vector<ClientLeaderRegInterface> clientLeaderServers;
Key clusterKey;
Reference<ClusterConnectionFile> ccf;
Reference<ClusterConnectionFile> ccf;
explicit ClientCoordinators( Reference<ClusterConnectionFile> ccf );
explicit ClientCoordinators( Key clusterKey, std::vector<NetworkAddress> coordinators );
explicit ClientCoordinators(Reference<ClusterConnectionFile> ccf);
explicit ClientCoordinators(Key clusterKey, std::vector<NetworkAddress> coordinators);
ClientCoordinators() {}
};

4059
fdbclient/DatabaseBackupAgent.actor.cpp
View File

File diff suppressed because it is too large Load Diff

296
fdbclient/DatabaseConfiguration.cpp
View File

@ -21,8 +21,7 @@
#include "fdbclient/DatabaseConfiguration.h"
#include "fdbclient/SystemData.h"
DatabaseConfiguration::DatabaseConfiguration()
{
DatabaseConfiguration::DatabaseConfiguration() {
resetInternal();
}
@ -46,7 +45,7 @@ void DatabaseConfiguration::resetInternal() {
backupWorkerEnabled = false;
}
void parse( int* i, ValueRef const& v ) {
void parse(int* i, ValueRef const& v) {
// FIXME: Sanity checking
*i = atoi(v.toString().c_str());
}
@ -56,11 +55,11 @@ void parseReplicationPolicy(Reference<IReplicationPolicy>* policy, ValueRef cons
serializeReplicationPolicy(reader, *policy);
}
void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
void parse(std::vector<RegionInfo>* regions, ValueRef const& v) {
try {
StatusObject statusObj = BinaryReader::fromStringRef<StatusObject>(v, IncludeVersion());
regions->clear();
if(statusObj["regions"].type() != json_spirit::array_type) {
if (statusObj["regions"].type() != json_spirit::array_type) {
return;
}
StatusArray regionArray = statusObj["regions"].get_array();
@ -79,51 +78,65 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
s.tryGet("satellite_logs", satInfo.satelliteDesiredTLogCount);
info.satellites.push_back(satInfo);
} else {
if (foundNonSatelliteDatacenter) throw invalid_option();
if (foundNonSatelliteDatacenter)
throw invalid_option();
foundNonSatelliteDatacenter = true;
s.get("id", idStr);
info.dcId = idStr;
s.get("priority", info.priority);
}
}
std::sort(info.satellites.begin(), info.satellites.end(), SatelliteInfo::sort_by_priority() );
if (!foundNonSatelliteDatacenter) throw invalid_option();
std::sort(info.satellites.begin(), info.satellites.end(), SatelliteInfo::sort_by_priority());
if (!foundNonSatelliteDatacenter)
throw invalid_option();
dc.tryGet("satellite_logs", info.satelliteDesiredTLogCount);
std::string satelliteReplication;
if(dc.tryGet("satellite_redundancy_mode", satelliteReplication)) {
if(satelliteReplication == "one_satellite_single") {
if (dc.tryGet("satellite_redundancy_mode", satelliteReplication)) {
if (satelliteReplication == "one_satellite_single") {
info.satelliteTLogReplicationFactor = 1;
info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyOne());
} else if(satelliteReplication == "one_satellite_double") {
} else if (satelliteReplication == "one_satellite_double") {
info.satelliteTLogReplicationFactor = 2;
info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if(satelliteReplication == "one_satellite_triple") {
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "one_satellite_triple") {
info.satelliteTLogReplicationFactor = 3;
info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if(satelliteReplication == "two_satellite_safe") {
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "two_satellite_safe") {
info.satelliteTLogReplicationFactor = 4;
info.satelliteTLogUsableDcs = 2;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"dcid",
Reference<IReplicationPolicy>(new PolicyAcross(
2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogReplicationFactorFallback = 2;
info.satelliteTLogUsableDcsFallback = 1;
info.satelliteTLogWriteAntiQuorumFallback = 0;
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if(satelliteReplication == "two_satellite_fast") {
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if (satelliteReplication == "two_satellite_fast") {
info.satelliteTLogReplicationFactor = 4;
info.satelliteTLogUsableDcs = 2;
info.satelliteTLogWriteAntiQuorum = 2;
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(2,
"dcid",
Reference<IReplicationPolicy>(new PolicyAcross(
2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogReplicationFactorFallback = 2;
info.satelliteTLogUsableDcsFallback = 1;
info.satelliteTLogWriteAntiQuorumFallback = 0;
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(
new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else {
throw invalid_option();
}
@ -136,7 +149,7 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
dc.tryGet("satellite_anti_quorum_fallback", info.satelliteTLogWriteAntiQuorumFallback);
regions->push_back(info);
}
std::sort(regions->begin(), regions->end(), RegionInfo::sort_by_priority() );
std::sort(regions->begin(), regions->end(), RegionInfo::sort_by_priority());
} catch (Error&) {
regions->clear();
return;
@ -144,76 +157,63 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
}
void DatabaseConfiguration::setDefaultReplicationPolicy() {
if(!storagePolicy) {
storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(storageTeamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
if (!storagePolicy) {
storagePolicy = Reference<IReplicationPolicy>(
new PolicyAcross(storageTeamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
if(!tLogPolicy) {
tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(tLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
if (!tLogPolicy) {
tLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(tLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
if(remoteTLogReplicationFactor > 0 && !remoteTLogPolicy) {
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(remoteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
if (remoteTLogReplicationFactor > 0 && !remoteTLogPolicy) {
remoteTLogPolicy = Reference<IReplicationPolicy>(
new PolicyAcross(remoteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
for(auto& r : regions) {
if(r.satelliteTLogReplicationFactor > 0 && !r.satelliteTLogPolicy) {
r.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(r.satelliteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
for (auto& r : regions) {
if (r.satelliteTLogReplicationFactor > 0 && !r.satelliteTLogPolicy) {
r.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(
r.satelliteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
if(r.satelliteTLogReplicationFactorFallback > 0 && !r.satelliteTLogPolicyFallback) {
r.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(r.satelliteTLogReplicationFactorFallback, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
if (r.satelliteTLogReplicationFactorFallback > 0 && !r.satelliteTLogPolicyFallback) {
r.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(
r.satelliteTLogReplicationFactorFallback, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
}
}
bool DatabaseConfiguration::isValid() const {
if( !(initialized &&
tLogWriteAntiQuorum >= 0 &&
tLogWriteAntiQuorum <= tLogReplicationFactor/2 &&
tLogReplicationFactor >= 1 &&
storageTeamSize >= 1 &&
getDesiredCommitProxies() >= 1 &&
getDesiredGrvProxies() >= 1 &&
getDesiredLogs() >= 1 &&
getDesiredResolvers() >= 1 &&
tLogVersion != TLogVersion::UNSET &&
tLogVersion >= TLogVersion::MIN_RECRUITABLE &&
tLogVersion <= TLogVersion::MAX_SUPPORTED &&
tLogDataStoreType != KeyValueStoreType::END &&
tLogSpillType != TLogSpillType::UNSET &&
!(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) &&
storageServerStoreType != KeyValueStoreType::END &&
autoCommitProxyCount >= 1 &&
autoGrvProxyCount >= 1 &&
autoResolverCount >= 1 &&
autoDesiredTLogCount >= 1 &&
storagePolicy &&
tLogPolicy &&
getDesiredRemoteLogs() >= 1 &&
remoteTLogReplicationFactor >= 0 &&
repopulateRegionAntiQuorum >= 0 &&
repopulateRegionAntiQuorum <= 1 &&
usableRegions >= 1 &&
usableRegions <= 2 &&
regions.size() <= 2 &&
( usableRegions == 1 || regions.size() == 2 ) &&
( regions.size() == 0 || regions[0].priority >= 0 ) &&
( regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") ) ) { //We cannot specify regions with three_datacenter replication
if (!(initialized && tLogWriteAntiQuorum >= 0 && tLogWriteAntiQuorum <= tLogReplicationFactor / 2 &&
tLogReplicationFactor >= 1 && storageTeamSize >= 1 && getDesiredCommitProxies() >= 1 &&
getDesiredGrvProxies() >= 1 && getDesiredLogs() >= 1 && getDesiredResolvers() >= 1 &&
tLogVersion != TLogVersion::UNSET && tLogVersion >= TLogVersion::MIN_RECRUITABLE &&
tLogVersion <= TLogVersion::MAX_SUPPORTED && tLogDataStoreType != KeyValueStoreType::END &&
tLogSpillType != TLogSpillType::UNSET &&
!(tLogSpillType == TLogSpillType::REFERENCE && tLogVersion < TLogVersion::V3) &&
storageServerStoreType != KeyValueStoreType::END && autoCommitProxyCount >= 1 && autoGrvProxyCount >= 1 &&
autoResolverCount >= 1 && autoDesiredTLogCount >= 1 && storagePolicy && tLogPolicy &&
getDesiredRemoteLogs() >= 1 && remoteTLogReplicationFactor >= 0 && repopulateRegionAntiQuorum >= 0 &&
repopulateRegionAntiQuorum <= 1 && usableRegions >= 1 && usableRegions <= 2 && regions.size() <= 2 &&
(usableRegions == 1 || regions.size() == 2) && (regions.size() == 0 || regions[0].priority >= 0) &&
(regions.size() == 0 ||
tLogPolicy->info() !=
"dcid^2 x zoneid^2 x 1"))) { // We cannot specify regions with three_datacenter replication
return false;
}
std::set<Key> dcIds;
dcIds.insert(Key());
for(auto& r : regions) {
if( !(!dcIds.count(r.dcId) &&
r.satelliteTLogReplicationFactor >= 0 &&
r.satelliteTLogWriteAntiQuorum >= 0 &&
r.satelliteTLogUsableDcs >= 1 &&
( r.satelliteTLogReplicationFactor == 0 || ( r.satelliteTLogPolicy && r.satellites.size() ) ) &&
( r.satelliteTLogUsableDcsFallback == 0 || ( r.satelliteTLogReplicationFactor > 0 && r.satelliteTLogReplicationFactorFallback > 0 ) ) ) ) {
for (auto& r : regions) {
if (!(!dcIds.count(r.dcId) && r.satelliteTLogReplicationFactor >= 0 && r.satelliteTLogWriteAntiQuorum >= 0 &&
r.satelliteTLogUsableDcs >= 1 &&
(r.satelliteTLogReplicationFactor == 0 || (r.satelliteTLogPolicy && r.satellites.size())) &&
(r.satelliteTLogUsableDcsFallback == 0 ||
(r.satelliteTLogReplicationFactor > 0 && r.satelliteTLogReplicationFactorFallback > 0)))) {
return false;
}
dcIds.insert(r.dcId);
std::set<Key> satelliteDcIds;
satelliteDcIds.insert(Key());
satelliteDcIds.insert(r.dcId);
for(auto& s : r.satellites) {
for (auto& s : r.satellites) {
if (satelliteDcIds.count(s.dcId)) {
return false;
}
@ -264,8 +264,10 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
result["storage_replicas"] = storageTeamSize;
result["log_replicas"] = tLogReplicationFactor;
result["log_anti_quorum"] = tLogWriteAntiQuorum;
if (!noPolicies) result["storage_replication_policy"] = storagePolicy->info();
if (!noPolicies) result["log_replication_policy"] = tLogPolicy->info();
if (!noPolicies)
result["storage_replication_policy"] = storagePolicy->info();
if (!noPolicies)
result["log_replication_policy"] = tLogPolicy->info();
}
if (tLogVersion > TLogVersion::DEFAULT || isOverridden("log_version")) {
@ -306,7 +308,8 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
result["remote_redundancy_mode"] = "remote_triple";
} else if (remoteTLogReplicationFactor > 3) {
result["remote_log_replicas"] = remoteTLogReplicationFactor;
if (noPolicies && remoteTLogPolicy) result["remote_log_policy"] = remoteTLogPolicy->info();
if (noPolicies && remoteTLogPolicy)
result["remote_log_policy"] = remoteTLogPolicy->info();
}
result["usable_regions"] = usableRegions;
@ -355,7 +358,7 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
StatusArray DatabaseConfiguration::getRegionJSON() const {
StatusArray regionArr;
for(auto& r : regions) {
for (auto& r : regions) {
StatusObject regionObj;
StatusArray dcArr;
StatusObject dcObj;
@ -363,38 +366,47 @@ StatusArray DatabaseConfiguration::getRegionJSON() const {
dcObj["priority"] = r.priority;
dcArr.push_back(dcObj);
if(r.satelliteTLogReplicationFactor == 1 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
if (r.satelliteTLogReplicationFactor == 1 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_single";
} else if(r.satelliteTLogReplicationFactor == 2 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
} else if (r.satelliteTLogReplicationFactor == 2 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_double";
} else if(r.satelliteTLogReplicationFactor == 3 && r.satelliteTLogUsableDcs == 1 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
} else if (r.satelliteTLogReplicationFactor == 3 && r.satelliteTLogUsableDcs == 1 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 0) {
regionObj["satellite_redundancy_mode"] = "one_satellite_triple";
} else if(r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 && r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 1 && r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
} else if (r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 &&
r.satelliteTLogWriteAntiQuorum == 0 && r.satelliteTLogUsableDcsFallback == 1 &&
r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
regionObj["satellite_redundancy_mode"] = "two_satellite_safe";
} else if(r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 && r.satelliteTLogWriteAntiQuorum == 2 && r.satelliteTLogUsableDcsFallback == 1 && r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
} else if (r.satelliteTLogReplicationFactor == 4 && r.satelliteTLogUsableDcs == 2 &&
r.satelliteTLogWriteAntiQuorum == 2 && r.satelliteTLogUsableDcsFallback == 1 &&
r.satelliteTLogReplicationFactorFallback == 2 && r.satelliteTLogWriteAntiQuorumFallback == 0) {
regionObj["satellite_redundancy_mode"] = "two_satellite_fast";
} else if(r.satelliteTLogReplicationFactor != 0) {
} else if (r.satelliteTLogReplicationFactor != 0) {
regionObj["satellite_log_replicas"] = r.satelliteTLogReplicationFactor;
regionObj["satellite_usable_dcs"] = r.satelliteTLogUsableDcs;
regionObj["satellite_anti_quorum"] = r.satelliteTLogWriteAntiQuorum;
if(r.satelliteTLogPolicy) regionObj["satellite_log_policy"] = r.satelliteTLogPolicy->info();
if (r.satelliteTLogPolicy)
regionObj["satellite_log_policy"] = r.satelliteTLogPolicy->info();
regionObj["satellite_log_replicas_fallback"] = r.satelliteTLogReplicationFactorFallback;
regionObj["satellite_usable_dcs_fallback"] = r.satelliteTLogUsableDcsFallback;
regionObj["satellite_anti_quorum_fallback"] = r.satelliteTLogWriteAntiQuorumFallback;
if(r.satelliteTLogPolicyFallback) regionObj["satellite_log_policy_fallback"] = r.satelliteTLogPolicyFallback->info();
if (r.satelliteTLogPolicyFallback)
regionObj["satellite_log_policy_fallback"] = r.satelliteTLogPolicyFallback->info();
}
if( r.satelliteDesiredTLogCount != -1 ) {
if (r.satelliteDesiredTLogCount != -1) {
regionObj["satellite_logs"] = r.satelliteDesiredTLogCount;
}
if(r.satellites.size()) {
for(auto& s : r.satellites) {
if (r.satellites.size()) {
for (auto& s : r.satellites) {
StatusObject satObj;
satObj["id"] = s.dcId.toString();
satObj["priority"] = s.priority;
satObj["satellite"] = 1;
if(s.satelliteDesiredTLogCount != -1) {
if (s.satelliteDesiredTLogCount != -1) {
satObj["satellite_logs"] = s.satelliteDesiredTLogCount;
}
@ -413,7 +425,7 @@ std::string DatabaseConfiguration::toString() const {
}
bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
KeyRef ck = key.removePrefix( configKeysPrefix );
KeyRef ck = key.removePrefix(configKeysPrefix);
int type;
if (ck == LiteralStringRef("initialized")) {
@ -428,10 +440,10 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
parse(&desiredTLogCount, value);
} else if (ck == LiteralStringRef("log_replicas")) {
parse(&tLogReplicationFactor, value);
tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor/2);
tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor / 2);
} else if (ck == LiteralStringRef("log_anti_quorum")) {
parse(&tLogWriteAntiQuorum, value);
if(tLogReplicationFactor > 0) {
if (tLogReplicationFactor > 0) {
tLogWriteAntiQuorum = std::min(tLogWriteAntiQuorum, tLogReplicationFactor / 2);
}
} else if (ck == LiteralStringRef("storage_replicas")) {
@ -445,11 +457,11 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
parse((&type), value);
tLogDataStoreType = (KeyValueStoreType::StoreType)type;
// TODO: Remove this once Redwood works as a log engine
if(tLogDataStoreType == KeyValueStoreType::SSD_REDWOOD_V1) {
if (tLogDataStoreType == KeyValueStoreType::SSD_REDWOOD_V1) {
tLogDataStoreType = KeyValueStoreType::SSD_BTREE_V2;
}
// TODO: Remove this once memroy radix tree works as a log engine
if(tLogDataStoreType == KeyValueStoreType::MEMORY_RADIXTREE) {
if (tLogDataStoreType == KeyValueStoreType::MEMORY_RADIXTREE) {
tLogDataStoreType = KeyValueStoreType::SSD_BTREE_V2;
}
} else if (ck == LiteralStringRef("log_spill")) {
@ -490,22 +502,22 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
} else {
return false;
}
return true; // All of the above options currently require recovery to take effect
return true; // All of the above options currently require recovery to take effect
}
inline static KeyValueRef * lower_bound( VectorRef<KeyValueRef> & config, KeyRef const& key ) {
return std::lower_bound( config.begin(), config.end(), KeyValueRef(key, ValueRef()), KeyValueRef::OrderByKey() );
inline static KeyValueRef* lower_bound(VectorRef<KeyValueRef>& config, KeyRef const& key) {
return std::lower_bound(config.begin(), config.end(), KeyValueRef(key, ValueRef()), KeyValueRef::OrderByKey());
}
inline static KeyValueRef const* lower_bound( VectorRef<KeyValueRef> const& config, KeyRef const& key ) {
return lower_bound( const_cast<VectorRef<KeyValueRef> &>(config), key );
inline static KeyValueRef const* lower_bound(VectorRef<KeyValueRef> const& config, KeyRef const& key) {
return lower_bound(const_cast<VectorRef<KeyValueRef>&>(config), key);
}
void DatabaseConfiguration::applyMutation( MutationRef m ) {
if( m.type == MutationRef::SetValue && m.param1.startsWith(configKeysPrefix) ) {
void DatabaseConfiguration::applyMutation(MutationRef m) {
if (m.type == MutationRef::SetValue && m.param1.startsWith(configKeysPrefix)) {
set(m.param1, m.param2);
} else if( m.type == MutationRef::ClearRange ) {
} else if (m.type == MutationRef::ClearRange) {
KeyRangeRef range(m.param1, m.param2);
if( range.intersects( configKeys ) ) {
if (range.intersects(configKeys)) {
clear(range & configKeys);
}
}
@ -513,78 +525,90 @@ void DatabaseConfiguration::applyMutation( MutationRef m ) {
bool DatabaseConfiguration::set(KeyRef key, ValueRef value) {
makeConfigurationMutable();
mutableConfiguration.get()[ key.toString() ] = value.toString();
return setInternal(key,value);
mutableConfiguration.get()[key.toString()] = value.toString();
return setInternal(key, value);
}
bool DatabaseConfiguration::clear( KeyRangeRef keys ) {
bool DatabaseConfiguration::clear(KeyRangeRef keys) {
makeConfigurationMutable();
auto& mc = mutableConfiguration.get();
mc.erase( mc.lower_bound( keys.begin.toString() ), mc.lower_bound( keys.end.toString() ) );
mc.erase(mc.lower_bound(keys.begin.toString()), mc.lower_bound(keys.end.toString()));
// FIXME: More efficient
bool wasValid = isValid();
resetInternal();
for(auto c = mc.begin(); c != mc.end(); ++c)
for (auto c = mc.begin(); c != mc.end(); ++c)
setInternal(c->first, c->second);
return wasValid && !isValid();
}
Optional<ValueRef> DatabaseConfiguration::get( KeyRef key ) const {
Optional<ValueRef> DatabaseConfiguration::get(KeyRef key) const {
if (mutableConfiguration.present()) {
auto i = mutableConfiguration.get().find(key.toString());
if (i == mutableConfiguration.get().end()) return Optional<ValueRef>();
if (i == mutableConfiguration.get().end())
return Optional<ValueRef>();
return ValueRef(i->second);
} else {
auto i = lower_bound(rawConfiguration, key);
if (i == rawConfiguration.end() || i->key != key) return Optional<ValueRef>();
if (i == rawConfiguration.end() || i->key != key)
return Optional<ValueRef>();
return i->value;
}
}
bool DatabaseConfiguration::isExcludedServer( NetworkAddressList a ) const {
return get( encodeExcludedServersKey( AddressExclusion(a.address.ip, a.address.port) ) ).present() ||
get( encodeExcludedServersKey( AddressExclusion(a.address.ip) ) ).present() ||
get( encodeFailedServersKey( AddressExclusion(a.address.ip, a.address.port) ) ).present() ||
get( encodeFailedServersKey( AddressExclusion(a.address.ip) ) ).present() ||
( a.secondaryAddress.present() && (
get( encodeExcludedServersKey( AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port) ) ).present() ||
get( encodeExcludedServersKey( AddressExclusion(a.secondaryAddress.get().ip) ) ).present() ||
get( encodeFailedServersKey( AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port) ) ).present() ||
get( encodeFailedServersKey( AddressExclusion(a.secondaryAddress.get().ip) ) ).present() ) );
bool DatabaseConfiguration::isExcludedServer(NetworkAddressList a) const {
return get(encodeExcludedServersKey(AddressExclusion(a.address.ip, a.address.port))).present() ||
get(encodeExcludedServersKey(AddressExclusion(a.address.ip))).present() ||
get(encodeFailedServersKey(AddressExclusion(a.address.ip, a.address.port))).present() ||
get(encodeFailedServersKey(AddressExclusion(a.address.ip))).present() ||
(a.secondaryAddress.present() &&
(get(encodeExcludedServersKey(AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port)))
.present() ||
get(encodeExcludedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present() ||
get(encodeFailedServersKey(AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port)))
.present() ||
get(encodeFailedServersKey(AddressExclusion(a.secondaryAddress.get().ip))).present()));
}
std::set<AddressExclusion> DatabaseConfiguration::getExcludedServers() const {
const_cast<DatabaseConfiguration*>(this)->makeConfigurationImmutable();
std::set<AddressExclusion> addrs;
for( auto i = lower_bound(rawConfiguration, excludedServersKeys.begin); i != rawConfiguration.end() && i->key < excludedServersKeys.end; ++i ) {
AddressExclusion a = decodeExcludedServersKey( i->key );
if (a.isValid()) addrs.insert(a);
for (auto i = lower_bound(rawConfiguration, excludedServersKeys.begin);
i != rawConfiguration.end() && i->key < excludedServersKeys.end;
++i) {
AddressExclusion a = decodeExcludedServersKey(i->key);
if (a.isValid())
addrs.insert(a);
}
for( auto i = lower_bound(rawConfiguration, failedServersKeys.begin); i != rawConfiguration.end() && i->key < failedServersKeys.end; ++i ) {
AddressExclusion a = decodeFailedServersKey( i->key );
if (a.isValid()) addrs.insert(a);
for (auto i = lower_bound(rawConfiguration, failedServersKeys.begin);
i != rawConfiguration.end() && i->key < failedServersKeys.end;
++i) {
AddressExclusion a = decodeFailedServersKey(i->key);
if (a.isValid())
addrs.insert(a);
}
return addrs;
}
void DatabaseConfiguration::makeConfigurationMutable() {
if (mutableConfiguration.present()) return;
mutableConfiguration = std::map<std::string,std::string>();
if (mutableConfiguration.present())
return;
mutableConfiguration = std::map<std::string, std::string>();
auto& mc = mutableConfiguration.get();
for(auto r = rawConfiguration.begin(); r != rawConfiguration.end(); ++r)
mc[ r->key.toString() ] = r->value.toString();
for (auto r = rawConfiguration.begin(); r != rawConfiguration.end(); ++r)
mc[r->key.toString()] = r->value.toString();
rawConfiguration = Standalone<VectorRef<KeyValueRef>>();
}
void DatabaseConfiguration::makeConfigurationImmutable() {
if (!mutableConfiguration.present()) return;
auto & mc = mutableConfiguration.get();
if (!mutableConfiguration.present())
return;
auto& mc = mutableConfiguration.get();
rawConfiguration = Standalone<VectorRef<KeyValueRef>>();
rawConfiguration.resize( rawConfiguration.arena(), mc.size() );
rawConfiguration.resize(rawConfiguration.arena(), mc.size());
int i = 0;
for(auto r = mc.begin(); r != mc.end(); ++r)
rawConfiguration[i++] = KeyValueRef( rawConfiguration.arena(), KeyValueRef( r->first, r->second ) );
mutableConfiguration = Optional<std::map<std::string,std::string>>();
for (auto r = mc.begin(); r != mc.end(); ++r)
rawConfiguration[i++] = KeyValueRef(rawConfiguration.arena(), KeyValueRef(r->first, r->second));
mutableConfiguration = Optional<std::map<std::string, std::string>>();
}
void DatabaseConfiguration::fromKeyValues(Standalone<VectorRef<KeyValueRef>> rawConfig) {

55
fdbclient/DatabaseConfiguration.h
View File

@ -37,7 +37,7 @@ struct SatelliteInfo {
SatelliteInfo() : priority(0) {}
struct sort_by_priority {
bool operator ()(SatelliteInfo const&a, SatelliteInfo const& b) const { return a.priority > b.priority; }
bool operator()(SatelliteInfo const& a, SatelliteInfo const& b) const { return a.priority > b.priority; }
};
template <class Ar>
@ -84,10 +84,19 @@ struct RegionInfo {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, dcId, priority, satelliteTLogPolicy, satelliteDesiredTLogCount, satelliteTLogReplicationFactor,
satelliteTLogWriteAntiQuorum, satelliteTLogUsableDcs, satelliteTLogPolicyFallback,
satelliteTLogReplicationFactorFallback, satelliteTLogWriteAntiQuorumFallback,
satelliteTLogUsableDcsFallback, satellites);
serializer(ar,
dcId,
priority,
satelliteTLogPolicy,
satelliteDesiredTLogCount,
satelliteTLogReplicationFactor,
satelliteTLogWriteAntiQuorum,
satelliteTLogUsableDcs,
satelliteTLogPolicyFallback,
satelliteTLogReplicationFactorFallback,
satelliteTLogWriteAntiQuorumFallback,
satelliteTLogUsableDcsFallback,
satellites);
}
};
@ -165,8 +174,8 @@ struct DatabaseConfiguration {
worstSatellite =
std::min(worstSatellite, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum);
if (r.satelliteTLogUsableDcsFallback > 0) {
worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactorFallback -
r.satelliteTLogWriteAntiQuorumFallback);
worstSatellite = std::min(
worstSatellite, r.satelliteTLogReplicationFactorFallback - r.satelliteTLogWriteAntiQuorumFallback);
}
}
if (usableRegions > 1 && fullyReplicatedRegions > 1 && worstSatellite > 0 &&
@ -215,7 +224,7 @@ struct DatabaseConfiguration {
bool backupWorkerEnabled;
// Data centers
int32_t usableRegions; // Number of regions which have a replica of the database.
int32_t usableRegions; // Number of regions which have a replica of the database.
int32_t repopulateRegionAntiQuorum;
std::vector<RegionInfo> regions;
@ -224,36 +233,44 @@ struct DatabaseConfiguration {
std::set<AddressExclusion> getExcludedServers() const;
int32_t getDesiredCommitProxies() const {
if (commitProxyCount == -1) return autoCommitProxyCount;
if (commitProxyCount == -1)
return autoCommitProxyCount;
return commitProxyCount;
}
int32_t getDesiredGrvProxies() const {
if (grvProxyCount == -1) return autoGrvProxyCount;
if (grvProxyCount == -1)
return autoGrvProxyCount;
return grvProxyCount;
}
int32_t getDesiredResolvers() const {
if (resolverCount == -1) return autoResolverCount;
if (resolverCount == -1)
return autoResolverCount;
return resolverCount;
}
int32_t getDesiredLogs() const {
if (desiredTLogCount == -1) return autoDesiredTLogCount;
if (desiredTLogCount == -1)
return autoDesiredTLogCount;
return desiredTLogCount;
}
int32_t getDesiredRemoteLogs() const {
if (remoteDesiredTLogCount == -1) return getDesiredLogs();
if (remoteDesiredTLogCount == -1)
return getDesiredLogs();
return remoteDesiredTLogCount;
}
int32_t getDesiredSatelliteLogs(Optional<Key> dcId) const {
auto desired = getRegion(dcId).satelliteDesiredTLogCount;
if (desired == -1) return autoDesiredTLogCount;
if (desired == -1)
return autoDesiredTLogCount;
return desired;
}
int32_t getRemoteTLogReplicationFactor() const {
if (remoteTLogReplicationFactor == 0) return tLogReplicationFactor;
if (remoteTLogReplicationFactor == 0)
return tLogReplicationFactor;
return remoteTLogReplicationFactor;
}
Reference<IReplicationPolicy> getRemoteTLogPolicy() const {
if (remoteTLogReplicationFactor == 0) return tLogPolicy;
if (remoteTLogReplicationFactor == 0)
return tLogPolicy;
return remoteTLogPolicy;
}
@ -266,10 +283,12 @@ struct DatabaseConfiguration {
template <class Ar>
void serialize(Ar& ar) {
if (!ar.isDeserializing) makeConfigurationImmutable();
if (!ar.isDeserializing)
makeConfigurationImmutable();
serializer(ar, rawConfiguration);
if (ar.isDeserializing) {
for (auto c = rawConfiguration.begin(); c != rawConfiguration.end(); ++c) setInternal(c->key, c->value);
for (auto c = rawConfiguration.begin(); c != rawConfiguration.end(); ++c)
setInternal(c->key, c->value);
setDefaultReplicationPolicy();
}
}

143
fdbclient/DatabaseContext.h
View File

@ -41,33 +41,31 @@
class StorageServerInfo : public ReferencedInterface<StorageServerInterface> {
public:
static Reference<StorageServerInfo> getInterface( DatabaseContext *cx, StorageServerInterface const& interf, LocalityData const& locality );
static Reference<StorageServerInfo> getInterface(DatabaseContext* cx,
StorageServerInterface const& interf,
LocalityData const& locality);
void notifyContextDestroyed();
~StorageServerInfo() override;
private:
DatabaseContext *cx;
StorageServerInfo( DatabaseContext *cx, StorageServerInterface const& interf, LocalityData const& locality ) : cx(cx), ReferencedInterface<StorageServerInterface>(interf, locality) {}
DatabaseContext* cx;
StorageServerInfo(DatabaseContext* cx, StorageServerInterface const& interf, LocalityData const& locality)
: cx(cx), ReferencedInterface<StorageServerInterface>(interf, locality) {}
};
struct LocationInfo : MultiInterface<ReferencedInterface<StorageServerInterface>>, FastAllocated<LocationInfo> {
using Locations = MultiInterface<ReferencedInterface<StorageServerInterface>>;
explicit LocationInfo(const std::vector<Reference<ReferencedInterface<StorageServerInterface>>>& v)
: Locations(v)
{}
: Locations(v) {}
LocationInfo(const std::vector<Reference<ReferencedInterface<StorageServerInterface>>>& v, bool hasCaches)
: Locations(v)
, hasCaches(hasCaches)
{}
: Locations(v), hasCaches(hasCaches) {}
LocationInfo(const LocationInfo&) = delete;
LocationInfo(LocationInfo&&) = delete;
LocationInfo& operator=(const LocationInfo&) = delete;
LocationInfo& operator=(LocationInfo&&) = delete;
bool hasCaches = false;
Reference<Locations> locations() {
return Reference<Locations>::addRef(this);
}
Reference<Locations> locations() { return Reference<Locations>::addRef(this); }
};
using CommitProxyInfo = ModelInterface<CommitProxyInterface>;
@ -85,9 +83,9 @@ private:
public:
ClientTagThrottleData(ClientTagThrottleLimits const& limits)
: tpsRate(limits.tpsRate), expiration(limits.expiration), lastCheck(now()), smoothRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW),
smoothReleased(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW)
{
: tpsRate(limits.tpsRate), expiration(limits.expiration), lastCheck(now()),
smoothRate(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW),
smoothReleased(CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW) {
ASSERT(tpsRate >= 0);
smoothRate.reset(tpsRate);
}
@ -96,44 +94,36 @@ public:
ASSERT(limits.tpsRate >= 0);
this->tpsRate = limits.tpsRate;
if(!rateSet || expired()) {
if (!rateSet || expired()) {
rateSet = true;
smoothRate.reset(limits.tpsRate);
}
else {
} else {
smoothRate.setTotal(limits.tpsRate);
}
expiration = limits.expiration;
}
void addReleased(int released) {
smoothReleased.addDelta(released);
}
void addReleased(int released) { smoothReleased.addDelta(released); }
bool expired() {
return expiration <= now();
}
bool expired() { return expiration <= now(); }
void updateChecked() {
lastCheck = now();
}
void updateChecked() { lastCheck = now(); }
bool canRecheck() {
return lastCheck < now() - CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL;
}
bool canRecheck() { return lastCheck < now() - CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL; }
double throttleDuration() {
if(expiration <= now()) {
if (expiration <= now()) {
return 0.0;
}
double capacity = (smoothRate.smoothTotal() - smoothReleased.smoothRate()) * CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW;
if(capacity >= 1) {
double capacity =
(smoothRate.smoothTotal() - smoothReleased.smoothRate()) * CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW;
if (capacity >= 1) {
return 0.0;
}
if(tpsRate == 0) {
if (tpsRate == 0) {
return std::max(0.0, expiration - now());
}
@ -141,7 +131,7 @@ public:
}
};
class WatchMetadata: public ReferenceCounted<WatchMetadata> {
class WatchMetadata : public ReferenceCounted<WatchMetadata> {
public:
Key key;
Optional<Value> value;
@ -163,20 +153,38 @@ public:
}
// For internal (fdbserver) use only
static Database create(Reference<AsyncVar<ClientDBInfo>> clientInfo, Future<Void> clientInfoMonitor,
LocalityData clientLocality, bool enableLocalityLoadBalance,
TaskPriority taskID = TaskPriority::DefaultEndpoint, bool lockAware = false,
int apiVersion = Database::API_VERSION_LATEST, bool switchable = false);
static Database create(Reference<AsyncVar<ClientDBInfo>> clientInfo,
Future<Void> clientInfoMonitor,
LocalityData clientLocality,
bool enableLocalityLoadBalance,
TaskPriority taskID = TaskPriority::DefaultEndpoint,
bool lockAware = false,
int apiVersion = Database::API_VERSION_LATEST,
bool switchable = false);
~DatabaseContext();
Database clone() const { return Database(new DatabaseContext( connectionFile, clientInfo, clientInfoMonitor, taskID, clientLocality, enableLocalityLoadBalance, lockAware, internal, apiVersion, switchable )); }
Database clone() const {
return Database(new DatabaseContext(connectionFile,
clientInfo,
clientInfoMonitor,
taskID,
clientLocality,
enableLocalityLoadBalance,
lockAware,
internal,
apiVersion,
switchable));
}
std::pair<KeyRange, Reference<LocationInfo>> getCachedLocation( const KeyRef&, bool isBackward = false );
bool getCachedLocations( const KeyRangeRef&, vector<std::pair<KeyRange,Reference<LocationInfo>>>&, int limit, bool reverse );
Reference<LocationInfo> setCachedLocation( const KeyRangeRef&, const vector<struct StorageServerInterface>& );
void invalidateCache( const KeyRef&, bool isBackward = false );
void invalidateCache( const KeyRangeRef& );
std::pair<KeyRange, Reference<LocationInfo>> getCachedLocation(const KeyRef&, bool isBackward = false);
bool getCachedLocations(const KeyRangeRef&,
vector<std::pair<KeyRange, Reference<LocationInfo>>>&,
int limit,
bool reverse);
Reference<LocationInfo> setCachedLocation(const KeyRangeRef&, const vector<struct StorageServerInterface>&);
void invalidateCache(const KeyRef&, bool isBackward = false);
void invalidateCache(const KeyRangeRef&);
bool sampleReadTags() const;
bool sampleOnCost(uint64_t cost) const;
@ -198,24 +206,23 @@ public:
void deleteWatchMetadata(KeyRef key);
void clearWatchMetadata();
void setOption( FDBDatabaseOptions::Option option, Optional<StringRef> value );
void setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value);
Error deferredError;
bool lockAware;
bool isError() {
return deferredError.code() != invalid_error_code;
}
bool isError() { return deferredError.code() != invalid_error_code; }
void checkDeferredError() {
if(isError()) {
if (isError()) {
throw deferredError;
}
}
int apiVersionAtLeast(int minVersion) { return apiVersion < 0 || apiVersion >= minVersion; }
Future<Void> onConnected(); // Returns after a majority of coordination servers are available and have reported a leader. The cluster file therefore is valid, but the database might be unavailable.
Future<Void> onConnected(); // Returns after a majority of coordination servers are available and have reported a
// leader. The cluster file therefore is valid, but the database might be unavailable.
Reference<ClusterConnectionFile> getConnectionFile();
// Switch the database to use the new connection file, and recreate all pending watches for committed transactions.
@ -228,20 +235,28 @@ public:
Future<Void> switchConnectionFile(Reference<ClusterConnectionFile> standby);
Future<Void> connectionFileChanged();
bool switchable = false;
// Management API, Attempt to kill or suspend a process, return 1 for request sent out, 0 for failure
Future<int64_t> rebootWorker(StringRef address, bool check = false, int duration = 0);
// Management API, force the database to recover into DCID, causing the database to lose the most recently committed mutations
// Management API, force the database to recover into DCID, causing the database to lose the most recently committed
// mutations
Future<Void> forceRecoveryWithDataLoss(StringRef dcId);
// Management API, create snapshot
Future<Void> createSnapshot(StringRef uid, StringRef snapshot_command);
//private:
explicit DatabaseContext( Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile, Reference<AsyncVar<ClientDBInfo>> clientDBInfo,
Future<Void> clientInfoMonitor, TaskPriority taskID, LocalityData const& clientLocality,
bool enableLocalityLoadBalance, bool lockAware, bool internal = true, int apiVersion = Database::API_VERSION_LATEST, bool switchable = false );
// private:
explicit DatabaseContext(Reference<AsyncVar<Reference<ClusterConnectionFile>>> connectionFile,
Reference<AsyncVar<ClientDBInfo>> clientDBInfo,
Future<Void> clientInfoMonitor,
TaskPriority taskID,
LocalityData const& clientLocality,
bool enableLocalityLoadBalance,
bool lockAware,
bool internal = true,
int apiVersion = Database::API_VERSION_LATEST,
bool switchable = false);
explicit DatabaseContext( const Error &err );
explicit DatabaseContext(const Error& err);
void expireThrottles();
@ -284,8 +299,8 @@ public:
// Client status updater
struct ClientStatusUpdater {
std::vector< std::pair<std::string, BinaryWriter> > inStatusQ;
std::vector< std::pair<std::string, BinaryWriter> > outStatusQ;
std::vector<std::pair<std::string, BinaryWriter>> inStatusQ;
std::vector<std::pair<std::string, BinaryWriter>> outStatusQ;
Future<Void> actor;
};
ClientStatusUpdater clientStatusUpdater;
@ -294,7 +309,7 @@ public:
int locationCacheSize;
CoalescedKeyRangeMap<Reference<LocationInfo>> locationCache;
std::map< UID, StorageServerInfo* > server_interf;
std::map<UID, StorageServerInfo*> server_interf;
UID dbId;
bool internal; // Only contexts created through the C client and fdbcli are non-internal
@ -343,7 +358,8 @@ public:
Counter transactionsThrottled;
Counter transactionsExpensiveClearCostEstCount;
ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit, bytesPerCommit;
ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit,
bytesPerCommit;
int outstandingWatches;
int maxOutstandingWatches;
@ -384,8 +400,9 @@ public:
AsyncTrigger updateCache;
std::vector<std::unique_ptr<SpecialKeyRangeReadImpl>> specialKeySpaceModules;
std::unique_ptr<SpecialKeySpace> specialKeySpace;
void registerSpecialKeySpaceModule(SpecialKeySpace::MODULE module, SpecialKeySpace::IMPLTYPE type,
std::unique_ptr<SpecialKeyRangeReadImpl> &&impl);
void registerSpecialKeySpaceModule(SpecialKeySpace::MODULE module,
SpecialKeySpace::IMPLTYPE type,
std::unique_ptr<SpecialKeyRangeReadImpl>&& impl);
static bool debugUseTags;
static const std::vector<std::string> debugTransactionTagChoices;

13
fdbclient/EventTypes.actor.h
View File

@ -20,19 +20,20 @@
#pragma once
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version.
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_EVENTTYPES_ACTOR_G_H)
#define FDBCLIENT_EVENTTYPES_ACTOR_G_H
#include "fdbclient/EventTypes.actor.g.h"
#define FDBCLIENT_EVENTTYPES_ACTOR_G_H
#include "fdbclient/EventTypes.actor.g.h"
#elif !defined(FDBCLIENT_EVENTTYPES_ACTOR_H)
#define FDBCLIENT_EVENTTYPESS_ACTOR_H
#define FDBCLIENT_EVENTTYPESS_ACTOR_H
#include "flow/flow.h"
#include "flow/TDMetric.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
DESCR struct GetValueComplete {
int64_t latency; //ns
int64_t latency; // ns
};
#include "flow/unactorcompiler.h"

43
fdbclient/FDBOptions.h
View File

@ -38,17 +38,24 @@ struct FDBOptionInfo {
bool persistent;
// If non-negative, this specifies the code for the transaction option that this option is the default value for.
// Options that have a defaultFor will only retain the value from time they were most recently set (i.e. there can be no cumulative effects from calling multiple times).
// Options that have a defaultFor will only retain the value from time they were most recently set (i.e. there can
// be no cumulative effects from calling multiple times).
int defaultFor;
FDBOptionInfo(std::string name, std::string comment, std::string parameterComment, bool hasParameter, bool hidden, bool persistent, int defaultFor)
: name(name), comment(comment), parameterComment(parameterComment), hasParameter(hasParameter), hidden(hidden), persistent(persistent),
defaultFor(defaultFor) { }
FDBOptionInfo(std::string name,
std::string comment,
std::string parameterComment,
bool hasParameter,
bool hidden,
bool persistent,
int defaultFor)
: name(name), comment(comment), parameterComment(parameterComment), hasParameter(hasParameter), hidden(hidden),
persistent(persistent), defaultFor(defaultFor) {}
FDBOptionInfo() { }
FDBOptionInfo() {}
};
template<class T>
template <class T>
class FDBOptionInfoMap {
private:
std::map<typename T::Option, FDBOptionInfo> optionInfo;
@ -56,24 +63,24 @@ private:
public:
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator begin() const { return optionInfo.begin(); }
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator end() const { return optionInfo.end(); }
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator find(const typename T::Option& key) const { return optionInfo.find(key); }
void insert(const typename T::Option& key, FDBOptionInfo info) {
optionInfo[key] = info;
typename std::map<typename T::Option, FDBOptionInfo>::const_iterator find(const typename T::Option& key) const {
return optionInfo.find(key);
}
FDBOptionInfo const& getMustExist(const typename T::Option& key) const {
void insert(const typename T::Option& key, FDBOptionInfo info) { optionInfo[key] = info; }
FDBOptionInfo const& getMustExist(const typename T::Option& key) const {
auto itr = optionInfo.find(key);
ASSERT(itr != optionInfo.end());
return itr->second;
return itr->second;
}
FDBOptionInfoMap() { T::init(); }
};
// An ordered list of options where each option is represented only once. Subsequent insertions will remove the option from its
// original location and add it to the end with the new value.
template<class T>
// An ordered list of options where each option is represented only once. Subsequent insertions will remove the option
// from its original location and add it to the end with the new value.
template <class T>
class UniqueOrderedOptionList {
public:
typedef std::list<std::pair<typename T::Option, Optional<Standalone<StringRef>>>> OptionList;
@ -85,7 +92,7 @@ private:
public:
void addOption(typename T::Option option, Optional<Standalone<StringRef>> value) {
auto itr = optionsIndexMap.find(option);
if(itr != optionsIndexMap.end()) {
if (itr != optionsIndexMap.end()) {
options.erase(itr->second);
}
options.push_back(std::make_pair(option, value));
@ -96,6 +103,8 @@ public:
typename OptionList::const_iterator end() const { return options.cend(); }
};
#define ADD_OPTION_INFO( type, var, name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor ) type::optionInfo.insert(var, FDBOptionInfo(name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor));
#define ADD_OPTION_INFO(type, var, name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor) \
type::optionInfo.insert( \
var, FDBOptionInfo(name, comment, parameterComment, hasParameter, hidden, persistent, defaultFor));
#endif

602
fdbclient/FDBTypes.h
View File

File diff suppressed because it is too large Load Diff

7291
fdbclient/FileBackupAgent.actor.cpp
View File

File diff suppressed because it is too large Load Diff

26
fdbclient/GrvProxyInterface.h
View File

@ -23,8 +23,8 @@
#define FDBCLIENT_GRVPROXYINTERFACE_H
#pragma once
// GrvProxy is proxy primarily specializing on serving GetReadVersion. It also serves health metrics since it communicates
// with RateKeeper to gather health information of the cluster.
// GrvProxy is proxy primarily specializing on serving GetReadVersion. It also serves health metrics since it
// communicates with RateKeeper to gather health information of the cluster.
struct GrvProxyInterface {
constexpr static FileIdentifier file_identifier = 8743216;
enum { LocationAwareLoadBalance = 1 };
@ -33,23 +33,28 @@ struct GrvProxyInterface {
Optional<Key> processId;
bool provisional;
RequestStream< struct GetReadVersionRequest > getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported committed (by a commit response) when this request was sent
// (at some point between when this request is sent and when its response is received, the latest version reported committed)
RequestStream<struct GetReadVersionRequest>
getConsistentReadVersion; // Returns a version which (1) is committed, and (2) is >= the latest version reported
// committed (by a commit response) when this request was sent
// (at some point between when this request is sent and when its response is received, the latest version reported
// committed)
RequestStream<ReplyPromise<Void>> waitFailure; // reports heartbeat to master.
RequestStream< struct GetHealthMetricsRequest > getHealthMetrics;
RequestStream<struct GetHealthMetricsRequest> getHealthMetrics;
UID id() const { return getConsistentReadVersion.getEndpoint().token; }
std::string toString() const { return id().shortString(); }
bool operator == (GrvProxyInterface const& r) const { return id() == r.id(); }
bool operator != (GrvProxyInterface const& r) const { return id() != r.id(); }
bool operator==(GrvProxyInterface const& r) const { return id() == r.id(); }
bool operator!=(GrvProxyInterface const& r) const { return id() != r.id(); }
NetworkAddress address() const { return getConsistentReadVersion.getEndpoint().getPrimaryAddress(); }
template <class Archive>
void serialize(Archive& ar) {
serializer(ar, processId, provisional, getConsistentReadVersion);
if( Archive::isDeserializing ) {
waitFailure = RequestStream<ReplyPromise<Void>>( getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(1) );
getHealthMetrics = RequestStream< struct GetHealthMetricsRequest >( getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(2) );
if (Archive::isDeserializing) {
waitFailure =
RequestStream<ReplyPromise<Void>>(getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(1));
getHealthMetrics = RequestStream<struct GetHealthMetricsRequest>(
getConsistentReadVersion.getEndpoint().getAdjustedEndpoint(2));
}
}
@ -62,5 +67,4 @@ struct GrvProxyInterface {
}
};
#endif // FDBCLIENT_GRVPROXYINTERFACE_H

875
fdbclient/HTTP.actor.cpp
View File

@ -26,421 +26,466 @@
namespace HTTP {
std::string urlEncode(const std::string &s) {
std::string o;
o.reserve(s.size() * 3);
char buf[4];
for(auto c : s)
if(std::isalnum(c) || c == '?' || c == '/' || c == '-' || c == '_' || c == '.' || c == ',' || c == ':')
o.append(&c, 1);
else {
sprintf(buf, "%%%.02X", c);
o.append(buf);
}
return o;
}
bool Response::verifyMD5(bool fail_if_header_missing, Optional<std::string> content_sum) {
auto i = headers.find("Content-MD5");
if(i != headers.end()) {
// If a content sum is not provided, calculate one from the response content
if(!content_sum.present()) {
MD5_CTX sum;
::MD5_Init(&sum);
::MD5_Update(&sum, content.data(), content.size());
std::string sumBytes;
sumBytes.resize(16);
::MD5_Final((unsigned char *)sumBytes.data(), &sum);
std::string sumStr = base64::encoder::from_string(sumBytes);
sumStr.resize(sumStr.size() - 1);
content_sum = sumStr;
}
return i->second == content_sum.get();
}
return !fail_if_header_missing;
}
std::string Response::toString() {
std::string r = format("Response Code: %d\n", code);
r += format("Response ContentLen: %lld\n", contentLen);
for(auto h : headers)
r += format("Reponse Header: %s: %s\n", h.first.c_str(), h.second.c_str());
r.append("-- RESPONSE CONTENT--\n");
r.append(content);
r.append("\n--------\n");
return r;
}
PacketBuffer * writeRequestHeader(std::string const &verb, std::string const &resource, HTTP::Headers const &headers, PacketBuffer *dest) {
PacketWriter writer(dest, nullptr, Unversioned());
writer.serializeBytes(verb);
writer.serializeBytes(" ", 1);
writer.serializeBytes(resource);
writer.serializeBytes(LiteralStringRef(" HTTP/1.1\r\n"));
for(auto h : headers) {
writer.serializeBytes(h.first);
writer.serializeBytes(LiteralStringRef(": "));
writer.serializeBytes(h.second);
writer.serializeBytes(LiteralStringRef("\r\n"));
}
writer.serializeBytes(LiteralStringRef("\r\n"));
return writer.finish();
}
// Read at least 1 bytes from conn and up to maxlen in a single read, append read data into *buf
// Returns the number of bytes read.
ACTOR Future<int> read_into_string(Reference<IConnection> conn, std::string *buf, int maxlen) {
loop {
// Read into buffer
int originalSize = buf->size();
// TODO: resize is zero-initializing the space we're about to overwrite, so do something else, which probably means
// not using a string for this buffer
buf->resize(originalSize + maxlen);
uint8_t *wptr = (uint8_t *)buf->data() + originalSize;
int len = conn->read(wptr, wptr + maxlen);
buf->resize(originalSize + len);
// Make sure data was actually read, it's possible for there to be none.
if(len > 0)
return len;
// Wait for connection to have something to read
wait(conn->onReadable());
wait( delay( 0, TaskPriority::ReadSocket ) );
}
}
// Returns the position of delim within buf, relative to pos. If delim is not found, continues to read from conn until
// either it is found or the connection ends, at which point connection_failed is thrown and buf contains
// everything that was read up to that point.
ACTOR Future<size_t> read_delimited_into_string(Reference<IConnection> conn, const char *delim, std::string *buf, size_t pos) {
state size_t sPos = pos;
state int lookBack = strlen(delim) - 1;
ASSERT(lookBack >= 0);
loop {
size_t endPos = buf->find(delim, sPos);
if(endPos != std::string::npos)
return endPos - pos;
// Next search will start at the current end of the buffer - delim size + 1
if(sPos >= lookBack)
sPos -= lookBack;
wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
}
}
// Reads from conn (as needed) until there are at least len bytes starting at pos in buf
ACTOR Future<Void> read_fixed_into_string(Reference<IConnection> conn, int len, std::string *buf, size_t pos) {
state int stop_size = pos + len;
while(buf->size() < stop_size)
wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
return Void();
}
ACTOR Future<Void> read_http_response_headers(Reference<IConnection> conn, Headers *headers, std::string *buf, size_t *pos) {
loop {
// Get a line, reading more data from conn if necessary
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", buf, *pos));
// If line is empty we have reached the end of the headers.
if(lineLen == 0) {
// Increment pos to move past the empty line.
*pos += 2;
return Void();
}
int nameEnd=-1, valueStart=-1, valueEnd=-1;
int len = -1;
// Read header of the form "Name: Value\n"
// Note that multi line header values are not supported here.
// Format string breaks down as follows:
// %*[^:]%n Some characters other than ':' which are discarded, save the end position
// :%*[ \t]%n A colon followed by 0 or more spaces or tabs only, save the end position
// %*[^\r]%n Some characters other than \r which are discarded, save the end position
// %*1[\r] Exactly one \r
// %*1[\n] Exactly one \n
// %n Save final end position
if(sscanf(buf->c_str() + *pos, "%*[^:]%n:%*[ \t]%n%*[^\r]%n%*1[\r]%*1[\n]%n", &nameEnd, &valueStart, &valueEnd, &len) >= 0 && len > 0) {
const std::string name(buf->substr(*pos, nameEnd));
const std::string value(buf->substr(*pos + valueStart, valueEnd - valueStart));
(*headers)[name] = value;
*pos += len;
len = -1;
}
else // Malformed header line (at least according to this simple parsing)
throw http_bad_response();
}
}
// Reads an HTTP response from a network connection
// If the connection fails while being read the exception will emitted
// If the response is not parseable or complete in some way, http_bad_response will be thrown
ACTOR Future<Void> read_http_response(Reference<HTTP::Response> r, Reference<IConnection> conn, bool header_only) {
state std::string buf;
state size_t pos = 0;
// Read HTTP reponse code and version line
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &buf, pos));
int reachedEnd = -1;
sscanf(buf.c_str() + pos, "HTTP/%f %d%n", &r->version, &r->code, &reachedEnd);
if(reachedEnd < 0)
throw http_bad_response();
// Move position past the line found and the delimiter length
pos += lineLen + 2;
// Read headers
r->headers.clear();
wait(read_http_response_headers(conn, &r->headers, &buf, &pos));
auto i = r->headers.find("Content-Length");
if(i != r->headers.end())
r->contentLen = atoi(i->second.c_str());
else
r->contentLen = -1; // Content length unknown
state std::string transferEncoding;
i = r->headers.find("Transfer-Encoding");
if(i != r->headers.end())
transferEncoding = i->second;
r->content.clear();
// If this is supposed to be a header-only response and the buffer has been fully processed then stop. Otherwise, there must be response content.
if(header_only && pos == buf.size())
return Void();
// There should be content (or at least metadata describing that there is no content.
// Chunked transfer and 'normal' mode (content length given, data in one segment after headers) are supported.
if(r->contentLen >= 0) {
// Use response content as the buffer so there's no need to copy it later.
r->content = buf.substr(pos);
pos = 0;
// Read until there are at least contentLen bytes available at pos
wait(read_fixed_into_string(conn, r->contentLen, &r->content, pos));
// There shouldn't be any bytes after content.
if(r->content.size() != r->contentLen)
throw http_bad_response();
}
else if(transferEncoding == "chunked") {
// Copy remaining buffer data to content which will now be the read buffer for the chunk encoded data.
// Overall this will be fairly efficient since most bytes will only be written once but some bytes will
// have to be copied forward in the buffer when removing chunk overhead bytes.
r->content = buf.substr(pos);
pos = 0;
loop {
{
// Read the line that contains the chunk length as text in hex
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
state int chunkLen = strtol(r->content.substr(pos, lineLen).c_str(), nullptr, 16);
// Instead of advancing pos, erase the chunk length header line (line length + delimiter size) from the content buffer
r->content.erase(pos, lineLen + 2);
// If chunkLen is 0 then this marks the end of the content chunks.
if(chunkLen == 0)
break;
// Read (if needed) until chunkLen bytes are available at pos, then advance pos by chunkLen
wait(read_fixed_into_string(conn, chunkLen, &r->content, pos));
pos += chunkLen;
}
{
// Read the final empty line at the end of the chunk (the required "\r\n" after the chunk bytes)
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
if(lineLen != 0)
throw http_bad_response();
// Instead of advancing pos, erase the empty line from the content buffer
r->content.erase(pos, 2);
}
}
// The content buffer now contains the de-chunked, contiguous content at position 0 to pos. Save this length.
r->contentLen = pos;
// Next is the post-chunk header block, so read that.
wait(read_http_response_headers(conn, &r->headers, &r->content, &pos));
// If the header parsing did not consume all of the buffer then something is wrong
if(pos != r->content.size())
throw http_bad_response();
// Now truncate the buffer to just the dechunked contiguous content.
r->content.erase(r->contentLen);
}
std::string urlEncode(const std::string& s) {
std::string o;
o.reserve(s.size() * 3);
char buf[4];
for (auto c : s)
if (std::isalnum(c) || c == '?' || c == '/' || c == '-' || c == '_' || c == '.' || c == ',' || c == ':')
o.append(&c, 1);
else {
// Some unrecogize response content scheme is being used.
throw http_bad_response();
sprintf(buf, "%%%.02X", c);
o.append(buf);
}
// If there is actual response content, check the MD5 sum against the Content-MD5 response header
if(r->content.size() > 0)
if(!r->verifyMD5(false)) // false arg means do not fail if the Content-MD5 header is missing.
throw http_bad_response();
return Void();
}
Future<Void> HTTP::Response::read(Reference<IConnection> conn, bool header_only) {
return read_http_response(Reference<HTTP::Response>::addRef(this), conn, header_only);
}
// Do a request, get a Response.
// Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue itself must live for the life of this actor
// and be destroyed by the caller
// TODO: pSent is very hackish, do something better.
ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn, std::string verb, std::string resource, HTTP::Headers headers, UnsentPacketQueue *pContent, int contentLen, Reference<IRateControl> sendRate, int64_t *pSent, Reference<IRateControl> recvRate, std::string requestIDHeader) {
state TraceEvent event(SevDebug, "HTTPRequest");
state UnsentPacketQueue empty;
if(pContent == nullptr)
pContent = &empty;
// There is no standard http request id header field, so either a global default can be set via a knob
// or it can be set per-request with the requestIDHeader argument (which overrides the default)
if(requestIDHeader.empty()) {
requestIDHeader = CLIENT_KNOBS->HTTP_REQUEST_ID_HEADER;
}
state bool earlyResponse = false;
state int total_sent = 0;
state double send_start;
event.detail("DebugID", conn->getDebugID());
event.detail("RemoteAddress", conn->getPeerAddress());
event.detail("Verb", verb);
event.detail("Resource", resource);
event.detail("RequestContentLen", contentLen);
try {
state std::string requestID;
if(!requestIDHeader.empty()) {
requestID = deterministicRandom()->randomUniqueID().toString();
requestID = requestID.insert(20, "-");
requestID = requestID.insert(16, "-");
requestID = requestID.insert(12, "-");
requestID = requestID.insert(8, "-");
headers[requestIDHeader] = requestID;
event.detail("RequestIDSent", requestID);
}
// Write headers to a packet buffer chain
PacketBuffer* pFirst = PacketBuffer::create();
PacketBuffer *pLast = writeRequestHeader(verb, resource, headers, pFirst);
// Prepend headers to content packer buffer chain
pContent->prependWriteBuffer(pFirst, pLast);
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 1)
printf("[%s] HTTP starting %s %s ContentLen:%d\n", conn->getDebugID().toString().c_str(), verb.c_str(), resource.c_str(), contentLen);
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
for(auto h : headers)
printf("Request Header: %s: %s\n", h.first.c_str(), h.second.c_str());
}
state Reference<HTTP::Response> r(new HTTP::Response());
state Future<Void> responseReading = r->read(conn, verb == "HEAD" || verb == "DELETE");
send_start = timer();
loop {
// If we already got a response, before finishing sending the request, then close the connection,
// set the Connection header to "close" as a hint to the caller that this connection can't be used
// again, and break out of the send loop.
if(responseReading.isReady()) {
conn->close();
r->headers["Connection"] = "close";
earlyResponse = true;
break;
}
state int trySend = CLIENT_KNOBS->HTTP_SEND_SIZE;
wait(sendRate->getAllowance(trySend));
int len = conn->write(pContent->getUnsent(), trySend);
if(pSent != nullptr)
*pSent += len;
sendRate->returnUnused(trySend - len);
total_sent += len;
pContent->sent(len);
if(pContent->empty())
break;
wait(conn->onWritable());
wait(yield(TaskPriority::WriteSocket));
}
wait(responseReading);
double elapsed = timer() - send_start;
event.detail("ResponseCode", r->code);
event.detail("ResponseContentLen", r->contentLen);
event.detail("Elapsed", elapsed);
Optional<Error> err;
if(!requestIDHeader.empty()) {
std::string responseID;
auto iid = r->headers.find(requestIDHeader);
if(iid != r->headers.end()) {
responseID = iid->second;
}
event.detail("RequestIDReceived", responseID);
// If the response code is 5xx (server error) then a response ID is not expected
// so a missing id will be ignored but a mismatching id will still be an error.
bool serverError = r->code >= 500 && r->code < 600;
// If request/response IDs do not match and either this is not a server error
// or it is but the response ID is not empty then log an error.
if(requestID != responseID && (!serverError || !responseID.empty()) ) {
err = http_bad_request_id();
TraceEvent(SevError, "HTTPRequestFailedIDMismatch")
.detail("DebugID", conn->getDebugID())
.detail("RemoteAddress", conn->getPeerAddress())
.detail("Verb", verb)
.detail("Resource", resource)
.detail("RequestContentLen", contentLen)
.detail("ResponseCode", r->code)
.detail("ResponseContentLen", r->contentLen)
.detail("RequestIDSent", requestID)
.detail("RequestIDReceived", responseID)
.error(err.get());
}
}
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0) {
printf("[%s] HTTP %scode=%d early=%d, time=%fs %s %s contentLen=%d [%d out, response content len %d]\n",
conn->getDebugID().toString().c_str(),
(err.present() ? format("*ERROR*=%s ", err.get().name()).c_str() : ""),
r->code, earlyResponse, elapsed, verb.c_str(), resource.c_str(), contentLen, total_sent, (int)r->contentLen);
}
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
printf("[%s] HTTP RESPONSE: %s %s\n%s\n", conn->getDebugID().toString().c_str(), verb.c_str(), resource.c_str(), r->toString().c_str());
}
if(err.present()) {
throw err.get();
}
return r;
} catch(Error &e) {
double elapsed = timer() - send_start;
// A bad_request_id error would have already been logged in verbose mode before err is thrown above.
if(CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0 && e.code() != error_code_http_bad_request_id) {
printf("[%s] HTTP *ERROR*=%s early=%d, time=%fs %s %s contentLen=%d [%d out]\n",
conn->getDebugID().toString().c_str(), e.name(), earlyResponse, elapsed, verb.c_str(), resource.c_str(), contentLen, total_sent);
}
event.error(e);
throw;
}
}
return o;
}
bool Response::verifyMD5(bool fail_if_header_missing, Optional<std::string> content_sum) {
auto i = headers.find("Content-MD5");
if (i != headers.end()) {
// If a content sum is not provided, calculate one from the response content
if (!content_sum.present()) {
MD5_CTX sum;
::MD5_Init(&sum);
::MD5_Update(&sum, content.data(), content.size());
std::string sumBytes;
sumBytes.resize(16);
::MD5_Final((unsigned char*)sumBytes.data(), &sum);
std::string sumStr = base64::encoder::from_string(sumBytes);
sumStr.resize(sumStr.size() - 1);
content_sum = sumStr;
}
return i->second == content_sum.get();
}
return !fail_if_header_missing;
}
std::string Response::toString() {
std::string r = format("Response Code: %d\n", code);
r += format("Response ContentLen: %lld\n", contentLen);
for (auto h : headers)
r += format("Reponse Header: %s: %s\n", h.first.c_str(), h.second.c_str());
r.append("-- RESPONSE CONTENT--\n");
r.append(content);
r.append("\n--------\n");
return r;
}
PacketBuffer* writeRequestHeader(std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
PacketBuffer* dest) {
PacketWriter writer(dest, nullptr, Unversioned());
writer.serializeBytes(verb);
writer.serializeBytes(" ", 1);
writer.serializeBytes(resource);
writer.serializeBytes(LiteralStringRef(" HTTP/1.1\r\n"));
for (auto h : headers) {
writer.serializeBytes(h.first);
writer.serializeBytes(LiteralStringRef(": "));
writer.serializeBytes(h.second);
writer.serializeBytes(LiteralStringRef("\r\n"));
}
writer.serializeBytes(LiteralStringRef("\r\n"));
return writer.finish();
}
// Read at least 1 bytes from conn and up to maxlen in a single read, append read data into *buf
// Returns the number of bytes read.
ACTOR Future<int> read_into_string(Reference<IConnection> conn, std::string* buf, int maxlen) {
loop {
// Read into buffer
int originalSize = buf->size();
// TODO: resize is zero-initializing the space we're about to overwrite, so do something else, which probably
// means not using a string for this buffer
buf->resize(originalSize + maxlen);
uint8_t* wptr = (uint8_t*)buf->data() + originalSize;
int len = conn->read(wptr, wptr + maxlen);
buf->resize(originalSize + len);
// Make sure data was actually read, it's possible for there to be none.
if (len > 0)
return len;
// Wait for connection to have something to read
wait(conn->onReadable());
wait(delay(0, TaskPriority::ReadSocket));
}
}
// Returns the position of delim within buf, relative to pos. If delim is not found, continues to read from conn until
// either it is found or the connection ends, at which point connection_failed is thrown and buf contains
// everything that was read up to that point.
ACTOR Future<size_t> read_delimited_into_string(Reference<IConnection> conn,
const char* delim,
std::string* buf,
size_t pos) {
state size_t sPos = pos;
state int lookBack = strlen(delim) - 1;
ASSERT(lookBack >= 0);
loop {
size_t endPos = buf->find(delim, sPos);
if (endPos != std::string::npos)
return endPos - pos;
// Next search will start at the current end of the buffer - delim size + 1
if (sPos >= lookBack)
sPos -= lookBack;
wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
}
}
// Reads from conn (as needed) until there are at least len bytes starting at pos in buf
ACTOR Future<Void> read_fixed_into_string(Reference<IConnection> conn, int len, std::string* buf, size_t pos) {
state int stop_size = pos + len;
while (buf->size() < stop_size)
wait(success(read_into_string(conn, buf, CLIENT_KNOBS->HTTP_READ_SIZE)));
return Void();
}
ACTOR Future<Void> read_http_response_headers(Reference<IConnection> conn,
Headers* headers,
std::string* buf,
size_t* pos) {
loop {
// Get a line, reading more data from conn if necessary
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", buf, *pos));
// If line is empty we have reached the end of the headers.
if (lineLen == 0) {
// Increment pos to move past the empty line.
*pos += 2;
return Void();
}
int nameEnd = -1, valueStart = -1, valueEnd = -1;
int len = -1;
// Read header of the form "Name: Value\n"
// Note that multi line header values are not supported here.
// Format string breaks down as follows:
// %*[^:]%n Some characters other than ':' which are discarded, save the end position
// :%*[ \t]%n A colon followed by 0 or more spaces or tabs only, save the end position
// %*[^\r]%n Some characters other than \r which are discarded, save the end position
// %*1[\r] Exactly one \r
// %*1[\n] Exactly one \n
// %n Save final end position
if (sscanf(buf->c_str() + *pos,
"%*[^:]%n:%*[ \t]%n%*[^\r]%n%*1[\r]%*1[\n]%n",
&nameEnd,
&valueStart,
&valueEnd,
&len) >= 0 &&
len > 0) {
const std::string name(buf->substr(*pos, nameEnd));
const std::string value(buf->substr(*pos + valueStart, valueEnd - valueStart));
(*headers)[name] = value;
*pos += len;
len = -1;
} else // Malformed header line (at least according to this simple parsing)
throw http_bad_response();
}
}
// Reads an HTTP response from a network connection
// If the connection fails while being read the exception will emitted
// If the response is not parseable or complete in some way, http_bad_response will be thrown
ACTOR Future<Void> read_http_response(Reference<HTTP::Response> r, Reference<IConnection> conn, bool header_only) {
state std::string buf;
state size_t pos = 0;
// Read HTTP reponse code and version line
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &buf, pos));
int reachedEnd = -1;
sscanf(buf.c_str() + pos, "HTTP/%f %d%n", &r->version, &r->code, &reachedEnd);
if (reachedEnd < 0)
throw http_bad_response();
// Move position past the line found and the delimiter length
pos += lineLen + 2;
// Read headers
r->headers.clear();
wait(read_http_response_headers(conn, &r->headers, &buf, &pos));
auto i = r->headers.find("Content-Length");
if (i != r->headers.end())
r->contentLen = atoi(i->second.c_str());
else
r->contentLen = -1; // Content length unknown
state std::string transferEncoding;
i = r->headers.find("Transfer-Encoding");
if (i != r->headers.end())
transferEncoding = i->second;
r->content.clear();
// If this is supposed to be a header-only response and the buffer has been fully processed then stop. Otherwise,
// there must be response content.
if (header_only && pos == buf.size())
return Void();
// There should be content (or at least metadata describing that there is no content.
// Chunked transfer and 'normal' mode (content length given, data in one segment after headers) are supported.
if (r->contentLen >= 0) {
// Use response content as the buffer so there's no need to copy it later.
r->content = buf.substr(pos);
pos = 0;
// Read until there are at least contentLen bytes available at pos
wait(read_fixed_into_string(conn, r->contentLen, &r->content, pos));
// There shouldn't be any bytes after content.
if (r->content.size() != r->contentLen)
throw http_bad_response();
} else if (transferEncoding == "chunked") {
// Copy remaining buffer data to content which will now be the read buffer for the chunk encoded data.
// Overall this will be fairly efficient since most bytes will only be written once but some bytes will
// have to be copied forward in the buffer when removing chunk overhead bytes.
r->content = buf.substr(pos);
pos = 0;
loop {
{
// Read the line that contains the chunk length as text in hex
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
state int chunkLen = strtol(r->content.substr(pos, lineLen).c_str(), nullptr, 16);
// Instead of advancing pos, erase the chunk length header line (line length + delimiter size) from the
// content buffer
r->content.erase(pos, lineLen + 2);
// If chunkLen is 0 then this marks the end of the content chunks.
if (chunkLen == 0)
break;
// Read (if needed) until chunkLen bytes are available at pos, then advance pos by chunkLen
wait(read_fixed_into_string(conn, chunkLen, &r->content, pos));
pos += chunkLen;
}
{
// Read the final empty line at the end of the chunk (the required "\r\n" after the chunk bytes)
size_t lineLen = wait(read_delimited_into_string(conn, "\r\n", &r->content, pos));
if (lineLen != 0)
throw http_bad_response();
// Instead of advancing pos, erase the empty line from the content buffer
r->content.erase(pos, 2);
}
}
// The content buffer now contains the de-chunked, contiguous content at position 0 to pos. Save this length.
r->contentLen = pos;
// Next is the post-chunk header block, so read that.
wait(read_http_response_headers(conn, &r->headers, &r->content, &pos));
// If the header parsing did not consume all of the buffer then something is wrong
if (pos != r->content.size())
throw http_bad_response();
// Now truncate the buffer to just the dechunked contiguous content.
r->content.erase(r->contentLen);
} else {
// Some unrecogize response content scheme is being used.
throw http_bad_response();
}
// If there is actual response content, check the MD5 sum against the Content-MD5 response header
if (r->content.size() > 0)
if (!r->verifyMD5(false)) // false arg means do not fail if the Content-MD5 header is missing.
throw http_bad_response();
return Void();
}
Future<Void> HTTP::Response::read(Reference<IConnection> conn, bool header_only) {
return read_http_response(Reference<HTTP::Response>::addRef(this), conn, header_only);
}
// Do a request, get a Response.
// Request content is provided as UnsentPacketQueue *pContent which will be depleted as bytes are sent but the queue
// itself must live for the life of this actor and be destroyed by the caller
// TODO: pSent is very hackish, do something better.
ACTOR Future<Reference<HTTP::Response>> doRequest(Reference<IConnection> conn,
std::string verb,
std::string resource,
HTTP::Headers headers,
UnsentPacketQueue* pContent,
int contentLen,
Reference<IRateControl> sendRate,
int64_t* pSent,
Reference<IRateControl> recvRate,
std::string requestIDHeader) {
state TraceEvent event(SevDebug, "HTTPRequest");
state UnsentPacketQueue empty;
if (pContent == nullptr)
pContent = &empty;
// There is no standard http request id header field, so either a global default can be set via a knob
// or it can be set per-request with the requestIDHeader argument (which overrides the default)
if (requestIDHeader.empty()) {
requestIDHeader = CLIENT_KNOBS->HTTP_REQUEST_ID_HEADER;
}
state bool earlyResponse = false;
state int total_sent = 0;
state double send_start;
event.detail("DebugID", conn->getDebugID());
event.detail("RemoteAddress", conn->getPeerAddress());
event.detail("Verb", verb);
event.detail("Resource", resource);
event.detail("RequestContentLen", contentLen);
try {
state std::string requestID;
if (!requestIDHeader.empty()) {
requestID = deterministicRandom()->randomUniqueID().toString();
requestID = requestID.insert(20, "-");
requestID = requestID.insert(16, "-");
requestID = requestID.insert(12, "-");
requestID = requestID.insert(8, "-");
headers[requestIDHeader] = requestID;
event.detail("RequestIDSent", requestID);
}
// Write headers to a packet buffer chain
PacketBuffer* pFirst = PacketBuffer::create();
PacketBuffer* pLast = writeRequestHeader(verb, resource, headers, pFirst);
// Prepend headers to content packer buffer chain
pContent->prependWriteBuffer(pFirst, pLast);
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 1)
printf("[%s] HTTP starting %s %s ContentLen:%d\n",
conn->getDebugID().toString().c_str(),
verb.c_str(),
resource.c_str(),
contentLen);
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
for (auto h : headers)
printf("Request Header: %s: %s\n", h.first.c_str(), h.second.c_str());
}
state Reference<HTTP::Response> r(new HTTP::Response());
state Future<Void> responseReading = r->read(conn, verb == "HEAD" || verb == "DELETE");
send_start = timer();
loop {
// If we already got a response, before finishing sending the request, then close the connection,
// set the Connection header to "close" as a hint to the caller that this connection can't be used
// again, and break out of the send loop.
if (responseReading.isReady()) {
conn->close();
r->headers["Connection"] = "close";
earlyResponse = true;
break;
}
state int trySend = CLIENT_KNOBS->HTTP_SEND_SIZE;
wait(sendRate->getAllowance(trySend));
int len = conn->write(pContent->getUnsent(), trySend);
if (pSent != nullptr)
*pSent += len;
sendRate->returnUnused(trySend - len);
total_sent += len;
pContent->sent(len);
if (pContent->empty())
break;
wait(conn->onWritable());
wait(yield(TaskPriority::WriteSocket));
}
wait(responseReading);
double elapsed = timer() - send_start;
event.detail("ResponseCode", r->code);
event.detail("ResponseContentLen", r->contentLen);
event.detail("Elapsed", elapsed);
Optional<Error> err;
if (!requestIDHeader.empty()) {
std::string responseID;
auto iid = r->headers.find(requestIDHeader);
if (iid != r->headers.end()) {
responseID = iid->second;
}
event.detail("RequestIDReceived", responseID);
// If the response code is 5xx (server error) then a response ID is not expected
// so a missing id will be ignored but a mismatching id will still be an error.
bool serverError = r->code >= 500 && r->code < 600;
// If request/response IDs do not match and either this is not a server error
// or it is but the response ID is not empty then log an error.
if (requestID != responseID && (!serverError || !responseID.empty())) {
err = http_bad_request_id();
TraceEvent(SevError, "HTTPRequestFailedIDMismatch")
.detail("DebugID", conn->getDebugID())
.detail("RemoteAddress", conn->getPeerAddress())
.detail("Verb", verb)
.detail("Resource", resource)
.detail("RequestContentLen", contentLen)
.detail("ResponseCode", r->code)
.detail("ResponseContentLen", r->contentLen)
.detail("RequestIDSent", requestID)
.detail("RequestIDReceived", responseID)
.error(err.get());
}
}
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0) {
printf("[%s] HTTP %scode=%d early=%d, time=%fs %s %s contentLen=%d [%d out, response content len %d]\n",
conn->getDebugID().toString().c_str(),
(err.present() ? format("*ERROR*=%s ", err.get().name()).c_str() : ""),
r->code,
earlyResponse,
elapsed,
verb.c_str(),
resource.c_str(),
contentLen,
total_sent,
(int)r->contentLen);
}
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 2) {
printf("[%s] HTTP RESPONSE: %s %s\n%s\n",
conn->getDebugID().toString().c_str(),
verb.c_str(),
resource.c_str(),
r->toString().c_str());
}
if (err.present()) {
throw err.get();
}
return r;
} catch (Error& e) {
double elapsed = timer() - send_start;
// A bad_request_id error would have already been logged in verbose mode before err is thrown above.
if (CLIENT_KNOBS->HTTP_VERBOSE_LEVEL > 0 && e.code() != error_code_http_bad_request_id) {
printf("[%s] HTTP *ERROR*=%s early=%d, time=%fs %s %s contentLen=%d [%d out]\n",
conn->getDebugID().toString().c_str(),
e.name(),
earlyResponse,
elapsed,
verb.c_str(),
resource.c_str(),
contentLen,
total_sent);
}
event.error(e);
throw;
}
}
} // namespace HTTP

56
fdbclient/HTTP.h
View File

@ -24,32 +24,42 @@
#include "fdbclient/Knobs.h"
namespace HTTP {
struct is_iless {
bool operator() (const std::string &a, const std::string &b) const {
return strcasecmp(a.c_str(), b.c_str()) < 0;
}
};
struct is_iless {
bool operator()(const std::string& a, const std::string& b) const { return strcasecmp(a.c_str(), b.c_str()) < 0; }
};
typedef std::map<std::string, std::string, is_iless> Headers;
typedef std::map<std::string, std::string, is_iless> Headers;
std::string urlEncode(const std::string &s);
std::string urlEncode(const std::string& s);
struct Response : ReferenceCounted<Response>{
Response() {}
Future<Void> read(Reference<IConnection> conn, bool header_only);
std::string toString();
float version;
int code;
Headers headers;
std::string content;
int64_t contentLen;
struct Response : ReferenceCounted<Response> {
Response() {}
Future<Void> read(Reference<IConnection> conn, bool header_only);
std::string toString();
float version;
int code;
Headers headers;
std::string content;
int64_t contentLen;
bool verifyMD5(bool fail_if_header_missing, Optional<std::string> content_sum = Optional<std::string>());
};
bool verifyMD5(bool fail_if_header_missing, Optional<std::string> content_sum = Optional<std::string>());
};
// Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain
PacketBuffer * writeRequestHeader(std::string const &verb, std::string const &resource, HTTP::Headers const &headers, PacketBuffer *dest);
// Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain
PacketBuffer* writeRequestHeader(std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
PacketBuffer* dest);
// Do an HTTP request to the blob store, parse the response.
Future<Reference<Response>> doRequest(Reference<IConnection> const &conn, std::string const &verb, std::string const &resource, HTTP::Headers const &headers, UnsentPacketQueue * const &pContent, int const &contentLen, Reference<IRateControl> const &sendRate, int64_t * const &pSent, Reference<IRateControl> const &recvRate, const std::string &requestHeader = std::string());
}
// Do an HTTP request to the blob store, parse the response.
Future<Reference<Response>> doRequest(Reference<IConnection> const& conn,
std::string const& verb,
std::string const& resource,
HTTP::Headers const& headers,
UnsentPacketQueue* const& pContent,
int const& contentLen,
Reference<IRateControl> const& sendRate,
int64_t* const& pSent,
Reference<IRateControl> const& recvRate,
const std::string& requestHeader = std::string());
} // namespace HTTP

44
fdbclient/IClientApi.h
View File

@ -36,15 +36,29 @@ public:
virtual void setVersion(Version v) = 0;
virtual ThreadFuture<Version> getReadVersion() = 0;
// These functions that read data return Standalone<...> objects, but these objects are not required to manage their own memory.
// It is guaranteed, however, that the ThreadFuture will hold a reference to the memory. It will persist until the ThreadFuture's
// ThreadSingleAssignmentVar has its memory released or it is destroyed.
virtual ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot=false) = 0;
virtual ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot=false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) = 0;
// These functions that read data return Standalone<...> objects, but these objects are not required to manage their
// own memory. It is guaranteed, however, that the ThreadFuture will hold a reference to the memory. It will persist
// until the ThreadFuture's ThreadSingleAssignmentVar has its memory released or it is destroyed.
virtual ThreadFuture<Optional<Value>> get(const KeyRef& key, bool snapshot = false) = 0;
virtual ThreadFuture<Key> getKey(const KeySelectorRef& key, bool snapshot = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
int limit,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeySelectorRef& begin,
const KeySelectorRef& end,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
int limit,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<RangeResultRef>> getRange(const KeyRangeRef& keys,
GetRangeLimits limits,
bool snapshot = false,
bool reverse = false) = 0;
virtual ThreadFuture<Standalone<VectorRef<const char*>>> getAddressesForKey(const KeyRef& key) = 0;
virtual ThreadFuture<Standalone<StringRef>> getVersionstamp() = 0;
@ -67,7 +81,7 @@ public:
virtual Version getCommittedVersion() = 0;
virtual ThreadFuture<int64_t> getApproximateSize() = 0;
virtual void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value=Optional<StringRef>()) = 0;
virtual void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual ThreadFuture<Void> onError(Error const& e) = 0;
virtual void reset() = 0;
@ -88,7 +102,8 @@ public:
// Management API, attempt to kill or suspend a process, return 1 for request sent out, 0 for failure
virtual ThreadFuture<int64_t> rebootWorker(const StringRef& address, bool check, int duration) = 0;
// Management API, force the database to recover into DCID, causing the database to lose the most recently committed mutations
// Management API, force the database to recover into DCID, causing the database to lose the most recently committed
// mutations
virtual ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0;
// Management API, create snapshot
virtual ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) = 0;
@ -102,14 +117,15 @@ public:
virtual const char* getClientVersion() = 0;
virtual ThreadFuture<uint64_t> getServerProtocol(const char* clusterFilePath) = 0;
virtual void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual void setNetworkOption(FDBNetworkOptions::Option option,
Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual void setupNetwork() = 0;
virtual void runNetwork() = 0;
virtual void stopNetwork() = 0;
virtual Reference<IDatabase> createDatabase(const char *clusterFilePath) = 0;
virtual Reference<IDatabase> createDatabase(const char* clusterFilePath) = 0;
virtual void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter) = 0;
virtual void addNetworkThreadCompletionHook(void (*hook)(void*), void* hookParameter) = 0;
};
#endif

140
fdbclient/JSONDoc.h
View File

@ -40,9 +40,8 @@
// // See if JSON doc path a.b.c exists
// bool exists = r.has("a.b.c");
//
// // See if JSON doc path a.b.c exists, if it does then assign value to x. Throws if path exists but T is not compatible.
// T x;
// bool exists = r.has("a.b.c", x);
// // See if JSON doc path a.b.c exists, if it does then assign value to x. Throws if path exists but T is not
// compatible. T x; bool exists = r.has("a.b.c", x);
//
// // This way you can chain things like this:
// bool is_two = r.has("a.b.c", x) && x == 2;
@ -52,7 +51,7 @@
// bool is_int = r.has("a.b.c") && r.last().type == json_spirit::int_type;
// bool is_two = r.has("a.b.c") && r.last().get_int() == 2;
//
// // The familiar at() method also exists but now supports the same path concept.
// // The familiar at() method also exists but now supports the same path concept.
// // It will throw in the same circumstances as the original method
// int x = r.at("a.b.c").get_int();
//
@ -71,20 +70,20 @@ struct JSONDoc {
// Construction from const json_spirit::mObject, trivial and will never throw.
// Resulting JSONDoc will not allow modifications.
JSONDoc(const json_spirit::mObject &o) : pObj(&o), wpObj(nullptr) {}
JSONDoc(const json_spirit::mObject& o) : pObj(&o), wpObj(nullptr) {}
// Construction from json_spirit::mObject. Allows modifications.
JSONDoc(json_spirit::mObject &o) : pObj(&o), wpObj(&o) {}
JSONDoc(json_spirit::mObject& o) : pObj(&o), wpObj(&o) {}
// Construction from const json_spirit::mValue (which is a Variant type) which will try to
// convert it to an mObject. This will throw if that fails, just as it would
// if the caller called get_obj() itself and used the previous constructor instead.
JSONDoc(const json_spirit::mValue &v) : pObj(&v.get_obj()), wpObj(nullptr) {}
JSONDoc(const json_spirit::mValue& v) : pObj(&v.get_obj()), wpObj(nullptr) {}
// Construction from non-const json_spirit::mValue - will convert the mValue to
// an object if it isn't already and then attach to it.
JSONDoc(json_spirit::mValue &v) {
if(v.type() != json_spirit::obj_type)
JSONDoc(json_spirit::mValue& v) {
if (v.type() != json_spirit::obj_type)
v = json_spirit::mObject();
wpObj = &v.get_obj();
pObj = wpObj;
@ -97,16 +96,15 @@ struct JSONDoc {
// If the "split" flag is set to "false", then this skips the splitting of a
// path into on the "dot" character.
// When a path is found, pLast is updated.
bool has(std::string path, bool split=true) {
bool has(std::string path, bool split = true) {
if (pObj == nullptr)
return false;
if (path.empty())
return false;
size_t start = 0;
const json_spirit::mValue *curVal = nullptr;
while (start < path.size())
{
const json_spirit::mValue* curVal = nullptr;
while (start < path.size()) {
// If a path segment is found then curVal must be an object
size_t dot;
if (split) {
@ -120,7 +118,7 @@ struct JSONDoc {
// Get pointer to the current Object that the key has to be in
// This will throw if the value is not an Object
const json_spirit::mObject *curObj = curVal ? &curVal->get_obj() : pObj;
const json_spirit::mObject* curObj = curVal ? &curVal->get_obj() : pObj;
// Make sure key exists, if not then return false
if (!curObj->count(key))
@ -139,14 +137,13 @@ struct JSONDoc {
// Creates the given path (forcing Objects to exist along its depth, replacing whatever else might have been there)
// and returns a reference to the Value at that location.
json_spirit::mValue & create(std::string path, bool split=true) {
json_spirit::mValue& create(std::string path, bool split = true) {
if (wpObj == nullptr || path.empty())
throw std::runtime_error("JSON Object not writable or bad JSON path");
size_t start = 0;
json_spirit::mValue *curVal = nullptr;
while (start < path.size())
{
json_spirit::mValue* curVal = nullptr;
while (start < path.size()) {
// Get next path segment name
size_t dot;
if (split) {
@ -157,18 +154,17 @@ struct JSONDoc {
dot = path.size();
}
std::string key = path.substr(start, dot - start);
if(key.empty())
if (key.empty())
throw std::runtime_error("invalid JSON path");
// Get/create pointer to the current Object that the key has to be in
// If curVal is defined then force it to be an Object
json_spirit::mObject *curObj;
if(curVal != nullptr) {
if(curVal->type() != json_spirit::obj_type)
json_spirit::mObject* curObj;
if (curVal != nullptr) {
if (curVal->type() != json_spirit::obj_type)
*curVal = json_spirit::mObject();
curObj = &curVal->get_obj();
}
else // Otherwise start with the object *this is writing to
} else // Otherwise start with the object *this is writing to
curObj = wpObj;
// Make sure key exists, if not then return false
@ -186,77 +182,86 @@ struct JSONDoc {
}
// Creates the path given, puts a value at it, and returns a reference to the value
template<typename T>
T & put(std::string path, const T & value, bool split=true) {
json_spirit::mValue &v = create(path, split);
template <typename T>
T& put(std::string path, const T& value, bool split = true) {
json_spirit::mValue& v = create(path, split);
v = value;
return v.get_value<T>();
}
// Ensures that an Object exists at path and returns a JSONDoc that writes to it.
JSONDoc subDoc(std::string path, bool split=true) {
json_spirit::mValue &v = create(path, split);
if(v.type() != json_spirit::obj_type)
JSONDoc subDoc(std::string path, bool split = true) {
json_spirit::mValue& v = create(path, split);
if (v.type() != json_spirit::obj_type)
v = json_spirit::mObject();
return JSONDoc(v.get_obj());
}
// Apply a merge operation to two values. Works for int, double, and string
template <typename T>
static json_spirit::mObject mergeOperator(const std::string &op, const json_spirit::mObject &op_a, const json_spirit::mObject &op_b, T const &a, T const &b) {
if(op == "$max")
return {{op, std::max<T>(a, b)}};
if(op == "$min")
return {{op, std::min<T>(a, b)}};
if(op == "$sum")
return {{op, a + b}};
static json_spirit::mObject mergeOperator(const std::string& op,
const json_spirit::mObject& op_a,
const json_spirit::mObject& op_b,
T const& a,
T const& b) {
if (op == "$max")
return { { op, std::max<T>(a, b) } };
if (op == "$min")
return { { op, std::min<T>(a, b) } };
if (op == "$sum")
return { { op, a + b } };
throw std::exception();
}
// This is just a convenience function to make calling mergeOperator look cleaner
template <typename T>
static json_spirit::mObject mergeOperatorWrapper(const std::string &op, const json_spirit::mObject &op_a, const json_spirit::mObject &op_b, const json_spirit::mValue &a, const json_spirit::mValue &b) {
static json_spirit::mObject mergeOperatorWrapper(const std::string& op,
const json_spirit::mObject& op_a,
const json_spirit::mObject& op_b,
const json_spirit::mValue& a,
const json_spirit::mValue& b) {
return mergeOperator<T>(op, op_a, op_b, a.get_value<T>(), b.get_value<T>());
}
static inline const std::string & getOperator(const json_spirit::mObject &obj) {
static inline const std::string& getOperator(const json_spirit::mObject& obj) {
static const std::string empty;
for(auto &k : obj)
if(!k.first.empty() && k.first[0] == '$')
for (auto& k : obj)
if (!k.first.empty() && k.first[0] == '$')
return k.first;
return empty;
}
// Merge src into dest, applying merge operators
static void mergeInto(json_spirit::mObject &dst, const json_spirit::mObject &src);
static void mergeValueInto(json_spirit::mValue &d, const json_spirit::mValue &s);
static void mergeInto(json_spirit::mObject& dst, const json_spirit::mObject& src);
static void mergeValueInto(json_spirit::mValue& d, const json_spirit::mValue& s);
// Remove any merge operators that never met any mates.
static void cleanOps(json_spirit::mObject &obj);
static void cleanOps(json_spirit::mObject& obj);
void cleanOps() {
if(wpObj == nullptr)
if (wpObj == nullptr)
throw std::runtime_error("JSON Object not writable");
return cleanOps(*wpObj);
}
void absorb(const JSONDoc &doc) {
if(wpObj == nullptr)
void absorb(const JSONDoc& doc) {
if (wpObj == nullptr)
throw std::runtime_error("JSON Object not writable");
if(doc.pObj == nullptr)
if (doc.pObj == nullptr)
throw std::runtime_error("JSON Object not readable");
mergeInto(*wpObj, *doc.pObj);
}
// Returns whether or not a "path" exists.
// Returns true if all elements along path exist
// Returns false if any elements along the path are MISSING
// Sets out to the value of the thing that path refers to
// Will throw if a non-terminating path element exists BUT is not a JSON Object.
// Will throw if all elements along path exists but T is an incompatible type
template <typename T> bool get(const std::string path, T &out, bool split=true) {
template <typename T>
bool get(const std::string path, T& out, bool split = true) {
bool r = has(path, split);
if (r)
out = pLast->get_value<T>();
@ -264,25 +269,27 @@ struct JSONDoc {
}
// For convenience, wraps get() in a try/catch and returns false UNLESS the path existed and was a compatible type.
template <typename T> bool tryGet(const std::string path, T &out, bool split=true) {
try { return get(path, out, split); } catch(...) {}
template <typename T>
bool tryGet(const std::string path, T& out, bool split = true) {
try {
return get(path, out, split);
} catch (...) {
}
return false;
}
const json_spirit::mValue & at(const std::string path, bool split=true) {
const json_spirit::mValue& at(const std::string path, bool split = true) {
if (has(path, split))
return last();
throw std::runtime_error("JSON path doesn't exist");
}
const json_spirit::mValue & operator[](const std::string path) {
return at(path);
}
const json_spirit::mValue& operator[](const std::string path) { return at(path); }
const json_spirit::mValue & last() const { return *pLast; }
const json_spirit::mValue& last() const { return *pLast; }
bool valid() const { return pObj != nullptr; }
const json_spirit::mObject & obj() {
const json_spirit::mObject& obj() {
// This dummy object is necessary to make working with obj() easier when this does not currently
// point to a valid mObject. valid() can be called to explicitly check for this scenario, but
// calling obj() at least will not seg fault and instead return a const reference to an empty mObject.
@ -291,8 +298,9 @@ struct JSONDoc {
return pObj ? *pObj : dummy;
}
// Return reference to writeable underlying mObject but only if *this was initialized with a writeable value or object
json_spirit::mObject & wobj() {
// Return reference to writeable underlying mObject but only if *this was initialized with a writeable value or
// object
json_spirit::mObject& wobj() {
ASSERT(wpObj != nullptr);
return *wpObj;
}
@ -302,10 +310,10 @@ struct JSONDoc {
// it is intended to be used.
// This is slightly hackish but otherwise the JSON merge functions would require a Transaction.
static uint64_t expires_reference_version;
private:
const json_spirit::mObject *pObj;
// Writeable pointer to the same object. Will be nullptr if initialized from a const object.
json_spirit::mObject *wpObj;
const json_spirit::mValue *pLast;
};
private:
const json_spirit::mObject* pObj;
// Writeable pointer to the same object. Will be nullptr if initialized from a const object.
json_spirit::mObject* wpObj;
const json_spirit::mValue* pLast;
};

65
fdbclient/JsonBuilder.cpp Executable file → Normal file
View File

@ -1,7 +1,7 @@
#include "fdbclient/JsonBuilder.h"
#include <iostream>
JsonBuilderObject JsonBuilder::makeMessage(const char *name, const char *description) {
JsonBuilderObject JsonBuilder::makeMessage(const char* name, const char* description) {
JsonBuilderObject out;
out["name"] = name;
out["description"] = description;
@ -10,28 +10,28 @@ JsonBuilderObject JsonBuilder::makeMessage(const char *name, const char *descrip
// dst must have at least len + 3 bytes available (".e" becomes "0.0e0")
// Returns bytes written, or 0 on failure.
int JsonBuilder::coerceAsciiNumberToJSON(const char *s, int len, char *dst) {
if(len == 0) {
int JsonBuilder::coerceAsciiNumberToJSON(const char* s, int len, char* dst) {
if (len == 0) {
return 0;
}
const char *send = s + len;
char *wptr = dst;
const char* send = s + len;
char* wptr = dst;
bool dot = false;
// Allow one optional sign
if(*s == '-') {
if (*s == '-') {
*wptr++ = *s++;
// Output not yet valid so return failure
if(s == send) {
if (s == send) {
return 0;
}
}
// 'inf' becomes 1e99
if(*s == 'i') {
if(len >= 3 && (strncmp(s, "inf", 3) == 0)) {
if (*s == 'i') {
if (len >= 3 && (strncmp(s, "inf", 3) == 0)) {
strcpy(wptr, "1e99");
return 4 + wptr - dst;
}
@ -40,107 +40,104 @@ int JsonBuilder::coerceAsciiNumberToJSON(const char *s, int len, char *dst) {
}
// Skip leading zeroes
while(*s == '0') {
while (*s == '0') {
++s;
// If found end, number is valid and zero
if(s == send) {
if (s == send) {
*wptr++ = '0';
return wptr - dst;
}
}
// If a dot is found, write a zero before it
if(*s == '.') {
if (*s == '.') {
dot = true;
*wptr++ = '0';
*wptr++ = *s++;
// If found end, add a zero and return
if(s == send) {
if (s == send) {
*wptr++ = '0';
return wptr - dst;
}
// If there is no digit after the dot, write a zero
if(!isdigit(*s)) {
if (!isdigit(*s)) {
*wptr++ = '0';
}
}
// Write all digits found
while(isdigit(*s)) {
while (isdigit(*s)) {
*wptr++ = *s++;
// If found end, number is valid so return
if(s == send) {
if (s == send) {
return wptr - dst;
}
}
// If there is a dot, return unless its the first
if(*s == '.') {
if(dot) {
if (*s == '.') {
if (dot) {
return wptr - dst;
}
*wptr++ = *s++;
// If found end, add a zero and return
if(s == send) {
if (s == send) {
*wptr++ = '0';
return wptr - dst;
}
// If there are more digits write them, else write a 0
if(isdigit(*s)) {
if (isdigit(*s)) {
do {
*wptr++ = *s++;
// If found end, number is valid so return
if(s == send) {
if (s == send) {
return wptr - dst;
}
} while(isdigit(*s));
}
else {
} while (isdigit(*s));
} else {
*wptr++ = '0';
}
}
// Now we can have an e or E, else stop
if(*s == 'e' || *s == 'E') {
if (*s == 'e' || *s == 'E') {
*wptr++ = *s++;
// If found end, add a zero and return
if(s == send) {
if (s == send) {
*wptr++ = '0';
return wptr - dst;
}
// Allow one optional sign
if(*s == '-' || *s == '+') {
if (*s == '-' || *s == '+') {
*wptr++ = *s++;
}
// If found end, add a zero and return
if(s == send) {
if (s == send) {
*wptr++ = '0';
return wptr - dst;
}
// If there are more digits write then, else write a 0
if(isdigit(*s)) {
if (isdigit(*s)) {
do {
*wptr++ = *s++;
// If found end, number is valid so return
if(s == send) {
if (s == send) {
return wptr - dst;
}
} while(isdigit(*s));
}
else {
} while (isdigit(*s));
} else {
*wptr++ = '0';
}
}

225
fdbclient/JsonBuilder.h
View File

@ -11,7 +11,8 @@ class JsonBuilder;
class JsonBuilderObject;
class JsonBuilderArray;
typedef JsonBuilder JsonString;
template <typename T> class JsonBuilderObjectSetter;
template <typename T>
class JsonBuilderObjectSetter;
// Class for building JSON string values.
// Default value is null, as in the JSON type
@ -20,38 +21,31 @@ protected:
enum EType { NULLVALUE, OBJECT, ARRAY };
typedef VectorRef<char> VString;
public:
// Default value is null, which will be considered "empty"
JsonBuilder() : type(NULLVALUE), elements(0), bytes(0) {
jsonText.resize(arena, 1);
}
JsonBuilder() : type(NULLVALUE), elements(0), bytes(0) { jsonText.resize(arena, 1); }
int getFinalLength() const {
return bytes + strlen(getEnd());
}
int getFinalLength() const { return bytes + strlen(getEnd()); }
// TODO: Remove the need for this by changing usages to steal this's content
std::string getJson() const {
std::string result;
result.reserve(bytes + 1);
for(auto& it : jsonText) {
for (auto& it : jsonText) {
result.append(it.begin(), it.end());
}
result.append(getEnd());
return result;
}
int size() const {
return elements;
}
int size() const { return elements; }
bool empty() const {
return elements == 0;
}
bool empty() const { return elements == 0; }
static JsonBuilderObject makeMessage(const char *name, const char *description);
static JsonBuilderObject makeMessage(const char* name, const char* description);
static int coerceAsciiNumberToJSON(const char *s, int len, char *dst);
static int coerceAsciiNumberToJSON(const char* s, int len, char* dst);
protected:
EType type;
@ -61,18 +55,14 @@ protected:
int bytes;
// 'raw' write methods
inline void write(const char *s, int len) {
inline void write(const char* s, int len) {
bytes += len;
jsonText.back().append(arena, s, len);
}
inline void write(const char* s) {
write(s, strlen(s));
}
inline void write(const char* s) { write(s, strlen(s)); }
inline void write(const StringRef &s) {
write((char *)s.begin(), s.size());
}
inline void write(const StringRef& s) { write((char*)s.begin(), s.size()); }
inline void write(char s) {
++bytes;
@ -80,82 +70,72 @@ protected:
}
// writeValue() methods write JSON form of the value
void writeValue(const json_spirit::mValue &val) {
switch(val.type()) {
case json_spirit::int_type:
return writeValue(val.get_int64());
case json_spirit::bool_type:
return writeValue(val.get_bool());
case json_spirit::real_type:
return writeValue(val.get_real());
case json_spirit::str_type:
return writeValue(val.get_str());
default:
// Catch-all for objects/arrays
return write(json_spirit::write_string(val));
void writeValue(const json_spirit::mValue& val) {
switch (val.type()) {
case json_spirit::int_type:
return writeValue(val.get_int64());
case json_spirit::bool_type:
return writeValue(val.get_bool());
case json_spirit::real_type:
return writeValue(val.get_real());
case json_spirit::str_type:
return writeValue(val.get_str());
default:
// Catch-all for objects/arrays
return write(json_spirit::write_string(val));
};
}
void writeValue(const bool& val) {
write(val ? "true" : "false");
}
void writeValue(const bool& val) { write(val ? "true" : "false"); }
template<typename T> inline void writeFormat(const char *fmt, const T &val) {
VString &dst = jsonText.back();
template <typename T>
inline void writeFormat(const char* fmt, const T& val) {
VString& dst = jsonText.back();
const int limit = 30;
dst.reserve(arena, dst.size() + limit);
int len = snprintf(dst.end(), limit, fmt, val);
if(len > 0 && len < limit) {
if (len > 0 && len < limit) {
dst.extendUnsafeNoReallocNoInit(len);
}
else {
} else {
write(format(fmt, val));
}
}
void writeValue(const int64_t& val) {
writeFormat("%lld", val);
}
void writeValue(const int64_t& val) { writeFormat("%lld", val); }
void writeValue(const uint64_t& val) {
writeFormat("%llu", val);
}
void writeValue(const uint64_t& val) { writeFormat("%llu", val); }
void writeValue(const int& val) {
writeFormat("%d", val);
}
void writeValue(const int& val) { writeFormat("%d", val); }
void writeValue(const double& val) {
if(std::isfinite(val)) {
if (std::isfinite(val)) {
writeFormat("%g", val);
}
else if(std::isnan(val)) {
} else if (std::isnan(val)) {
write("-999");
}
else {
} else {
write("1e99");
}
}
bool shouldEscape(char c) {
switch( c ) {
case '"':
case '\\':
case '\b':
case '\f':
case '\n':
case '\r':
case '\t':
return true;
default:
return false;
switch (c) {
case '"':
case '\\':
case '\b':
case '\f':
case '\n':
case '\r':
case '\t':
return true;
default:
return false;
}
}
void writeValue(const char *val, int len) {
void writeValue(const char* val, int len) {
write('"');
int beginCopy = 0;
VString &dst = jsonText.back();
VString& dst = jsonText.back();
for (int i = 0; i < len; i++) {
if (shouldEscape(val[i])) {
dst.append(arena, val + beginCopy, i - beginCopy);
@ -164,26 +144,20 @@ protected:
write(val[i]);
}
}
if(beginCopy < len) {
if (beginCopy < len) {
dst.append(arena, val + beginCopy, len - beginCopy);
}
write('"');
}
inline void writeValue(const std::string& val) {
writeValue(val.data(), val.size());
}
inline void writeValue(const std::string& val) { writeValue(val.data(), val.size()); }
inline void writeValue(const char* val) {
writeValue(val, strlen(val));
}
inline void writeValue(const char* val) { writeValue(val, strlen(val)); }
inline void writeValue(const StringRef &s) {
writeValue((const char *)s.begin(), s.size());
}
inline void writeValue(const StringRef& s) { writeValue((const char*)s.begin(), s.size()); }
// Write the finalized (closed) form of val
void writeValue(const JsonBuilder &val) {
void writeValue(const JsonBuilder& val) {
bytes += val.bytes;
jsonText.append(arena, val.jsonText.begin(), val.jsonText.size());
val.jsonText.push_back(arena, VString());
@ -191,41 +165,38 @@ protected:
write(val.getEnd());
}
void writeCoercedAsciiNumber(const char *s, int len) {
VString &val = jsonText.back();
void writeCoercedAsciiNumber(const char* s, int len) {
VString& val = jsonText.back();
val.reserve(arena, val.size() + len + 3);
int written = coerceAsciiNumberToJSON(s, len, val.end());
if(written > 0) {
if (written > 0) {
val.extendUnsafeNoReallocNoInit(written);
}
else {
} else {
write("-999");
}
}
inline void writeCoercedAsciiNumber(const StringRef &s) {
writeCoercedAsciiNumber((const char *)s.begin(), s.size());
inline void writeCoercedAsciiNumber(const StringRef& s) {
writeCoercedAsciiNumber((const char*)s.begin(), s.size());
}
inline void writeCoercedAsciiNumber(const std::string &s) {
writeCoercedAsciiNumber(s.data(), s.size());
}
inline void writeCoercedAsciiNumber(const std::string& s) { writeCoercedAsciiNumber(s.data(), s.size()); }
// Helper function to add contents of another JsonBuilder to this one.
// This is only used by the subclasses to combine like-typed (at compile time) objects,
// so it can be assumed that the other object has been initialized with an opening character.
void _addContents(const JsonBuilder &other) {
if(other.empty()) {
void _addContents(const JsonBuilder& other) {
if (other.empty()) {
return;
}
if(elements > 0) {
if (elements > 0) {
write(',');
}
// Add everything but the first byte of the first string in arr
bytes += other.bytes - 1;
const VString &front = other.jsonText.front();
const VString& front = other.jsonText.front();
jsonText.push_back(arena, front.slice(1, front.size()));
jsonText.append(arena, other.jsonText.begin() + 1, other.jsonText.size() - 1);
@ -238,16 +209,16 @@ protected:
}
// Get the text necessary to finish the JSON string
const char * getEnd() const {
switch(type) {
case NULLVALUE:
return "null";
case OBJECT:
return "}";
case ARRAY:
return "]";
default:
return "";
const char* getEnd() const {
switch (type) {
case NULLVALUE:
return "null";
case OBJECT:
return "}";
case ARRAY:
return "]";
default:
return "";
};
}
};
@ -259,22 +230,23 @@ public:
write('[');
}
template<typename VT> inline JsonBuilderArray & push_back(const VT &val) {
if(elements++ > 0) {
template <typename VT>
inline JsonBuilderArray& push_back(const VT& val) {
if (elements++ > 0) {
write(',');
}
writeValue(val);
return *this;
}
JsonBuilderArray & addContents(const json_spirit::mArray &arr) {
for(auto &v : arr) {
JsonBuilderArray& addContents(const json_spirit::mArray& arr) {
for (auto& v : arr) {
push_back(v);
}
return *this;
}
JsonBuilderArray & addContents(const JsonBuilderArray &arr) {
JsonBuilderArray& addContents(const JsonBuilderArray& arr) {
_addContents(arr);
return *this;
}
@ -287,8 +259,9 @@ public:
write('{');
}
template<typename KT, typename VT> inline JsonBuilderObject & setKey(const KT &name, const VT &val) {
if(elements++ > 0) {
template <typename KT, typename VT>
inline JsonBuilderObject& setKey(const KT& name, const VT& val) {
if (elements++ > 0) {
write(',');
}
write('"');
@ -298,8 +271,9 @@ public:
return *this;
}
template<typename KT, typename VT> inline JsonBuilderObject & setKeyRawNumber(const KT &name, const VT &val) {
if(elements++ > 0) {
template <typename KT, typename VT>
inline JsonBuilderObject& setKeyRawNumber(const KT& name, const VT& val) {
if (elements++ > 0) {
write(',');
}
write('"');
@ -309,30 +283,31 @@ public:
return *this;
}
template<typename T> inline JsonBuilderObjectSetter<T> operator[](T &&name);
template <typename T>
inline JsonBuilderObjectSetter<T> operator[](T&& name);
JsonBuilderObject & addContents(const json_spirit::mObject &obj) {
for(auto &kv : obj) {
JsonBuilderObject& addContents(const json_spirit::mObject& obj) {
for (auto& kv : obj) {
setKey(kv.first, kv.second);
}
return *this;
}
JsonBuilderObject & addContents(const JsonBuilderObject &obj) {
JsonBuilderObject& addContents(const JsonBuilderObject& obj) {
_addContents(obj);
return *this;
}
};
// Template is the key name, accepted as an r-value if possible to avoid copying if it's a string
template<typename KT>
template <typename KT>
class JsonBuilderObjectSetter {
public:
JsonBuilderObjectSetter(JsonBuilderObject &dest, KT &&name) : dest(dest), name(std::forward<KT>(name)) {}
JsonBuilderObjectSetter(JsonBuilderObject& dest, KT&& name) : dest(dest), name(std::forward<KT>(name)) {}
// Value is accepted as an rvalue if possible
template <class VT> inline void operator=(const VT &value) {
template <class VT>
inline void operator=(const VT& value) {
dest.setKey(name, value);
}
@ -341,7 +316,7 @@ protected:
KT name;
};
template<typename T> inline JsonBuilderObjectSetter<T> JsonBuilderObject::operator[](T &&name) {
template <typename T>
inline JsonBuilderObjectSetter<T> JsonBuilderObject::operator[](T&& name) {
return JsonBuilderObjectSetter<T>(*this, std::forward<T>(name));
}

239
fdbclient/KeyBackedTypes.h
View File

@ -37,45 +37,83 @@
// Since Codec is a struct, partial specialization can be used, such as the std::pair
// partial specialization below allowing any std::pair<T1,T2> where T1 and T2 are already
// supported by Codec.
template<typename T>
template <typename T>
struct Codec {
static inline Tuple pack(T const &val) { return val.pack(); }
static inline T unpack(Tuple const &t) { return T::unpack(t); }
static inline Tuple pack(T const& val) { return val.pack(); }
static inline T unpack(Tuple const& t) { return T::unpack(t); }
};
// If T is Tuple then conversion is simple.
template<> inline Tuple Codec<Tuple>::pack(Tuple const &val) { return val; }
template<> inline Tuple Codec<Tuple>::unpack(Tuple const &val) { return val; }
template <>
inline Tuple Codec<Tuple>::pack(Tuple const& val) {
return val;
}
template <>
inline Tuple Codec<Tuple>::unpack(Tuple const& val) {
return val;
}
template<> inline Tuple Codec<int64_t>::pack(int64_t const &val) { return Tuple().append(val); }
template<> inline int64_t Codec<int64_t>::unpack(Tuple const &val) { return val.getInt(0); }
template <>
inline Tuple Codec<int64_t>::pack(int64_t const& val) {
return Tuple().append(val);
}
template <>
inline int64_t Codec<int64_t>::unpack(Tuple const& val) {
return val.getInt(0);
}
template<> inline Tuple Codec<bool>::pack(bool const &val) { return Tuple().append(val ? 1 : 0); }
template<> inline bool Codec<bool>::unpack(Tuple const &val) { return val.getInt(0) == 1; }
template <>
inline Tuple Codec<bool>::pack(bool const& val) {
return Tuple().append(val ? 1 : 0);
}
template <>
inline bool Codec<bool>::unpack(Tuple const& val) {
return val.getInt(0) == 1;
}
template<> inline Tuple Codec<Standalone<StringRef>>::pack(Standalone<StringRef> const &val) { return Tuple().append(val); }
template<> inline Standalone<StringRef> Codec<Standalone<StringRef>>::unpack(Tuple const &val) { return val.getString(0); }
template <>
inline Tuple Codec<Standalone<StringRef>>::pack(Standalone<StringRef> const& val) {
return Tuple().append(val);
}
template <>
inline Standalone<StringRef> Codec<Standalone<StringRef>>::unpack(Tuple const& val) {
return val.getString(0);
}
template<> inline Tuple Codec<UID>::pack(UID const &val) { return Codec<Standalone<StringRef>>::pack(BinaryWriter::toValue<UID>(val, Unversioned())); }
template<> inline UID Codec<UID>::unpack(Tuple const &val) { return BinaryReader::fromStringRef<UID>(Codec<Standalone<StringRef>>::unpack(val), Unversioned()); }
template <>
inline Tuple Codec<UID>::pack(UID const& val) {
return Codec<Standalone<StringRef>>::pack(BinaryWriter::toValue<UID>(val, Unversioned()));
}
template <>
inline UID Codec<UID>::unpack(Tuple const& val) {
return BinaryReader::fromStringRef<UID>(Codec<Standalone<StringRef>>::unpack(val), Unversioned());
}
// This is backward compatible with Codec<Standalone<StringRef>>
template<> inline Tuple Codec<std::string>::pack(std::string const &val) { return Tuple().append(StringRef(val)); }
template<> inline std::string Codec<std::string>::unpack(Tuple const &val) { return val.getString(0).toString(); }
template <>
inline Tuple Codec<std::string>::pack(std::string const& val) {
return Tuple().append(StringRef(val));
}
template <>
inline std::string Codec<std::string>::unpack(Tuple const& val) {
return val.getString(0).toString();
}
// Partial specialization to cover all std::pairs as long as the component types are Codec compatible
template<typename First, typename Second>
template <typename First, typename Second>
struct Codec<std::pair<First, Second>> {
static Tuple pack(typename std::pair<First, Second> const &val) { return Tuple().append(Codec<First>::pack(val.first)).append(Codec<Second>::pack(val.second)); }
static std::pair<First, Second> unpack(Tuple const &t) {
static Tuple pack(typename std::pair<First, Second> const& val) {
return Tuple().append(Codec<First>::pack(val.first)).append(Codec<Second>::pack(val.second));
}
static std::pair<First, Second> unpack(Tuple const& t) {
ASSERT(t.size() == 2);
return {Codec<First>::unpack(t.subTuple(0, 1)), Codec<Second>::unpack(t.subTuple(1, 2))};
return { Codec<First>::unpack(t.subTuple(0, 1)), Codec<Second>::unpack(t.subTuple(1, 2)) };
}
};
template<typename T>
template <typename T>
struct Codec<std::vector<T>> {
static Tuple pack(typename std::vector<T> const &val) {
static Tuple pack(typename std::vector<T> const& val) {
Tuple t;
for (T item : val) {
Tuple itemTuple = Codec<T>::pack(item);
@ -85,7 +123,7 @@ struct Codec<std::vector<T>> {
return t;
}
static std::vector<T> unpack(Tuple const &t) {
static std::vector<T> unpack(Tuple const& t) {
std::vector<T> v;
for (int i = 0; i < t.size(); i++) {
@ -97,8 +135,14 @@ struct Codec<std::vector<T>> {
}
};
template<> inline Tuple Codec<KeyRange>::pack(KeyRange const &val) { return Tuple().append(val.begin).append(val.end); }
template<> inline KeyRange Codec<KeyRange>::unpack(Tuple const &val) { return KeyRangeRef(val.getString(0), val.getString(1)); }
template <>
inline Tuple Codec<KeyRange>::pack(KeyRange const& val) {
return Tuple().append(val.begin).append(val.end);
}
template <>
inline KeyRange Codec<KeyRange>::unpack(Tuple const& val) {
return KeyRangeRef(val.getString(0), val.getString(1));
}
// Convenient read/write access to a single value of type T stored at key
// Even though 'this' is not actually mutated, methods that change the db key are not const.
@ -107,8 +151,8 @@ class KeyBackedProperty {
public:
KeyBackedProperty(KeyRef key) : key(key) {}
Future<Optional<T>> get(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) const {
return map(tr->get(key, snapshot), [](Optional<Value> const &val) -> Optional<T> {
if(val.present())
return map(tr->get(key, snapshot), [](Optional<Value> const& val) -> Optional<T> {
if (val.present())
return Codec<T>::unpack(Tuple::unpack(val.get()));
return {};
});
@ -118,15 +162,17 @@ public:
return map(get(tr, snapshot), [=](Optional<T> val) -> T { return val.present() ? val.get() : defaultValue; });
}
// Get property's value or throw error if it doesn't exist
Future<T> getOrThrow(Reference<ReadYourWritesTransaction> tr, bool snapshot = false, Error err = key_not_found()) const {
Future<T> getOrThrow(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false,
Error err = key_not_found()) const {
auto keyCopy = key;
auto backtrace = platform::get_backtrace();
return map(get(tr, snapshot), [=](Optional<T> val) -> T {
if (!val.present()) {
TraceEvent(SevInfo, "KeyBackedProperty_KeyNotFound")
.detail("Key", keyCopy)
.detail("Err", err.code())
.detail("ParentTrace", backtrace.c_str());
.detail("Key", keyCopy)
.detail("Err", err.code())
.detail("ParentTrace", backtrace.c_str());
throw err;
}
@ -135,7 +181,7 @@ public:
}
Future<Optional<T>> get(Database cx, bool snapshot = false) const {
auto &copy = *this;
auto& copy = *this;
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@ -145,7 +191,7 @@ public:
}
Future<T> getD(Database cx, bool snapshot = false, T defaultValue = T()) const {
auto &copy = *this;
auto& copy = *this;
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@ -155,7 +201,7 @@ public:
}
Future<T> getOrThrow(Database cx, bool snapshot = false, Error err = key_not_found()) const {
auto &copy = *this;
auto& copy = *this;
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
@ -164,11 +210,9 @@ public:
});
}
void set(Reference<ReadYourWritesTransaction> tr, T const &val) {
return tr->set(key, Codec<T>::pack(val).pack());
}
void set(Reference<ReadYourWritesTransaction> tr, T const& val) { return tr->set(key, Codec<T>::pack(val).pack()); }
Future<Void> set(Database cx, T const &val) {
Future<Void> set(Database cx, T const& val) {
auto _key = key;
Value _val = Codec<T>::pack(val).pack();
return runRYWTransaction(cx, [_key, _val](Reference<ReadYourWritesTransaction> tr) {
@ -180,9 +224,7 @@ public:
});
}
void clear(Reference<ReadYourWritesTransaction> tr) {
return tr->clear(key);
}
void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(key); }
Key key;
};
@ -194,8 +236,8 @@ class KeyBackedBinaryValue {
public:
KeyBackedBinaryValue(KeyRef key) : key(key) {}
Future<Optional<T>> get(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) const {
return map(tr->get(key, snapshot), [](Optional<Value> const &val) -> Optional<T> {
if(val.present())
return map(tr->get(key, snapshot), [](Optional<Value> const& val) -> Optional<T> {
if (val.present())
return BinaryReader::fromStringRef<T>(val.get(), Unversioned());
return {};
});
@ -204,15 +246,13 @@ public:
Future<T> getD(Reference<ReadYourWritesTransaction> tr, bool snapshot = false, T defaultValue = T()) const {
return map(get(tr, false), [=](Optional<T> val) -> T { return val.present() ? val.get() : defaultValue; });
}
void set(Reference<ReadYourWritesTransaction> tr, T const &val) {
void set(Reference<ReadYourWritesTransaction> tr, T const& val) {
return tr->set(key, BinaryWriter::toValue<T>(val, Unversioned()));
}
void atomicOp(Reference<ReadYourWritesTransaction> tr, T const &val, MutationRef::Type type) {
void atomicOp(Reference<ReadYourWritesTransaction> tr, T const& val, MutationRef::Type type) {
return tr->atomicOp(key, BinaryWriter::toValue<T>(val, Unversioned()), type);
}
void clear(Reference<ReadYourWritesTransaction> tr) {
return tr->clear(key);
}
void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(key); }
Key key;
};
@ -229,53 +269,59 @@ public:
typedef std::vector<PairType> PairsType;
// If end is not present one key past the end of the map is used.
Future<PairsType> getRange(Reference<ReadYourWritesTransaction> tr, KeyType const &begin, Optional<KeyType> const &end, int limit, bool snapshot = false, bool reverse = false) const {
Subspace s = space; // 'this' could be invalid inside lambda
Future<PairsType> getRange(Reference<ReadYourWritesTransaction> tr,
KeyType const& begin,
Optional<KeyType> const& end,
int limit,
bool snapshot = false,
bool reverse = false) const {
Subspace s = space; // 'this' could be invalid inside lambda
Key endKey = end.present() ? s.pack(Codec<KeyType>::pack(end.get())) : space.range().end;
return map(tr->getRange(KeyRangeRef(s.pack(Codec<KeyType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot, reverse),
[s] (Standalone<RangeResultRef> const &kvs) -> PairsType {
PairsType results;
for(int i = 0; i < kvs.size(); ++i) {
KeyType key = Codec<KeyType>::unpack(s.unpack(kvs[i].key));
ValueType val = Codec<ValueType>::unpack(Tuple::unpack(kvs[i].value));
results.push_back(PairType(key, val));
}
return results;
});
return map(
tr->getRange(
KeyRangeRef(s.pack(Codec<KeyType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot, reverse),
[s](Standalone<RangeResultRef> const& kvs) -> PairsType {
PairsType results;
for (int i = 0; i < kvs.size(); ++i) {
KeyType key = Codec<KeyType>::unpack(s.unpack(kvs[i].key));
ValueType val = Codec<ValueType>::unpack(Tuple::unpack(kvs[i].value));
results.push_back(PairType(key, val));
}
return results;
});
}
Future<Optional<ValueType>> get(Reference<ReadYourWritesTransaction> tr, KeyType const &key, bool snapshot = false) const {
return map(tr->get(space.pack(Codec<KeyType>::pack(key)), snapshot), [](Optional<Value> const &val) -> Optional<ValueType> {
if(val.present())
return Codec<ValueType>::unpack(Tuple::unpack(val.get()));
return {};
});
Future<Optional<ValueType>> get(Reference<ReadYourWritesTransaction> tr,
KeyType const& key,
bool snapshot = false) const {
return map(tr->get(space.pack(Codec<KeyType>::pack(key)), snapshot),
[](Optional<Value> const& val) -> Optional<ValueType> {
if (val.present())
return Codec<ValueType>::unpack(Tuple::unpack(val.get()));
return {};
});
}
// Returns a Property that can be get/set that represents key's entry in this this.
KeyBackedProperty<ValueType> getProperty(KeyType const &key) const {
return space.pack(Codec<KeyType>::pack(key));
}
KeyBackedProperty<ValueType> getProperty(KeyType const& key) const { return space.pack(Codec<KeyType>::pack(key)); }
// Returns the expectedSize of the set key
int set(Reference<ReadYourWritesTransaction> tr, KeyType const &key, ValueType const &val) {
int set(Reference<ReadYourWritesTransaction> tr, KeyType const& key, ValueType const& val) {
Key k = space.pack(Codec<KeyType>::pack(key));
Value v = Codec<ValueType>::pack(val).pack();
tr->set(k, v);
return k.expectedSize() + v.expectedSize();
}
void erase(Reference<ReadYourWritesTransaction> tr, KeyType const &key) {
void erase(Reference<ReadYourWritesTransaction> tr, KeyType const& key) {
return tr->clear(space.pack(Codec<KeyType>::pack(key)));
}
void erase(Reference<ReadYourWritesTransaction> tr, KeyType const &begin, KeyType const &end) {
void erase(Reference<ReadYourWritesTransaction> tr, KeyType const& begin, KeyType const& end) {
return tr->clear(KeyRangeRef(space.pack(Codec<KeyType>::pack(begin)), space.pack(Codec<KeyType>::pack(end))));
}
void clear(Reference<ReadYourWritesTransaction> tr) {
return tr->clear(space.range());
}
void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(space.range()); }
Subspace space;
};
@ -289,43 +335,46 @@ public:
typedef std::vector<ValueType> Values;
// If end is not present one key past the end of the map is used.
Future<Values> getRange(Reference<ReadYourWritesTransaction> tr, ValueType const &begin, Optional<ValueType> const &end, int limit, bool snapshot = false) const {
Subspace s = space; // 'this' could be invalid inside lambda
Future<Values> getRange(Reference<ReadYourWritesTransaction> tr,
ValueType const& begin,
Optional<ValueType> const& end,
int limit,
bool snapshot = false) const {
Subspace s = space; // 'this' could be invalid inside lambda
Key endKey = end.present() ? s.pack(Codec<ValueType>::pack(end.get())) : space.range().end;
return map(tr->getRange(KeyRangeRef(s.pack(Codec<ValueType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot),
[s] (Standalone<RangeResultRef> const &kvs) -> Values {
Values results;
for(int i = 0; i < kvs.size(); ++i) {
results.push_back(Codec<ValueType>::unpack(s.unpack(kvs[i].key)));
}
return results;
});
return map(
tr->getRange(KeyRangeRef(s.pack(Codec<ValueType>::pack(begin)), endKey), GetRangeLimits(limit), snapshot),
[s](Standalone<RangeResultRef> const& kvs) -> Values {
Values results;
for (int i = 0; i < kvs.size(); ++i) {
results.push_back(Codec<ValueType>::unpack(s.unpack(kvs[i].key)));
}
return results;
});
}
Future<bool> exists(Reference<ReadYourWritesTransaction> tr, ValueType const &val, bool snapshot = false) const {
return map(tr->get(space.pack(Codec<ValueType>::pack(val)), snapshot), [](Optional<Value> const &val) -> bool {
return val.present();
});
Future<bool> exists(Reference<ReadYourWritesTransaction> tr, ValueType const& val, bool snapshot = false) const {
return map(tr->get(space.pack(Codec<ValueType>::pack(val)), snapshot),
[](Optional<Value> const& val) -> bool { return val.present(); });
}
// Returns the expectedSize of the set key
int insert(Reference<ReadYourWritesTransaction> tr, ValueType const &val) {
int insert(Reference<ReadYourWritesTransaction> tr, ValueType const& val) {
Key k = space.pack(Codec<ValueType>::pack(val));
tr->set(k, StringRef());
return k.expectedSize();
}
void erase(Reference<ReadYourWritesTransaction> tr, ValueType const &val) {
void erase(Reference<ReadYourWritesTransaction> tr, ValueType const& val) {
return tr->clear(space.pack(Codec<ValueType>::pack(val)));
}
void erase(Reference<ReadYourWritesTransaction> tr, ValueType const &begin, ValueType const &end) {
return tr->clear(KeyRangeRef(space.pack(Codec<ValueType>::pack(begin)), space.pack(Codec<ValueType>::pack(end))));
void erase(Reference<ReadYourWritesTransaction> tr, ValueType const& begin, ValueType const& end) {
return tr->clear(
KeyRangeRef(space.pack(Codec<ValueType>::pack(begin)), space.pack(Codec<ValueType>::pack(end))));
}
void clear(Reference<ReadYourWritesTransaction> tr) {
return tr->clear(space.range());
}
void clear(Reference<ReadYourWritesTransaction> tr) { return tr->clear(space.range()); }
Subspace space;
};

204
fdbclient/KeyRangeMap.actor.cpp
View File

@ -25,23 +25,20 @@
#include "fdbclient/ReadYourWrites.h"
#include "flow/actorcompiler.h" // has to be last include
void KeyRangeActorMap::getRangesAffectedByInsertion( const KeyRangeRef& keys, vector< KeyRange >& affectedRanges ) {
auto s = map.rangeContaining( keys.begin );
void KeyRangeActorMap::getRangesAffectedByInsertion(const KeyRangeRef& keys, vector<KeyRange>& affectedRanges) {
auto s = map.rangeContaining(keys.begin);
if (s.begin() != keys.begin && s.value().isValid() && !s.value().isReady())
affectedRanges.push_back( KeyRangeRef( s.begin(), keys.begin ) );
affectedRanges.push_back( keys );
auto e = map.rangeContaining( keys.end );
affectedRanges.push_back(KeyRangeRef(s.begin(), keys.begin));
affectedRanges.push_back(keys);
auto e = map.rangeContaining(keys.end);
if (e.begin() != keys.end && e.value().isValid() && !e.value().isReady())
affectedRanges.push_back( KeyRangeRef( keys.end, e.end() ) );
affectedRanges.push_back(KeyRangeRef(keys.end, e.end()));
}
Standalone<RangeResultRef> krmDecodeRanges(
KeyRef mapPrefix,
KeyRange keys,
Standalone<RangeResultRef> kv )
{
Standalone<RangeResultRef> krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, Standalone<RangeResultRef> kv) {
ASSERT(!kv.more || kv.size() > 1);
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
ValueRef beginValue, endValue;
if (kv.size() && kv[0].key.startsWith(mapPrefix))
@ -50,20 +47,19 @@ Standalone<RangeResultRef> krmDecodeRanges(
endValue = kv.end()[-1].value;
Standalone<RangeResultRef> result;
result.arena().dependsOn( kv.arena() );
result.arena().dependsOn( keys.arena() );
result.arena().dependsOn(kv.arena());
result.arena().dependsOn(keys.arena());
result.push_back(result.arena(), KeyValueRef(keys.begin, beginValue));
for(int i=0; i<kv.size(); i++) {
for (int i = 0; i < kv.size(); i++) {
if (kv[i].key > withPrefix.begin && kv[i].key < withPrefix.end) {
KeyRef k = kv[i].key.removePrefix(mapPrefix);
result.push_back(result.arena(), KeyValueRef( k, kv[i].value ));
}
else if(kv[i].key >= withPrefix.end)
result.push_back(result.arena(), KeyValueRef(k, kv[i].value));
} else if (kv[i].key >= withPrefix.end)
kv.more = false;
}
if(!kv.more)
if (!kv.more)
result.push_back(result.arena(), KeyValueRef(keys.end, endValue));
result.more = kv.more;
@ -71,140 +67,170 @@ Standalone<RangeResultRef> krmDecodeRanges(
}
// Returns keys.begin, all transitional points in keys, and keys.end, and their values
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges( Transaction* tr, Key mapPrefix, KeyRange keys, int limit, int limitBytes )
{
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges(Transaction* tr,
Key mapPrefix,
KeyRange keys,
int limit,
int limitBytes) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
state GetRangeLimits limits(limit, limitBytes);
limits.minRows = 2;
Standalone<RangeResultRef> kv = wait( tr->getRange( lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits ) );
Standalone<RangeResultRef> kv =
wait(tr->getRange(lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits));
return krmDecodeRanges( mapPrefix, keys, kv );
return krmDecodeRanges(mapPrefix, keys, kv);
}
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges( Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange keys, int limit, int limitBytes )
{
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
ACTOR Future<Standalone<RangeResultRef>> krmGetRanges(Reference<ReadYourWritesTransaction> tr,
Key mapPrefix,
KeyRange keys,
int limit,
int limitBytes) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
state GetRangeLimits limits(limit, limitBytes);
limits.minRows = 2;
Standalone<RangeResultRef> kv = wait( tr->getRange( lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits ) );
Standalone<RangeResultRef> kv =
wait(tr->getRange(lastLessOrEqual(withPrefix.begin), firstGreaterThan(withPrefix.end), limits));
return krmDecodeRanges( mapPrefix, keys, kv );
return krmDecodeRanges(mapPrefix, keys, kv);
}
void krmSetPreviouslyEmptyRange( Transaction* tr, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue )
{
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
tr->set( withPrefix.begin, newValue );
tr->set( withPrefix.end, oldEndValue );
void krmSetPreviouslyEmptyRange(Transaction* tr,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
tr->set(withPrefix.begin, newValue);
tr->set(withPrefix.end, oldEndValue);
}
void krmSetPreviouslyEmptyRange( CommitTransactionRef& tr, Arena& trArena, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue )
{
KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString() );
tr.set( trArena, withPrefix.begin, newValue );
tr.set( trArena, withPrefix.end, oldEndValue );
void krmSetPreviouslyEmptyRange(CommitTransactionRef& tr,
Arena& trArena,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue) {
KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + keys.begin.toString(), mapPrefix.toString() + keys.end.toString());
tr.set(trArena, withPrefix.begin, newValue);
tr.set(trArena, withPrefix.end, oldEndValue);
}
ACTOR Future<Void> krmSetRange( Transaction* tr, Key mapPrefix, KeyRange range, Value value ) {
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() );
Standalone<RangeResultRef> old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
ACTOR Future<Void> krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, Value value) {
state KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
Standalone<RangeResultRef> old =
wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
Value oldValue;
bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix);
if(hasResult)
if (hasResult)
oldValue = old[0].value;
KeyRange conflictRange = KeyRangeRef( hasResult ? old[0].key : mapPrefix.toString(), keyAfter(withPrefix.end) );
if( !conflictRange.empty() )
tr->addReadConflictRange( conflictRange );
KeyRange conflictRange = KeyRangeRef(hasResult ? old[0].key : mapPrefix.toString(), keyAfter(withPrefix.end));
if (!conflictRange.empty())
tr->addReadConflictRange(conflictRange);
tr->clear(withPrefix);
tr->set( withPrefix.begin, value );
tr->set( withPrefix.end, oldValue );
tr->set(withPrefix.begin, value);
tr->set(withPrefix.end, oldValue);
return Void();
}
ACTOR Future<Void> krmSetRange( Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange range, Value value ) {
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() );
Standalone<RangeResultRef> old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
ACTOR Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> tr, Key mapPrefix, KeyRange range, Value value) {
state KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
Standalone<RangeResultRef> old =
wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true));
Value oldValue;
bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix);
if(hasResult)
if (hasResult)
oldValue = old[0].value;
KeyRange conflictRange = KeyRangeRef( hasResult ? old[0].key : mapPrefix.toString(), keyAfter(withPrefix.end) );
if( !conflictRange.empty() )
tr->addReadConflictRange( conflictRange );
KeyRange conflictRange = KeyRangeRef(hasResult ? old[0].key : mapPrefix.toString(), keyAfter(withPrefix.end));
if (!conflictRange.empty())
tr->addReadConflictRange(conflictRange);
tr->clear(withPrefix);
tr->set( withPrefix.begin, value );
tr->set( withPrefix.end, oldValue );
tr->set(withPrefix.begin, value);
tr->set(withPrefix.end, oldValue);
return Void();
}
//Sets a range of keys in a key range map, coalescing with adjacent regions if the values match
//Ranges outside of maxRange will not be coalesced
//CAUTION: use care when attempting to coalesce multiple ranges in the same prefix in a single transaction
// Sets a range of keys in a key range map, coalescing with adjacent regions if the values match
// Ranges outside of maxRange will not be coalesced
// CAUTION: use care when attempting to coalesce multiple ranges in the same prefix in a single transaction
ACTOR template <class Transaction>
static Future<Void> krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRange range, KeyRange maxRange,
static Future<Void> krmSetRangeCoalescing_(Transaction* tr,
Key mapPrefix,
KeyRange range,
KeyRange maxRange,
Value value) {
ASSERT(maxRange.contains(range));
state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() );
state KeyRange maxWithPrefix = KeyRangeRef( mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString() );
state KeyRange withPrefix =
KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString());
state KeyRange maxWithPrefix =
KeyRangeRef(mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString());
state vector<Future<Standalone<RangeResultRef>>> keys;
keys.push_back(tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, true));
keys.push_back(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end) + 1, 2, true));
wait(waitForAll(keys));
//Determine how far to extend this range at the beginning
// Determine how far to extend this range at the beginning
auto beginRange = keys[0].get();
bool hasBegin = beginRange.size() > 0 && beginRange[0].key.startsWith(mapPrefix);
Value beginValue = hasBegin ? beginRange[0].value : LiteralStringRef("");
state Key beginKey = withPrefix.begin;
if(beginValue == value) {
if (beginValue == value) {
bool outsideRange = !hasBegin || beginRange[0].key < maxWithPrefix.begin;
beginKey = outsideRange ? maxWithPrefix.begin : beginRange[0].key;
}
//Determine how far to extend this range at the end
// Determine how far to extend this range at the end
auto endRange = keys[1].get();
bool hasEnd = endRange.size() >= 1 && endRange[0].key.startsWith(mapPrefix) && endRange[0].key <= withPrefix.end;
bool hasNext = (endRange.size() == 2 && endRange[1].key.startsWith(mapPrefix)) || (endRange.size() == 1 && withPrefix.end < endRange[0].key && endRange[0].key.startsWith(mapPrefix));
bool hasNext = (endRange.size() == 2 && endRange[1].key.startsWith(mapPrefix)) ||
(endRange.size() == 1 && withPrefix.end < endRange[0].key && endRange[0].key.startsWith(mapPrefix));
Value existingValue = hasEnd ? endRange[0].value : LiteralStringRef("");
bool valueMatches = value == existingValue;
KeyRange conflictRange = KeyRangeRef( hasBegin ? beginRange[0].key : mapPrefix, withPrefix.begin );
if( !conflictRange.empty() )
tr->addReadConflictRange( conflictRange );
KeyRange conflictRange = KeyRangeRef(hasBegin ? beginRange[0].key : mapPrefix, withPrefix.begin);
if (!conflictRange.empty())
tr->addReadConflictRange(conflictRange);
conflictRange = KeyRangeRef( hasEnd ? endRange[0].key : mapPrefix, hasNext ? keyAfter(endRange.end()[-1].key) : strinc( mapPrefix ) );
if( !conflictRange.empty() )
tr->addReadConflictRange( conflictRange );
conflictRange = KeyRangeRef(hasEnd ? endRange[0].key : mapPrefix,
hasNext ? keyAfter(endRange.end()[-1].key) : strinc(mapPrefix));
if (!conflictRange.empty())
tr->addReadConflictRange(conflictRange);
state Key endKey;
state Value endValue;
//Case 1: Coalesce completely with the following range
if(hasNext && endRange.end()[-1].key <= maxWithPrefix.end && valueMatches) {
// Case 1: Coalesce completely with the following range
if (hasNext && endRange.end()[-1].key <= maxWithPrefix.end && valueMatches) {
endKey = endRange.end()[-1].key;
endValue = endRange.end()[-1].value;
}
//Case 2: Coalesce with the following range only up to the end of maxRange
else if(valueMatches) {
// Case 2: Coalesce with the following range only up to the end of maxRange
else if (valueMatches) {
endKey = maxWithPrefix.end;
endValue = existingValue;
}
//Case 3: Don't coalesce
// Case 3: Don't coalesce
else {
endKey = withPrefix.end;
endValue = existingValue;
@ -218,11 +244,17 @@ static Future<Void> krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRa
return Void();
}
Future<Void> krmSetRangeCoalescing(Transaction* const& tr, Key const& mapPrefix, KeyRange const& range,
KeyRange const& maxRange, Value const& value) {
Future<Void> krmSetRangeCoalescing(Transaction* const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value) {
return krmSetRangeCoalescing_(tr, mapPrefix, range, maxRange, value);
}
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix,
KeyRange const& range, KeyRange const& maxRange, Value const& value) {
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value) {
return holdWhile(tr, krmSetRangeCoalescing_(tr.getPtr(), mapPrefix, range, maxRange, value));
}

298
fdbclient/KeyRangeMap.h
View File

@ -32,300 +32,354 @@
using boost::iterator_range;
template <class Val, class Metric=int, class MetricFunc = ConstantMetric<Metric>>
class KeyRangeMap : public RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>, NonCopyable, public ReferenceCounted<KeyRangeMap<Val>> {
template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class KeyRangeMap : public RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>,
NonCopyable,
public ReferenceCounted<KeyRangeMap<Val>> {
public:
explicit KeyRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {}
explicit KeyRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(KeyRangeMap&& r) noexcept {
mapEnd = std::move(r.mapEnd);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
}
void insert( const KeyRangeRef& keys, const Val& value ) { RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::insert(keys, value); }
void insert( const KeyRef& key, const Val& value ) { RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::insert( singleKeyRange(key), value); }
std::vector<KeyRangeWith<Val>> getAffectedRangesAfterInsertion( const KeyRangeRef& keys, const Val &insertionValue = Val());
typename RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::Ranges modify( const KeyRangeRef& keys ) // Returns ranges, the first of which begins at keys.begin and the last of which ends at keys.end
void insert(const KeyRangeRef& keys, const Val& value) {
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::insert(keys, value);
}
void insert(const KeyRef& key, const Val& value) {
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::insert(singleKeyRange(key), value);
}
std::vector<KeyRangeWith<Val>> getAffectedRangesAfterInsertion(const KeyRangeRef& keys,
const Val& insertionValue = Val());
typename RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::Ranges modify(
const KeyRangeRef&
keys) // Returns ranges, the first of which begins at keys.begin and the last of which ends at keys.end
{
MapPair<Key,Val> valueBeforeRange(keys.begin, RangeMap<Key,Val,KeyRangeRef,Metric>::rangeContaining(keys.begin).value());
MapPair<Key,Val> valueAfterRange(keys.end, RangeMap<Key,Val,KeyRangeRef,Metric>::rangeContaining(keys.end).value());
MapPair<Key, Val> valueBeforeRange(
keys.begin, RangeMap<Key, Val, KeyRangeRef, Metric>::rangeContaining(keys.begin).value());
MapPair<Key, Val> valueAfterRange(keys.end,
RangeMap<Key, Val, KeyRangeRef, Metric>::rangeContaining(keys.end).value());
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(std::move(valueBeforeRange));
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(std::move(valueAfterRange));
return RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::intersectingRanges( keys );
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(std::move(valueBeforeRange));
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(std::move(valueAfterRange));
return RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::intersectingRanges(keys);
}
void rawErase( KeyRange const& range ) {
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound(range.begin), RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound(range.end));
void rawErase(KeyRange const& range) {
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(range.begin),
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(range.end));
}
void rawInsert( Key const& key, Val const& value ) {
MapPair<Key,Val> pair(key, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(pair, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(pair));
void rawInsert(Key const& key, Val const& value) {
MapPair<Key, Val> pair(key, value);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
pair, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(pair));
}
void rawInsert( const std::vector<std::pair<MapPair<Key,Val>, Metric>>& pairs ) {
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(pairs);
void rawInsert(const std::vector<std::pair<MapPair<Key, Val>, Metric>>& pairs) {
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(pairs);
}
Key mapEnd;
};
template <class Val, class Metric=int, class MetricFunc = ConstantMetric<Metric>>
class CoalescedKeyRefRangeMap : public RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>, NonCopyable {
template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class CoalescedKeyRefRangeMap : public RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>, NonCopyable {
public:
explicit CoalescedKeyRefRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {}
explicit CoalescedKeyRefRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(CoalescedKeyRefRangeMap&& r) noexcept {
mapEnd = std::move(r.mapEnd);
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
}
void insert( const KeyRangeRef& keys, const Val& value );
void insert( const KeyRef& key, const Val& value, Arena& arena );
void insert(const KeyRangeRef& keys, const Val& value);
void insert(const KeyRef& key, const Val& value, Arena& arena);
Key mapEnd;
};
template <class Val, class Metric=int, class MetricFunc = ConstantMetric<Metric>>
class CoalescedKeyRangeMap : public RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>, NonCopyable {
template <class Val, class Metric = int, class MetricFunc = ConstantMetric<Metric>>
class CoalescedKeyRangeMap : public RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>, NonCopyable {
public:
explicit CoalescedKeyRangeMap(Val v=Val(), Key endKey = allKeys.end) : RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>(endKey, v), mapEnd(endKey) {}
explicit CoalescedKeyRangeMap(Val v = Val(), Key endKey = allKeys.end)
: RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>(endKey, v), mapEnd(endKey) {}
void operator=(CoalescedKeyRangeMap&& r) noexcept {
mapEnd = std::move(r.mapEnd);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::operator=(std::move(r));
}
void insert( const KeyRangeRef& keys, const Val& value );
void insert( const KeyRef& key, const Val& value );
void insert(const KeyRangeRef& keys, const Val& value);
void insert(const KeyRef& key, const Val& value);
Key mapEnd;
};
class KeyRangeActorMap {
public:
void getRangesAffectedByInsertion( const KeyRangeRef& keys, vector< KeyRange >& affectedRanges );
void insert( const KeyRangeRef& keys, const Future<Void>& value ) { map.insert( keys, value ); }
void cancel( const KeyRangeRef& keys ) { insert( keys, Future<Void>() ); }
bool liveActorAt( const KeyRef& key ) { Future<Void> actorAt = map[key]; return actorAt.isValid() && !actorAt.isReady(); }
void getRangesAffectedByInsertion(const KeyRangeRef& keys, vector<KeyRange>& affectedRanges);
void insert(const KeyRangeRef& keys, const Future<Void>& value) { map.insert(keys, value); }
void cancel(const KeyRangeRef& keys) { insert(keys, Future<Void>()); }
bool liveActorAt(const KeyRef& key) {
Future<Void> actorAt = map[key];
return actorAt.isValid() && !actorAt.isReady();
}
private:
KeyRangeMap< Future<Void> > map;
KeyRangeMap<Future<Void>> map;
};
// krm*(): KeyRangeMap-like abstraction stored in the database, accessed through Transactions
class Transaction;
class ReadYourWritesTransaction;
Future<Standalone<RangeResultRef>> krmGetRanges( Transaction* const& tr, Key const& mapPrefix, KeyRange const& keys, int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT, int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES );
Future<Standalone<RangeResultRef>> krmGetRanges( Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix, KeyRange const& keys, int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT, int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES );
void krmSetPreviouslyEmptyRange( Transaction* tr, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue );
void krmSetPreviouslyEmptyRange( struct CommitTransactionRef& tr, Arena& trArena, const KeyRef& mapPrefix, const KeyRangeRef& keys, const ValueRef& newValue, const ValueRef& oldEndValue );
Future<Void> krmSetRange( Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value );
Future<Void> krmSetRange( Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value );
Future<Void> krmSetRangeCoalescing( Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, KeyRange const& maxRange, Value const& value );
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr, Key const& mapPrefix,
KeyRange const& range, KeyRange const& maxRange, Value const& value);
Standalone<RangeResultRef> krmDecodeRanges( KeyRef mapPrefix, KeyRange keys, Standalone<RangeResultRef> kv );
Future<Standalone<RangeResultRef>> krmGetRanges(Transaction* const& tr,
Key const& mapPrefix,
KeyRange const& keys,
int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
Future<Standalone<RangeResultRef>> krmGetRanges(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& keys,
int const& limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT,
int const& limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES);
void krmSetPreviouslyEmptyRange(Transaction* tr,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue);
void krmSetPreviouslyEmptyRange(struct CommitTransactionRef& tr,
Arena& trArena,
const KeyRef& mapPrefix,
const KeyRangeRef& keys,
const ValueRef& newValue,
const ValueRef& oldEndValue);
Future<Void> krmSetRange(Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value);
Future<Void> krmSetRange(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& range,
Value const& value);
Future<Void> krmSetRangeCoalescing(Transaction* const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value);
Future<Void> krmSetRangeCoalescing(Reference<ReadYourWritesTransaction> const& tr,
Key const& mapPrefix,
KeyRange const& range,
KeyRange const& maxRange,
Value const& value);
Standalone<RangeResultRef> krmDecodeRanges(KeyRef mapPrefix, KeyRange keys, Standalone<RangeResultRef> kv);
template <class Val, class Metric, class MetricFunc>
std::vector<KeyRangeWith<Val>> KeyRangeMap<Val,Metric,MetricFunc>::getAffectedRangesAfterInsertion( const KeyRangeRef& keys, const Val &insertionValue) {
std::vector<KeyRangeWith<Val>> KeyRangeMap<Val, Metric, MetricFunc>::getAffectedRangesAfterInsertion(
const KeyRangeRef& keys,
const Val& insertionValue) {
std::vector<KeyRangeWith<Val>> affectedRanges;
{ // possible first range if no exact alignment
auto r = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::rangeContaining( keys.begin );
auto r = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::rangeContaining(keys.begin);
if (r->begin() != keys.begin)
affectedRanges.push_back(
KeyRangeWith<Val>(
KeyRangeRef(r->begin(), keys.begin), r->value() ) );
affectedRanges.push_back(KeyRangeWith<Val>(KeyRangeRef(r->begin(), keys.begin), r->value()));
}
affectedRanges.push_back( KeyRangeWith<Val>( keys, insertionValue) );
affectedRanges.push_back(KeyRangeWith<Val>(keys, insertionValue));
{ // possible last range if no exact alignment
auto r = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::rangeContaining( keys.end );
auto r = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::rangeContaining(keys.end);
if (r->begin() != keys.end)
affectedRanges.push_back(
KeyRangeWith<Val>(
KeyRangeRef(keys.end, r->end()), r->value() ) );
affectedRanges.push_back(KeyRangeWith<Val>(KeyRangeRef(keys.end, r->end()), r->value()));
}
return affectedRanges;
}
template <class Val, class Metric, class MetricFunc>
void CoalescedKeyRangeMap<Val,Metric,MetricFunc>::insert( const KeyRangeRef& keys, const Val& value ) {
void CoalescedKeyRangeMap<Val, Metric, MetricFunc>::insert(const KeyRangeRef& keys, const Val& value) {
ASSERT(keys.end <= mapEnd);
if( keys.empty() )
if (keys.empty())
return;
auto begin = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( keys.begin );
auto end = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( keys.end );
auto begin = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(keys.begin);
auto end = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(keys.end);
bool insertEnd = false;
bool insertBegin = false;
Val endVal;
if( keys.end != mapEnd ) {
if( end->key != keys.end ) {
if (keys.end != mapEnd) {
if (end->key != keys.end) {
auto before_end = end;
before_end.decrementNonEnd();
if( value != before_end->value ) {
if (value != before_end->value) {
insertEnd = true;
endVal = before_end->value;
}
}
if( !insertEnd && end->value == value && end->key != mapEnd ) {
if (!insertEnd && end->value == value && end->key != mapEnd) {
++end;
}
}
if( keys.begin == allKeys.begin ) {
if (keys.begin == allKeys.begin) {
insertBegin = true;
} else {
auto before_begin = begin;
before_begin.decrementNonEnd();
if( before_begin->value != value )
if (before_begin->value != value)
insertBegin = true;
}
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(begin, end);
if(insertEnd) {
MapPair<Key,Val> p(keys.end, endVal);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) {
MapPair<Key, Val> p(keys.end, endVal);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
if(insertBegin) {
MapPair<Key,Val> p(keys.begin, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
if (insertBegin) {
MapPair<Key, Val> p(keys.begin, value);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
}
template <class Val, class Metric, class MetricFunc>
void CoalescedKeyRangeMap<Val,Metric,MetricFunc>::insert( const KeyRef& key, const Val& value ) {
void CoalescedKeyRangeMap<Val, Metric, MetricFunc>::insert(const KeyRef& key, const Val& value) {
ASSERT(key < mapEnd);
auto begin = RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( key );
auto begin = RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(key);
auto end = begin;
if( end->key == key )
if (end->key == key)
++end;
bool insertEnd = false;
bool insertBegin = false;
Val endVal;
if( !equalsKeyAfter( key, end->key ) ) {
if (!equalsKeyAfter(key, end->key)) {
auto before_end = end;
before_end.decrementNonEnd();
if( value != before_end->value ) {
if (value != before_end->value) {
insertEnd = true;
endVal = before_end->value;
}
}
if( !insertEnd && end->value == value && end->key != mapEnd ) {
if (!insertEnd && end->value == value && end->key != mapEnd) {
++end;
}
if( key == allKeys.begin ) {
if (key == allKeys.begin) {
insertBegin = true;
} else {
auto before_begin = begin;
before_begin.decrementNonEnd();
if( before_begin->value != value )
if (before_begin->value != value)
insertBegin = true;
}
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(begin, end);
if(insertEnd) {
MapPair<Key,Val> p(keyAfter(key), endVal);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) {
MapPair<Key, Val> p(keyAfter(key), endVal);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
if(insertBegin) {
MapPair<Key,Val> p(key, value);
RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<Key,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
if (insertBegin) {
MapPair<Key, Val> p(key, value);
RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<Key, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
}
template <class Val, class Metric, class MetricFunc>
void CoalescedKeyRefRangeMap<Val,Metric,MetricFunc>::insert( const KeyRangeRef& keys, const Val& value ) {
void CoalescedKeyRefRangeMap<Val, Metric, MetricFunc>::insert(const KeyRangeRef& keys, const Val& value) {
ASSERT(keys.end <= mapEnd);
if( keys.empty() )
if (keys.empty())
return;
auto begin = RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( keys.begin );
auto end = RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( keys.end );
auto begin = RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(keys.begin);
auto end = RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(keys.end);
bool insertEnd = false;
bool insertBegin = false;
Val endVal;
if( keys.end != mapEnd ) {
if( end->key != keys.end ) {
if (keys.end != mapEnd) {
if (end->key != keys.end) {
auto before_end = end;
before_end.decrementNonEnd();
if( value != before_end->value ) {
if (value != before_end->value) {
insertEnd = true;
endVal = before_end->value;
}
}
if( !insertEnd && end->value == value && end->key != mapEnd ) {
if (!insertEnd && end->value == value && end->key != mapEnd) {
++end;
}
}
if( keys.begin == allKeys.begin ) {
if (keys.begin == allKeys.begin) {
insertBegin = true;
} else {
auto before_begin = begin;
before_begin.decrementNonEnd();
if( before_begin->value != value )
if (before_begin->value != value)
insertBegin = true;
}
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(begin, end);
if(insertEnd) {
MapPair<KeyRef,Val> p(keys.end, endVal);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) {
MapPair<KeyRef, Val> p(keys.end, endVal);
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
if(insertBegin) {
MapPair<KeyRef,Val> p(keys.begin, value);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
if (insertBegin) {
MapPair<KeyRef, Val> p(keys.begin, value);
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
}
template <class Val, class Metric, class MetricFunc>
void CoalescedKeyRefRangeMap<Val,Metric,MetricFunc>::insert( const KeyRef& key, const Val& value, Arena& arena ) {
void CoalescedKeyRefRangeMap<Val, Metric, MetricFunc>::insert(const KeyRef& key, const Val& value, Arena& arena) {
ASSERT(key < mapEnd);
auto begin = RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.lower_bound( key );
auto begin = RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.lower_bound(key);
auto end = begin;
if( end->key == key )
if (end->key == key)
++end;
bool insertEnd = false;
bool insertBegin = false;
Val endVal;
if( !equalsKeyAfter( key, end->key ) ) {
if (!equalsKeyAfter(key, end->key)) {
auto before_end = end;
before_end.decrementNonEnd();
if( value != before_end->value ) {
if (value != before_end->value) {
insertEnd = true;
endVal = before_end->value;
}
}
if( !insertEnd && end->value == value && end->key != mapEnd ) {
if (!insertEnd && end->value == value && end->key != mapEnd) {
++end;
}
if( key == allKeys.begin ) {
if (key == allKeys.begin) {
insertBegin = true;
} else {
auto before_begin = begin;
before_begin.decrementNonEnd();
if( before_begin->value != value )
if (before_begin->value != value)
insertBegin = true;
}
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.erase(begin, end);
if(insertEnd) {
MapPair<KeyRef,Val> p(keyAfter(key, arena), endVal);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.erase(begin, end);
if (insertEnd) {
MapPair<KeyRef, Val> p(keyAfter(key, arena), endVal);
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
if(insertBegin) {
MapPair<KeyRef,Val> p(key,value);
RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::map.insert(p, true, RangeMap<KeyRef,Val,KeyRangeRef,Metric,MetricFunc>::mf(p));
if (insertBegin) {
MapPair<KeyRef, Val> p(key, value);
RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::map.insert(
p, true, RangeMap<KeyRef, Val, KeyRangeRef, Metric, MetricFunc>::mf(p));
}
}

2
fdbclient/Knobs.cpp
View File

@ -26,7 +26,7 @@
std::unique_ptr<ClientKnobs> globalClientKnobs = std::make_unique<ClientKnobs>();
ClientKnobs const* CLIENT_KNOBS = globalClientKnobs.get();
#define init( knob, value ) initKnob( knob, value, #knob )
#define init(knob, value) initKnob(knob, value, #knob)
ClientKnobs::ClientKnobs() {
initialize();

14
fdbclient/Knobs.h
View File

@ -27,7 +27,6 @@
class ClientKnobs : public Knobs {
public:
int TOO_MANY; // FIXME: this should really be split up so we can control these more specifically
double SYSTEM_MONITOR_INTERVAL;
@ -51,7 +50,8 @@ public:
double STATUS_IDLE_TIMEOUT;
// wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin
double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is mostly wrong (e.g. dumping the database after a test)
double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is
// mostly wrong (e.g. dumping the database after a test)
double FUTURE_VERSION_RETRY_DELAY;
int REPLY_BYTE_LIMIT;
double DEFAULT_BACKOFF;
@ -89,12 +89,13 @@ public:
double MID_SHARD_SIZE_MAX_STALENESS;
bool TAG_ENCODE_KEY_SERVERS;
//KeyRangeMap
// KeyRangeMap
int KRM_GET_RANGE_LIMIT;
int KRM_GET_RANGE_LIMIT_BYTES; //This must be sufficiently larger than KEY_SIZE_LIMIT to ensure that at least two entries will be returned from an attempt to read a key range map
int KRM_GET_RANGE_LIMIT_BYTES; // This must be sufficiently larger than KEY_SIZE_LIMIT to ensure that at least two
// entries will be returned from an attempt to read a key range map
int DEFAULT_MAX_OUTSTANDING_WATCHES;
int ABSOLUTE_MAX_WATCHES; //The client cannot set the max outstanding watches higher than this
int ABSOLUTE_MAX_WATCHES; // The client cannot set the max outstanding watches higher than this
double WATCH_POLLING_TIME;
double NO_RECENT_UPDATES_DURATION;
double FAST_WATCH_TIMEOUT;
@ -102,9 +103,8 @@ public:
double IS_ACCEPTABLE_DELAY;
// Core
int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value
int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value
int LOG_RANGE_BLOCK_SIZE;
int MUTATION_BLOCK_SIZE;

1392
fdbclient/ManagementAPI.actor.cpp
View File

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More