wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:apple/foundationdb into add-dd-and-maintenance

This commit is contained in:
Chaoguang Lin 2021-03-26 12:22:37 -07:00
parent 0eff74f205 244c537bc6
commit 747fa2a353
638 changed files with 121351 additions and 102614 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.clang-format
View File

@ -11,14 +11,14 @@ AllowAllParametersOfDeclarationOnNextLine: false
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: Inline
AllowShortIfStatementsOnASingleLine: true
AllowShortLoopsOnASingleLine: true
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: true
BinPackArguments: true
BinPackParameters: true
BinPackArguments: false
BinPackParameters: false
BreakBeforeBinaryOperators: None
BreakBeforeBraces: Attach
ColumnLimit: 120

3
.gitignore vendored
View File

@ -70,6 +70,8 @@ trace.*.xml
*.user
.idea/
.project
.projectile
.dir-locals.el
.pydevproject
.vscode
.vs/
@ -85,6 +87,7 @@ flow/coveragetool/obj
/compile_commands.json
/.ccls-cache
/.clangd
/.cache
# Temporary and user configuration files
*~

7
FDBLibTLS/FDBLibTLSPlugin.cpp
View File

@ -31,10 +31,9 @@ FDBLibTLSPlugin::FDBLibTLSPlugin() {
rc = tls_init();
}
FDBLibTLSPlugin::~FDBLibTLSPlugin() {
}
FDBLibTLSPlugin::~FDBLibTLSPlugin() {}
ITLSPolicy *FDBLibTLSPlugin::create_policy() {
ITLSPolicy* FDBLibTLSPlugin::create_policy() {
if (rc < 0) {
// Log the failure from tls_init during our constructor.
TraceEvent(SevError, "FDBLibTLSInitError").detail("LibTLSErrorMessage", "failed to initialize libtls");
@ -43,7 +42,7 @@ ITLSPolicy *FDBLibTLSPlugin::create_policy() {
return new FDBLibTLSPolicy(Reference<FDBLibTLSPlugin>::addRef(this));
}
extern "C" BOOST_SYMBOL_EXPORT void *get_tls_plugin(const char *plugin_type_name_and_version) {
extern "C" BOOST_SYMBOL_EXPORT void* get_tls_plugin(const char* plugin_type_name_and_version) {
if (strcmp(plugin_type_name_and_version, FDBLibTLSPlugin::get_plugin_type_name_and_version()) == 0) {
return new FDBLibTLSPlugin;
}

2
FDBLibTLS/FDBLibTLSPlugin.h
View File

@ -35,7 +35,7 @@ struct FDBLibTLSPlugin : ITLSPlugin, ReferenceCounted<FDBLibTLSPlugin> {
virtual void addref() { ReferenceCounted<FDBLibTLSPlugin>::addref(); }
virtual void delref() { ReferenceCounted<FDBLibTLSPlugin>::delref(); }
virtual ITLSPolicy *create_policy();
virtual ITLSPolicy* create_policy();
int rc;
};

51
FDBLibTLS/FDBLibTLSPolicy.cpp
View File

@ -55,8 +55,13 @@ FDBLibTLSPolicy::~FDBLibTLSPolicy() {
tls_config_free(tls_cfg);
}
ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func,
void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid) {
ITLSSession* FDBLibTLSPolicy::create_session(bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid) {
if (is_client) {
// If verify peers has been set then there is no point specifying a
// servername, since this will be ignored - the servername should be
@ -76,20 +81,27 @@ ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servern
session_created = true;
try {
return new FDBLibTLSSession(Reference<FDBLibTLSPolicy>::addRef(this), is_client, servername, send_func,
send_ctx, recv_func, recv_ctx, uid);
} catch ( ... ) {
return new FDBLibTLSSession(Reference<FDBLibTLSPolicy>::addRef(this),
is_client,
servername,
send_func,
send_ctx,
recv_func,
recv_ctx,
uid);
} catch (...) {
return nullptr;
}
}
static int password_cb(char *buf, int size, int rwflag, void *u) {
const char *password = (const char *)u;
static int password_cb(char* buf, int size, int rwflag, void* u) {
const char* password = (const char*)u;
int plen;
if (size < 0)
return 0;
if (u == nullptr) return 0;
if (u == nullptr)
return 0;
plen = strlen(password);
if (plen > size)
@ -146,7 +158,7 @@ struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, s
return certs;
err:
err:
sk_X509_pop_free(certs, X509_free);
X509_free(cert);
BIO_free(bio);
@ -167,7 +179,8 @@ bool FDBLibTLSPolicy::set_ca_data(const uint8_t* ca_data, int ca_len) {
if (ca_len < 0)
return false;
sk_X509_pop_free(roots, X509_free);
if ((roots = parse_cert_pem(ca_data, ca_len)) == nullptr) return false;
if ((roots = parse_cert_pem(ca_data, ca_len)) == nullptr)
return false;
if (tls_config_set_ca_mem(tls_cfg, ca_data, ca_len) == -1) {
TraceEvent(SevError, "FDBLibTLSCAError").detail("LibTLSErrorMessage", tls_config_error(tls_cfg));
@ -214,7 +227,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
}
if (password != nullptr) {
char *data;
char* data;
long len;
if ((bio = BIO_new_mem_buf((void*)key_data, key_len)) == nullptr) {
@ -227,7 +240,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
char errbuf[256];
if ((ERR_GET_LIB(errnum) == ERR_LIB_PEM && ERR_GET_REASON(errnum) == PEM_R_BAD_DECRYPT) ||
(ERR_GET_LIB(errnum) == ERR_LIB_EVP && ERR_GET_REASON(errnum) == EVP_R_BAD_DECRYPT)) {
(ERR_GET_LIB(errnum) == ERR_LIB_EVP && ERR_GET_REASON(errnum) == EVP_R_BAD_DECRYPT)) {
TraceEvent(SevError, "FDBLibTLSIncorrectPassword");
} else {
ERR_error_string_n(errnum, errbuf, sizeof(errbuf));
@ -248,7 +261,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
TraceEvent(SevError, "FDBLibTLSOutOfMemory");
goto err;
}
if (tls_config_set_key_mem(tls_cfg, (const uint8_t *)data, len) == -1) {
if (tls_config_set_key_mem(tls_cfg, (const uint8_t*)data, len) == -1) {
TraceEvent(SevError, "FDBLibTLSKeyError").detail("LibTLSErrorMessage", tls_config_error(tls_cfg));
goto err;
}
@ -262,7 +275,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c
key_data_set = true;
rc = true;
err:
err:
BIO_free(bio);
EVP_PKEY_free(key);
return rc;
@ -287,20 +300,20 @@ bool FDBLibTLSPolicy::set_verify_peers(int count, const uint8_t* verify_peers[],
try {
std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]);
int start = 0;
while(start < verifyString.size()) {
while (start < verifyString.size()) {
int split = verifyString.find('|', start);
if(split == std::string::npos) {
if (split == std::string::npos) {
break;
}
if(split == start || verifyString[split-1] != '\\') {
if (split == start || verifyString[split - 1] != '\\') {
auto verify = makeReference<FDBLibTLSVerify>(verifyString.substr(start, split - start));
verify_rules.push_back(verify);
start = split+1;
start = split + 1;
}
}
auto verify = makeReference<FDBLibTLSVerify>(verifyString.substr(start));
verify_rules.push_back(verify);
} catch ( const std::runtime_error& ) {
} catch (const std::runtime_error&) {
verify_rules.clear();
std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]);
TraceEvent(SevError, "FDBLibTLSVerifyPeersParseError").detail("Config", verifyString);

10
FDBLibTLS/FDBLibTLSPolicy.h
View File

@ -32,7 +32,7 @@
#include <string>
#include <vector>
struct FDBLibTLSPolicy: ITLSPolicy, ReferenceCounted<FDBLibTLSPolicy> {
struct FDBLibTLSPolicy : ITLSPolicy, ReferenceCounted<FDBLibTLSPolicy> {
FDBLibTLSPolicy(Reference<FDBLibTLSPlugin> plugin);
virtual ~FDBLibTLSPolicy();
@ -41,7 +41,13 @@ struct FDBLibTLSPolicy: ITLSPolicy, ReferenceCounted<FDBLibTLSPolicy> {
Reference<FDBLibTLSPlugin> plugin;
virtual ITLSSession* create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid);
virtual ITLSSession* create_session(bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid);
struct stack_st_X509* parse_cert_pem(const uint8_t* cert_pem, size_t cert_pem_len);
void parse_verify(std::string input);

109
FDBLibTLS/FDBLibTLSSession.cpp
View File

@ -36,11 +36,10 @@
#include <string.h>
#include <limits.h>
static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb_arg)
{
FDBLibTLSSession *session = (FDBLibTLSSession *)cb_arg;
static ssize_t tls_read_func(struct tls* ctx, void* buf, size_t buflen, void* cb_arg) {
FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg;
int rv = session->recv_func(session->recv_ctx, (uint8_t *)buf, buflen);
int rv = session->recv_func(session->recv_ctx, (uint8_t*)buf, buflen);
if (rv < 0)
return 0;
if (rv == 0)
@ -48,11 +47,10 @@ static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb
return (ssize_t)rv;
}
static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, void *cb_arg)
{
FDBLibTLSSession *session = (FDBLibTLSSession *)cb_arg;
static ssize_t tls_write_func(struct tls* ctx, const void* buf, size_t buflen, void* cb_arg) {
FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg;
int rv = session->send_func(session->send_ctx, (const uint8_t *)buf, buflen);
int rv = session->send_func(session->send_ctx, (const uint8_t*)buf, buflen);
if (rv < 0)
return 0;
if (rv == 0)
@ -60,13 +58,18 @@ static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, v
return (ssize_t)rv;
}
FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_client, const char* servername,
TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func,
void* recv_ctx, void* uidptr)
FDBLibTLSSession::FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy,
bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uidptr)
: tls_ctx(nullptr), tls_sctx(nullptr), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx),
recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) {
if (uidptr)
uid = * (UID*) uidptr;
uid = *(UID*)uidptr;
if (is_client) {
if ((tls_ctx = tls_client()) == nullptr) {
@ -116,7 +119,8 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match
unsigned char* entry_utf8 = nullptr;
int entry_utf8_len = 0;
if ((asn_criteria = ASN1_IA5STRING_new()) == nullptr) goto err;
if ((asn_criteria = ASN1_IA5STRING_new()) == nullptr)
goto err;
if (ASN1_STRING_set(asn_criteria, criteria.c_str(), criteria.size()) != 1)
goto err;
if ((criteria_utf8_len = ASN1_STRING_to_UTF8(&criteria_utf8, asn_criteria)) < 1)
@ -124,12 +128,10 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match
if ((entry_utf8_len = ASN1_STRING_to_UTF8(&entry_utf8, entry)) < 1)
goto err;
if (mt == MatchType::EXACT) {
if (criteria_utf8_len == entry_utf8_len &&
memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
if (criteria_utf8_len == entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
rc = true;
} else if (mt == MatchType::PREFIX) {
if (criteria_utf8_len <= entry_utf8_len &&
memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
if (criteria_utf8_len <= entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0)
rc = true;
} else if (mt == MatchType::SUFFIX) {
if (criteria_utf8_len <= entry_utf8_len &&
@ -137,15 +139,15 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match
rc = true;
}
err:
err:
ASN1_STRING_free(asn_criteria);
free(criteria_utf8);
free(entry_utf8);
return rc;
}
bool match_name_criteria(X509_NAME *name, NID nid, const std::string& criteria, MatchType mt) {
X509_NAME_ENTRY *name_entry;
bool match_name_criteria(X509_NAME* name, NID nid, const std::string& criteria, MatchType mt) {
X509_NAME_ENTRY* name_entry;
int idx;
// If name does not exist, or has multiple of this RDN, refuse to proceed.
@ -153,12 +155,13 @@ bool match_name_criteria(X509_NAME *name, NID nid, const std::string& criteria,
return false;
if (X509_NAME_get_index_by_NID(name, nid, idx) != -1)
return false;
if ((name_entry = X509_NAME_get_entry(name, idx)) == nullptr) return false;
if ((name_entry = X509_NAME_get_entry(name, idx)) == nullptr)
return false;
return match_criteria_entry(criteria, name_entry->value, mt);
}
bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, MatchType mt) {
bool match_extension_criteria(X509* cert, NID nid, const std::string& value, MatchType mt) {
if (nid != NID_subject_alt_name && nid != NID_issuer_alt_name) {
// I have no idea how other extensions work.
return false;
@ -168,29 +171,27 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
return false;
}
std::string value_gen = value.substr(0, pos);
std::string value_val = value.substr(pos+1, value.npos);
std::string value_val = value.substr(pos + 1, value.npos);
STACK_OF(GENERAL_NAME)* sans =
reinterpret_cast<STACK_OF(GENERAL_NAME)*>(X509_get_ext_d2i(cert, nid, nullptr, nullptr));
if (sans == nullptr) {
return false;
}
int num_sans = sk_GENERAL_NAME_num( sans );
int num_sans = sk_GENERAL_NAME_num(sans);
bool rc = false;
for( int i = 0; i < num_sans && !rc; ++i ) {
GENERAL_NAME* altname = sk_GENERAL_NAME_value( sans, i );
for (int i = 0; i < num_sans && !rc; ++i) {
GENERAL_NAME* altname = sk_GENERAL_NAME_value(sans, i);
std::string matchable;
switch (altname->type) {
case GEN_OTHERNAME:
break;
case GEN_EMAIL:
if (value_gen == "EMAIL" &&
match_criteria_entry( value_val, altname->d.rfc822Name, mt)) {
if (value_gen == "EMAIL" && match_criteria_entry(value_val, altname->d.rfc822Name, mt)) {
rc = true;
break;
}
case GEN_DNS:
if (value_gen == "DNS" &&
match_criteria_entry( value_val, altname->d.dNSName, mt )) {
if (value_gen == "DNS" && match_criteria_entry(value_val, altname->d.dNSName, mt)) {
rc = true;
break;
}
@ -199,14 +200,12 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
case GEN_EDIPARTY:
break;
case GEN_URI:
if (value_gen == "URI" &&
match_criteria_entry( value_val, altname->d.uniformResourceIdentifier, mt )) {
if (value_gen == "URI" && match_criteria_entry(value_val, altname->d.uniformResourceIdentifier, mt)) {
rc = true;
break;
}
case GEN_IPADD:
if (value_gen == "IP" &&
match_criteria_entry( value_val, altname->d.iPAddress, mt )) {
if (value_gen == "IP" && match_criteria_entry(value_val, altname->d.iPAddress, mt)) {
rc = true;
break;
}
@ -218,8 +217,13 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat
return rc;
}
bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string& criteria, MatchType mt, X509Location loc) {
switch(loc) {
bool match_criteria(X509* cert,
X509_NAME* subject,
NID nid,
const std::string& criteria,
MatchType mt,
X509Location loc) {
switch (loc) {
case X509Location::NAME: {
return match_name_criteria(subject, nid, criteria, mt);
}
@ -231,7 +235,8 @@ bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string&
return false;
}
std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs) {
std::tuple<bool, std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify,
struct stack_st_X509* certs) {
X509_STORE_CTX* store_ctx = nullptr;
X509_NAME *subject, *issuer;
bool rc = false;
@ -258,7 +263,7 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
if (!verify->verify_time)
X509_VERIFY_PARAM_set_flags(X509_STORE_CTX_get0_param(store_ctx), X509_V_FLAG_NO_CHECK_TIME);
if (X509_verify_cert(store_ctx) <= 0) {
const char *errstr = X509_verify_cert_error_string(X509_STORE_CTX_get_error(store_ctx));
const char* errstr = X509_verify_cert_error_string(X509_STORE_CTX_get_error(store_ctx));
reason = "Verify cert error: " + std::string(errstr);
goto err;
}
@ -269,8 +274,9 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
reason = "Cert subject error";
goto err;
}
for (auto &pair: verify->subject_criteria) {
if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
for (auto& pair : verify->subject_criteria) {
if (!match_criteria(
cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Cert subject match failure";
goto err;
}
@ -281,8 +287,9 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
reason = "Cert issuer error";
goto err;
}
for (auto &pair: verify->issuer_criteria) {
if (!match_criteria(cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
for (auto& pair : verify->issuer_criteria) {
if (!match_criteria(
cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Cert issuer match failure";
goto err;
}
@ -294,8 +301,9 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
reason = "Root subject error";
goto err;
}
for (auto &pair: verify->root_criteria) {
if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
for (auto& pair : verify->root_criteria) {
if (!match_criteria(
cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
reason = "Root subject match failure";
goto err;
}
@ -304,7 +312,7 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
// If we got this far, everything checked out...
rc = true;
err:
err:
X509_STORE_CTX_free(store_ctx);
return std::make_tuple(rc, reason);
@ -312,7 +320,7 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
bool FDBLibTLSSession::verify_peer() {
struct stack_st_X509* certs = nullptr;
const uint8_t *cert_pem;
const uint8_t* cert_pem;
size_t cert_pem_len;
bool rc = false;
std::set<std::string> verify_failure_reasons;
@ -328,10 +336,11 @@ bool FDBLibTLSSession::verify_peer() {
TraceEvent(SevError, "FDBLibTLSNoCertError", uid);
goto err;
}
if ((certs = policy->parse_cert_pem(cert_pem, cert_pem_len)) == nullptr) goto err;
if ((certs = policy->parse_cert_pem(cert_pem, cert_pem_len)) == nullptr)
goto err;
// Any matching rule is sufficient.
for (auto &verify_rule: policy->verify_rules) {
for (auto& verify_rule : policy->verify_rules) {
std::tie(verify_success, verify_failure_reason) = check_verify(verify_rule, certs);
if (verify_success) {
rc = true;
@ -344,7 +353,7 @@ bool FDBLibTLSSession::verify_peer() {
if (!rc) {
// log the various failure reasons
if(now() - lastVerifyFailureLogged > 1.0) {
if (now() - lastVerifyFailureLogged > 1.0) {
for (std::string reason : verify_failure_reasons) {
lastVerifyFailureLogged = now();
TraceEvent("FDBLibTLSVerifyFailure", uid).suppressFor(1.0).detail("Reason", reason);
@ -352,7 +361,7 @@ bool FDBLibTLSSession::verify_peer() {
}
}
err:
err:
sk_X509_pop_free(certs, X509_free);
return rc;

15
FDBLibTLS/FDBLibTLSSession.h
View File

@ -33,14 +33,21 @@
#include <tls.h>
struct FDBLibTLSSession : ITLSSession, ReferenceCounted<FDBLibTLSSession> {
FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy, bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid);
FDBLibTLSSession(Reference<FDBLibTLSPolicy> policy,
bool is_client,
const char* servername,
TLSSendCallbackFunc send_func,
void* send_ctx,
TLSRecvCallbackFunc recv_func,
void* recv_ctx,
void* uid);
virtual ~FDBLibTLSSession();
virtual void addref() { ReferenceCounted<FDBLibTLSSession>::addref(); }
virtual void delref() { ReferenceCounted<FDBLibTLSSession>::delref(); }
bool verify_peer();
std::tuple<bool,std::string> check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs);
std::tuple<bool, std::string> check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509* certs);
virtual int handshake();
virtual int read(uint8_t* data, int length);
@ -50,8 +57,8 @@ struct FDBLibTLSSession : ITLSSession, ReferenceCounted<FDBLibTLSSession> {
bool is_client;
struct tls *tls_ctx;
struct tls *tls_sctx;
struct tls* tls_ctx;
struct tls* tls_sctx;
TLSSendCallbackFunc send_func;
void* send_ctx;

43
FDBLibTLS/FDBLibTLSVerify.cpp
View File

@ -43,24 +43,24 @@ static int hexValue(char c) {
static std::string de4514(std::string const& input, int start, int& out_end) {
std::string output;
if(input[start] == '#' || input[start] == ' ') {
if (input[start] == '#' || input[start] == ' ') {
out_end = start;
return output;
}
int space_count = 0;
for(int p = start; p < input.size();) {
switch(input[p]) {
for (int p = start; p < input.size();) {
switch (input[p]) {
case '\\': // Handle escaped sequence
// Backslash escaping nothing!
if(p == input.size() - 1) {
if (p == input.size() - 1) {
out_end = p;
goto FIN;
}
switch(input[p+1]) {
switch (input[p + 1]) {
case ' ':
case '"':
case '#':
@ -72,24 +72,24 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
case '>':
case '|':
case '\\':
output += input[p+1];
output += input[p + 1];
p += 2;
space_count = 0;
continue;
default:
// Backslash escaping pair of hex digits requires two characters
if(p == input.size() - 2) {
if (p == input.size() - 2) {
out_end = p;
goto FIN;
}
try {
output += hexValue(input[p+1]) * 16 + hexValue(input[p+2]);
output += hexValue(input[p + 1]) * 16 + hexValue(input[p + 2]);
p += 3;
space_count = 0;
continue;
} catch( ... ) {
} catch (...) {
out_end = p;
goto FIN;
}
@ -109,7 +109,7 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
default:
// Character is what it is
output += input[p];
if(input[p] == ' ')
if (input[p] == ' ')
space_count++;
else
space_count = 0;
@ -119,7 +119,7 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
out_end = input.size();
FIN:
FIN:
out_end -= space_count;
output.resize(output.size() - space_count);
@ -128,16 +128,17 @@ static std::string de4514(std::string const& input, int start, int& out_end) {
static std::pair<std::string, std::string> splitPair(std::string const& input, char c) {
int p = input.find_first_of(c);
if(p == input.npos) {
if (p == input.npos) {
throw std::runtime_error("splitPair");
}
return std::make_pair(input.substr(0, p), input.substr(p+1, input.size()));
return std::make_pair(input.substr(0, p), input.substr(p + 1, input.size()));
}
static NID abbrevToNID(std::string const& sn) {
NID nid = NID_undef;
if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" || sn == "subjectAltName")
if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" ||
sn == "subjectAltName")
nid = OBJ_sn2nid(sn.c_str());
if (nid == NID_undef)
throw std::runtime_error("abbrevToNID");
@ -158,13 +159,11 @@ static X509Location locationForNID(NID nid) {
}
}
FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config):
verify_cert(true), verify_time(true) {
FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config) : verify_cert(true), verify_time(true) {
parse_verify(verify_config);
}
FDBLibTLSVerify::~FDBLibTLSVerify() {
}
FDBLibTLSVerify::~FDBLibTLSVerify() {}
void FDBLibTLSVerify::parse_verify(std::string input) {
int s = 0;
@ -176,8 +175,10 @@ void FDBLibTLSVerify::parse_verify(std::string input) {
throw std::runtime_error("parse_verify");
MatchType mt = MatchType::EXACT;
if (input[eq-1] == '>') mt = MatchType::PREFIX;
if (input[eq-1] == '<') mt = MatchType::SUFFIX;
if (input[eq - 1] == '>')
mt = MatchType::PREFIX;
if (input[eq - 1] == '<')
mt = MatchType::SUFFIX;
std::string term = input.substr(s, eq - s - (mt == MatchType::EXACT ? 0 : 1));
if (term.find("Check.") == 0) {
@ -206,7 +207,7 @@ void FDBLibTLSVerify::parse_verify(std::string input) {
s = eq + 3;
} else {
std::map< int, Criteria >* criteria = &subject_criteria;
std::map<int, Criteria>* criteria = &subject_criteria;
if (term.find('.') != term.npos) {
auto scoped = splitPair(term, '.');

20
FDBLibTLS/FDBLibTLSVerify.h
View File

@ -47,14 +47,10 @@ enum class X509Location {
};
struct Criteria {
Criteria( const std::string& s )
: criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {}
Criteria( const std::string& s, MatchType mt )
: criteria(s), match_type(mt), location(X509Location::NAME) {}
Criteria( const std::string& s, X509Location loc)
: criteria(s), match_type(MatchType::EXACT), location(loc) {}
Criteria( const std::string& s, MatchType mt, X509Location loc)
: criteria(s), match_type(mt), location(loc) {}
Criteria(const std::string& s) : criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {}
Criteria(const std::string& s, MatchType mt) : criteria(s), match_type(mt), location(X509Location::NAME) {}
Criteria(const std::string& s, X509Location loc) : criteria(s), match_type(MatchType::EXACT), location(loc) {}
Criteria(const std::string& s, MatchType mt, X509Location loc) : criteria(s), match_type(mt), location(loc) {}
std::string criteria;
MatchType match_type;
@ -65,7 +61,7 @@ struct Criteria {
}
};
struct FDBLibTLSVerify: ReferenceCounted<FDBLibTLSVerify> {
struct FDBLibTLSVerify : ReferenceCounted<FDBLibTLSVerify> {
FDBLibTLSVerify(std::string verify);
virtual ~FDBLibTLSVerify();
@ -77,9 +73,9 @@ struct FDBLibTLSVerify: ReferenceCounted<FDBLibTLSVerify> {
bool verify_cert;
bool verify_time;
std::map< NID, Criteria > subject_criteria;
std::map< NID, Criteria > issuer_criteria;
std::map< NID, Criteria > root_criteria;
std::map<NID, Criteria> subject_criteria;
std::map<NID, Criteria> issuer_criteria;
std::map<NID, Criteria> root_criteria;
};
#endif /* FDB_LIBTLS_VERIFY_H */

1138
FDBLibTLS/plugin-test.cpp
View File

File diff suppressed because it is too large Load Diff

140
FDBLibTLS/verify-test.cpp
View File

@ -33,11 +33,18 @@
#include "FDBLibTLS/FDBLibTLSPolicy.h"
struct FDBLibTLSVerifyTest {
FDBLibTLSVerifyTest(std::string input):
input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}), root_criteria({}) {};
FDBLibTLSVerifyTest(std::string input, bool verify_cert, bool verify_time, std::map<int, Criteria> subject, std::map<int, Criteria> issuer, std::map<int, Criteria> root):
input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject), issuer_criteria(issuer), root_criteria(root) {};
~FDBLibTLSVerifyTest() {};
FDBLibTLSVerifyTest(std::string input)
: input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}),
root_criteria({}){};
FDBLibTLSVerifyTest(std::string input,
bool verify_cert,
bool verify_time,
std::map<int, Criteria> subject,
std::map<int, Criteria> issuer,
std::map<int, Criteria> root)
: input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject),
issuer_criteria(issuer), root_criteria(root){};
~FDBLibTLSVerifyTest(){};
int run();
@ -54,20 +61,21 @@ struct FDBLibTLSVerifyTest {
static std::string criteriaToString(std::map<int, Criteria> const& criteria) {
std::string s;
for (auto &pair: criteria) {
s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " + boost::lexical_cast<std::string>((int)pair.second.match_type) + ", " + boost::lexical_cast<std::string>((int)pair.second.location) + ")}";
for (auto& pair : criteria) {
s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " +
boost::lexical_cast<std::string>((int)pair.second.match_type) + ", " +
boost::lexical_cast<std::string>((int)pair.second.location) + ")}";
}
return "{" + s + "}";
}
static void logf(const char* event, void* uid, bool is_error, ...) {
}
static void logf(const char* event, void* uid, bool is_error, ...) {}
int FDBLibTLSVerifyTest::run() {
Reference<FDBLibTLSVerify> verify;
try {
verify = makeReference<FDBLibTLSVerify>(input);
} catch ( const std::runtime_error& e ) {
} catch (const std::runtime_error& e) {
if (valid) {
std::cerr << "FAIL: Verify test failed, but should have succeeded - '" << input << "'\n";
return 1;
@ -87,15 +95,18 @@ int FDBLibTLSVerifyTest::run() {
return 1;
}
if (verify->subject_criteria != subject_criteria) {
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want " << criteriaToString(subject_criteria) << "\n";
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want "
<< criteriaToString(subject_criteria) << "\n";
return 1;
}
if (verify->issuer_criteria != issuer_criteria) {
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want " << criteriaToString(issuer_criteria) << "\n";
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want "
<< criteriaToString(issuer_criteria) << "\n";
return 1;
}
if (verify->root_criteria != root_criteria) {
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want " << criteriaToString(root_criteria) << "\n";
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want "
<< criteriaToString(root_criteria) << "\n";
return 1;
}
return 0;
@ -105,7 +116,7 @@ static int policy_verify_test() {
auto plugin = makeReference<FDBLibTLSPlugin>();
auto policy = makeReference<FDBLibTLSPolicy>(plugin, (ITLSLogFunc)logf);
const char *verify_peers[] = {
const char* verify_peers[] = {
"S.CN=abc",
"I.CN=def",
"R.CN=xyz,Check.Unexpired=0",
@ -121,7 +132,7 @@ static int policy_verify_test() {
makeReference<FDBLibTLSVerify>(std::string(verify_peers[2], verify_peers_len[2])),
};
if (!policy->set_verify_peers(3, (const uint8_t **)verify_peers, verify_peers_len)) {
if (!policy->set_verify_peers(3, (const uint8_t**)verify_peers, verify_peers_len)) {
std::cerr << "FAIL: Policy verify test failed, but should have succeeded\n";
return 1;
}
@ -131,25 +142,30 @@ static int policy_verify_test() {
}
int i = 0;
for (auto &verify_rule: policy->verify_rules) {
for (auto& verify_rule : policy->verify_rules) {
if (verify_rule->verify_cert != verify_rules[i]->verify_cert) {
std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want " << verify_rules[i]->verify_cert << "\n";
std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want "
<< verify_rules[i]->verify_cert << "\n";
return 1;
}
if (verify_rule->verify_time != verify_rules[i]->verify_time) {
std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want " << verify_rules[i]->verify_time << "\n";
std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want "
<< verify_rules[i]->verify_time << "\n";
return 1;
}
if (verify_rule->subject_criteria != verify_rules[i]->subject_criteria) {
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want " << criteriaToString(verify_rules[i]->subject_criteria) << "\n";
std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want "
<< criteriaToString(verify_rules[i]->subject_criteria) << "\n";
return 1;
}
if (verify_rule->issuer_criteria != verify_rules[i]->issuer_criteria) {
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want " << criteriaToString(verify_rules[i]->issuer_criteria) << "\n";
std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want "
<< criteriaToString(verify_rules[i]->issuer_criteria) << "\n";
return 1;
}
if (verify_rule->root_criteria != verify_rules[i]->root_criteria) {
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want " << criteriaToString(verify_rules[i]->root_criteria) << "\n";
std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want "
<< criteriaToString(verify_rules[i]->root_criteria) << "\n";
return 1;
}
i++;
@ -157,8 +173,7 @@ static int policy_verify_test() {
return 0;
}
int main(int argc, char **argv)
{
int main(int argc, char** argv) {
int failed = 0;
#define EXACT(x) Criteria(x, MatchType::EXACT, X509Location::NAME)
@ -173,29 +188,60 @@ int main(int argc, char **argv)
FDBLibTLSVerifyTest("Check.Unexpired=0", true, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Valid=1,Check.Unexpired=0", true, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,Check.Valid=0", false, false, {}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {}, {{NID_countryName, EXACT("US")}}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC", true, false,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC", true, true,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc", true, true,
{{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}},
{{NID_countryName, EXACT("US")}},
{{NID_commonName, EXACT("abc")}}),
FDBLibTLSVerifyTest("C=\\,S=abc", true, true, {{NID_countryName, EXACT(",S=abc")}}, {}, {}),
FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, {{NID_commonName, EXACT("abc")}}, {}, {}),
FDBLibTLSVerifyTest("CN=a\\62c", true, true, {{NID_commonName, EXACT("abc")}}, {}, {}),
FDBLibTLSVerifyTest("CN=a\\01c", true, true, {{NID_commonName, EXACT("a\001c")}}, {}, {}),
FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp", true, true, {{NID_subject_alt_name, {"XYZCorp", MatchType::EXACT, X509Location::EXTENSION}}}, {}, {}),
FDBLibTLSVerifyTest("S.O>=XYZ", true, true, {{NID_organizationName, PREFIX("XYZ")}}, {}, {}),
FDBLibTLSVerifyTest("S.O<=LLC", true, true, {{NID_organizationName, SUFFIX("LLC")}}, {}, {}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } },
{},
{ { NID_countryName, EXACT("US") } }),
FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC",
true,
false,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC",
true,
true,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{}),
FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc",
true,
true,
{ { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } },
{ { NID_countryName, EXACT("US") } },
{ { NID_commonName, EXACT("abc") } }),
FDBLibTLSVerifyTest("C=\\,S=abc", true, true, { { NID_countryName, EXACT(",S=abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=a\\62c", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}),
FDBLibTLSVerifyTest("CN=a\\01c", true, true, { { NID_commonName, EXACT("a\001c") } }, {}, {}),
FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp",
true,
true,
{ { NID_subject_alt_name, { "XYZCorp", MatchType::EXACT, X509Location::EXTENSION } } },
{},
{}),
FDBLibTLSVerifyTest("S.O>=XYZ", true, true, { { NID_organizationName, PREFIX("XYZ") } }, {}, {}),
FDBLibTLSVerifyTest("S.O<=LLC", true, true, { { NID_organizationName, SUFFIX("LLC") } }, {}, {}),
// Invalid cases.
FDBLibTLSVerifyTest("Check.Invalid=0"),
@ -212,7 +258,7 @@ int main(int argc, char **argv)
#undef PREFIX
#undef SUFFIX
for (auto &test: tests)
for (auto& test : tests)
failed |= test.run();
failed |= policy_verify_test();

2
README.md
View File

@ -1,5 +1,7 @@
<img alt="FoundationDB logo" src="documentation/FDB_logo.png?raw=true" width="400">
![Build Status](https://codebuild.us-west-2.amazonaws.com/badges?uuid=eyJlbmNyeXB0ZWREYXRhIjoidTh3TTlZa2FQdkdPL1drZzJUQnA1NWg0MzQ3c1VnMXlaVWQ0MVUwcUJpRlltUExBYmRCc3h2c0p1TXZLdWhDQ3BoS0Jmc2ZZdG5yVmxGUHNJM0JtV0MwPSIsIml2UGFyYW1ldGVyU3BlYyI6InBrclM3R0J2d3hmRUFDTjgiLCJtYXRlcmlhbFNldFNlcmlhbCI6MX0%3D&branch=master)
FoundationDB is a distributed database designed to handle large volumes of structured data across clusters of commodity servers. It organizes data as an ordered key-value store and employs ACID transactions for all operations. It is especially well-suited for read/write workloads but also has excellent performance for write-intensive workloads. Users interact with the database using API language binding.
To learn more about FoundationDB, visit [foundationdb.org](https://www.foundationdb.org/)

10
bindings/c/ThreadCleanup.cpp
View File

@ -25,17 +25,17 @@
#include <Windows.h>
BOOL WINAPI DllMain( HINSTANCE dll, DWORD reason, LPVOID reserved ) {
BOOL WINAPI DllMain(HINSTANCE dll, DWORD reason, LPVOID reserved) {
if (reason == DLL_THREAD_DETACH)
releaseAllThreadMagazines();
return TRUE;
}
#elif defined( __unixish__ )
#elif defined(__unixish__)
#ifdef __INTEL_COMPILER
#pragma warning ( disable:2415 )
#pragma warning(disable : 2415)
#endif
static pthread_key_t threadDestructorKey;
@ -45,13 +45,13 @@ static void threadDestructor(void*) {
}
void registerThread() {
pthread_setspecific( threadDestructorKey, (const void*)1 );
pthread_setspecific(threadDestructorKey, (const void*)1);
}
static int initThreadDestructorKey() {
if (!pthread_key_create(&threadDestructorKey, &threadDestructor)) {
registerThread();
setFastAllocatorThreadInitFunction( &registerThread );
setFastAllocatorThreadInitFunction(&registerThread);
}
return 0;

802
bindings/c/fdb_c.cpp
View File

File diff suppressed because it is too large Load Diff

3
bindings/c/foundationdb/ClientWorkload.h
View File

@ -44,7 +44,8 @@ enum class FDBSeverity { Debug, Info, Warn, WarnAlways, Error };
class FDBLogger {
public:
virtual void trace(FDBSeverity sev, const std::string& name,
virtual void trace(FDBSeverity sev,
const std::string& name,
const std::vector<std::pair<std::string, std::string>>& details) = 0;
};

464
bindings/c/foundationdb/fdb_c.h
View File

@ -22,7 +22,6 @@
#define FDB_C_H
#pragma once
#ifndef DLLEXPORT
#define DLLEXPORT
#endif
@ -52,10 +51,9 @@
* ensure a compile error in such cases, and attempt to make the compile error
* slightly informative.
*/
#define This_FoundationDB_API_function_is_removed_at_this_FDB_API_VERSION() \
[== == = ]
#define FDB_REMOVED_FUNCTION \
This_FoundationDB_API_function_is_removed_at_this_FDB_API_VERSION(0)
#define This_FoundationDB_API_function_is_removed_at_this_FDB_API_VERSION() \
{ == == = }
#define FDB_REMOVED_FUNCTION This_FoundationDB_API_function_is_removed_at_this_FDB_API_VERSION(0)
#include <stdint.h>
@ -65,333 +63,351 @@
extern "C" {
#endif
/* Pointers to these opaque types represent objects in the FDB API */
typedef struct FDB_future FDBFuture;
typedef struct FDB_database FDBDatabase;
typedef struct FDB_transaction FDBTransaction;
/* Pointers to these opaque types represent objects in the FDB API */
typedef struct FDB_future FDBFuture;
typedef struct FDB_database FDBDatabase;
typedef struct FDB_transaction FDBTransaction;
typedef int fdb_error_t;
typedef int fdb_bool_t;
typedef int fdb_error_t;
typedef int fdb_bool_t;
DLLEXPORT const char*
fdb_get_error( fdb_error_t code );
DLLEXPORT const char* fdb_get_error(fdb_error_t code);
DLLEXPORT fdb_bool_t
fdb_error_predicate( int predicate_test, fdb_error_t code );
DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_error_t code);
#define /* fdb_error_t */ fdb_select_api_version(v) fdb_select_api_version_impl(v, FDB_API_VERSION)
#define /* fdb_error_t */ fdb_select_api_version(v) fdb_select_api_version_impl(v, FDB_API_VERSION)
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_network_set_option( FDBNetworkOption option, uint8_t const* value,
int value_length );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_network_set_option(FDBNetworkOption option,
uint8_t const* value,
int value_length);
#if FDB_API_VERSION >= 14
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_setup_network();
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_setup_network();
#endif
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_run_network();
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_run_network();
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_stop_network();
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_stop_network();
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void *hook_parameter);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*),
void* hook_parameter);
#pragma pack(push, 4)
typedef struct key {
const uint8_t* key;
int key_length;
} FDBKey;
typedef struct key {
const uint8_t* key;
int key_length;
} FDBKey;
#if FDB_API_VERSION >= 700
typedef struct keyvalue {
const uint8_t* key;
int key_length;
const uint8_t* value;
int value_length;
} FDBKeyValue;
typedef struct keyvalue {
const uint8_t* key;
int key_length;
const uint8_t* value;
int value_length;
} FDBKeyValue;
#else
typedef struct keyvalue {
const void* key;
int key_length;
const void* value;
int value_length;
} FDBKeyValue;
typedef struct keyvalue {
const void* key;
int key_length;
const void* value;
int value_length;
} FDBKeyValue;
#endif
#pragma pack(pop)
DLLEXPORT void fdb_future_cancel( FDBFuture* f );
DLLEXPORT void fdb_future_cancel(FDBFuture* f);
DLLEXPORT void fdb_future_release_memory( FDBFuture* f );
DLLEXPORT void fdb_future_release_memory(FDBFuture* f);
DLLEXPORT void fdb_future_destroy( FDBFuture* f );
DLLEXPORT void fdb_future_destroy(FDBFuture* f);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_block_until_ready( FDBFuture* f );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_block_until_ready(FDBFuture* f);
DLLEXPORT fdb_bool_t fdb_future_is_ready( FDBFuture* f );
DLLEXPORT fdb_bool_t fdb_future_is_ready(FDBFuture* f);
typedef void (*FDBCallback)(FDBFuture* future, void* callback_parameter);
typedef void (*FDBCallback)(FDBFuture* future, void* callback_parameter);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_set_callback( FDBFuture* f, FDBCallback callback,
void* callback_parameter );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_set_callback(FDBFuture* f,
FDBCallback callback,
void* callback_parameter);
#if FDB_API_VERSION >= 23
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_error( FDBFuture* f );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_error(FDBFuture* f);
#endif
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_int64( FDBFuture* f, int64_t* out );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_int64(FDBFuture* f, int64_t* out);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_uint64( FDBFuture* f, uint64_t* out );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_uint64(FDBFuture* f, uint64_t* out);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_key( FDBFuture* f, uint8_t const** out_key,
int* out_key_length );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_key(FDBFuture* f, uint8_t const** out_key, int* out_key_length);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_value( FDBFuture* f, fdb_bool_t *out_present,
uint8_t const** out_value,
int* out_value_length );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_value(FDBFuture* f,
fdb_bool_t* out_present,
uint8_t const** out_value,
int* out_value_length);
#if FDB_API_VERSION >= 14
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_keyvalue_array( FDBFuture* f, FDBKeyValue const** out_kv,
int* out_count, fdb_bool_t* out_more );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_keyvalue_array(FDBFuture* f,
FDBKeyValue const** out_kv,
int* out_count,
fdb_bool_t* out_more);
#endif
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_key_array( FDBFuture* f, FDBKey const** out_key_array,
int* out_count);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_key_array(FDBFuture* f,
FDBKey const** out_key_array,
int* out_count);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_string_array(FDBFuture* f,
const char*** out_strings, int* out_count);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_string_array(FDBFuture* f,
const char*** out_strings,
int* out_count);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_create_database( const char* cluster_file_path, FDBDatabase** out_database );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_create_database(const char* cluster_file_path, FDBDatabase** out_database);
DLLEXPORT void fdb_database_destroy( FDBDatabase* d );
DLLEXPORT void fdb_database_destroy(FDBDatabase* d);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_database_set_option( FDBDatabase* d, FDBDatabaseOption option,
uint8_t const* value, int value_length );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_database_set_option(FDBDatabase* d,
FDBDatabaseOption option,
uint8_t const* value,
int value_length);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_database_create_transaction( FDBDatabase* d,
FDBTransaction** out_transaction );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_database_create_transaction(FDBDatabase* d,
FDBTransaction** out_transaction);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_database_reboot_worker( FDBDatabase* db, uint8_t const* address,
int address_length, fdb_bool_t check, int duration);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_database_force_recovery_with_data_loss( FDBDatabase* db, uint8_t const* dcid, int dcid_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_reboot_worker(FDBDatabase* db,
uint8_t const* address,
int address_length,
fdb_bool_t check,
int duration);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_database_create_snapshot(FDBDatabase *db, uint8_t const *uid,
int uid_length, uint8_t const *snap_command,
int snap_command_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_force_recovery_with_data_loss(FDBDatabase* db,
uint8_t const* dcid,
int dcid_length);
DLLEXPORT void fdb_transaction_destroy( FDBTransaction* tr);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_database_create_snapshot(FDBDatabase* db,
uint8_t const* uid,
int uid_length,
uint8_t const* snap_command,
int snap_command_length);
DLLEXPORT void fdb_transaction_cancel( FDBTransaction* tr);
DLLEXPORT WARN_UNUSED_RESULT double fdb_database_get_main_thread_busyness(FDBDatabase* db);
DLLEXPORT void fdb_transaction_destroy(FDBTransaction* tr);
DLLEXPORT void fdb_transaction_cancel(FDBTransaction* tr);
#if FDB_API_VERSION >= 14
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_transaction_set_option( FDBTransaction* tr, FDBTransactionOption option,
uint8_t const* value, int value_length );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_transaction_set_option(FDBTransaction* tr,
FDBTransactionOption option,
uint8_t const* value,
int value_length);
#endif
DLLEXPORT void
fdb_transaction_set_read_version( FDBTransaction* tr, int64_t version );
DLLEXPORT void fdb_transaction_set_read_version(FDBTransaction* tr, int64_t version);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_read_version( FDBTransaction* tr );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_read_version(FDBTransaction* tr);
#if FDB_API_VERSION >= 14
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_transaction_get( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length, fdb_bool_t snapshot );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
fdb_bool_t snapshot);
#endif
#if FDB_API_VERSION >= 14
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_transaction_get_key( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length, fdb_bool_t or_equal,
int offset, fdb_bool_t snapshot );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_key(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
fdb_bool_t or_equal,
int offset,
fdb_bool_t snapshot);
#endif
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_transaction_get_addresses_for_key(FDBTransaction* tr, uint8_t const* key_name,
int key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_addresses_for_key(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length);
#if FDB_API_VERSION >= 14
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_range(
FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, fdb_bool_t begin_or_equal, int begin_offset,
uint8_t const* end_key_name, int end_key_name_length,
fdb_bool_t end_or_equal, int end_offset, int limit, int target_bytes,
FDBStreamingMode mode, int iteration, fdb_bool_t snapshot,
fdb_bool_t reverse );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_range(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
fdb_bool_t begin_or_equal,
int begin_offset,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t end_or_equal,
int end_offset,
int limit,
int target_bytes,
FDBStreamingMode mode,
int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse);
#endif
DLLEXPORT void
fdb_transaction_set( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length, uint8_t const* value,
int value_length );
DLLEXPORT void fdb_transaction_set(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
uint8_t const* value,
int value_length);
DLLEXPORT void
fdb_transaction_atomic_op( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length, uint8_t const* param,
int param_length, FDBMutationType operation_type );
DLLEXPORT void fdb_transaction_atomic_op(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
uint8_t const* param,
int param_length,
FDBMutationType operation_type);
DLLEXPORT void
fdb_transaction_clear( FDBTransaction* tr, uint8_t const* key_name,
int key_name_length );
DLLEXPORT void fdb_transaction_clear(FDBTransaction* tr, uint8_t const* key_name, int key_name_length);
DLLEXPORT void fdb_transaction_clear_range(
FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name,
int end_key_name_length );
DLLEXPORT void fdb_transaction_clear_range(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_watch( FDBTransaction *tr,
uint8_t const* key_name,
int key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_watch(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_commit( FDBTransaction* tr );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_commit(FDBTransaction* tr);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_transaction_get_committed_version( FDBTransaction* tr,
int64_t* out_version );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_transaction_get_committed_version(FDBTransaction* tr,
int64_t* out_version);
/*
* This function intentionally returns an FDBFuture instead of an integer
* directly, so that calling this API can see the effect of previous
* mutations on the transaction. Specifically, mutations are applied
* asynchronously by the main thread. In order to see them, this call has to
* be serviced by the main thread too.
*/
DLLEXPORT WARN_UNUSED_RESULT FDBFuture *
fdb_transaction_get_approximate_size(FDBTransaction *tr);
/*
* This function intentionally returns an FDBFuture instead of an integer
* directly, so that calling this API can see the effect of previous
* mutations on the transaction. Specifically, mutations are applied
* asynchronously by the main thread. In order to see them, this call has to
* be serviced by the main thread too.
*/
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* tr);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_get_server_protocol(const char* clusterFilePath);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_get_server_protocol(const char* clusterFilePath);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_versionstamp( FDBTransaction* tr );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_versionstamp(FDBTransaction* tr);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_transaction_on_error( FDBTransaction* tr, fdb_error_t error );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_on_error(FDBTransaction* tr, fdb_error_t error);
DLLEXPORT void fdb_transaction_reset( FDBTransaction* tr );
DLLEXPORT void fdb_transaction_reset(FDBTransaction* tr);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_transaction_add_conflict_range(FDBTransaction *tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
FDBConflictRangeType type);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_transaction_add_conflict_range(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
FDBConflictRangeType type);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_estimated_range_size_bytes(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunk_size);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_range_split_points(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
int64_t chunk_size);
#define FDB_KEYSEL_LAST_LESS_THAN(k, l) k, l, 0, 0
#define FDB_KEYSEL_LAST_LESS_OR_EQUAL(k, l) k, l, 1, 0
#define FDB_KEYSEL_FIRST_GREATER_THAN(k, l) k, l, 1, 1
#define FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(k, l) k, l, 0, 1
#define FDB_KEYSEL_LAST_LESS_THAN(k, l) k, l, 0, 0
#define FDB_KEYSEL_LAST_LESS_OR_EQUAL(k, l) k, l, 1, 0
#define FDB_KEYSEL_FIRST_GREATER_THAN(k, l) k, l, 1, 1
#define FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(k, l) k, l, 0, 1
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_select_api_version_impl( int runtime_version, int header_version );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_select_api_version_impl(int runtime_version, int header_version);
DLLEXPORT int fdb_get_max_api_version();
DLLEXPORT const char* fdb_get_client_version();
DLLEXPORT int fdb_get_max_api_version();
DLLEXPORT const char* fdb_get_client_version();
/* LEGACY API VERSIONS */
/* LEGACY API VERSIONS */
#if FDB_API_VERSION < 620
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_version( FDBFuture* f, int64_t* out_version );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_version(FDBFuture* f, int64_t* out_version);
#else
#define fdb_future_get_version(f, ov) FDB_REMOVED_FUNCTION
#define fdb_future_get_version(f, ov) FDB_REMOVED_FUNCTION
#endif
#if FDB_API_VERSION < 610 || defined FDB_INCLUDE_LEGACY_TYPES
typedef struct FDB_cluster FDBCluster;
typedef struct FDB_cluster FDBCluster;
typedef enum {
/* This option is only a placeholder for C compatibility and should not be used */
FDB_CLUSTER_OPTION_DUMMY_DO_NOT_USE=-1
} FDBClusterOption;
typedef enum {
/* This option is only a placeholder for C compatibility and should not be used */
FDB_CLUSTER_OPTION_DUMMY_DO_NOT_USE = -1
} FDBClusterOption;
#endif
#if FDB_API_VERSION < 610
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_cluster( FDBFuture* f, FDBCluster** out_cluster );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_cluster(FDBFuture* f, FDBCluster** out_cluster);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_database( FDBFuture* f, FDBDatabase** out_database );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_database(FDBFuture* f, FDBDatabase** out_database);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_create_cluster( const char* cluster_file_path );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_create_cluster(const char* cluster_file_path);
DLLEXPORT void fdb_cluster_destroy( FDBCluster* c );
DLLEXPORT void fdb_cluster_destroy(FDBCluster* c);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_cluster_set_option( FDBCluster* c, FDBClusterOption option,
uint8_t const* value, int value_length );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_cluster_set_option(FDBCluster* c,
FDBClusterOption option,
uint8_t const* value,
int value_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_cluster_create_database( FDBCluster* c, uint8_t const* db_name,
int db_name_length );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_cluster_create_database(FDBCluster* c,
uint8_t const* db_name,
int db_name_length);
#else
#define fdb_future_get_cluster(f, oc) FDB_REMOVED_FUNCTION
#define fdb_future_get_database(f, od) FDB_REMOVED_FUNCTION
#define fdb_create_cluster(cfp) FDB_REMOVED_FUNCTION
#define fdb_cluster_destroy(c) FDB_REMOVED_FUNCTION
#define fdb_cluster_set_option(c, o, v, vl) FDB_REMOVED_FUNCTION
#define fdb_cluster_create_database(c, dn, dnl) FDB_REMOVED_FUNCTION
#define fdb_future_get_cluster(f, oc) FDB_REMOVED_FUNCTION
#define fdb_future_get_database(f, od) FDB_REMOVED_FUNCTION
#define fdb_create_cluster(cfp) FDB_REMOVED_FUNCTION
#define fdb_cluster_destroy(c) FDB_REMOVED_FUNCTION
#define fdb_cluster_set_option(c, o, v, vl) FDB_REMOVED_FUNCTION
#define fdb_cluster_create_database(c, dn, dnl) FDB_REMOVED_FUNCTION
#endif
#if FDB_API_VERSION < 23
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t
fdb_future_get_error( FDBFuture* f,
const char** out_description /* = NULL */ );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_error(FDBFuture* f, const char** out_description /* = NULL */);
DLLEXPORT fdb_bool_t fdb_future_is_error( FDBFuture* f );
DLLEXPORT fdb_bool_t fdb_future_is_error(FDBFuture* f);
#else
#define fdb_future_is_error(x) FDB_REMOVED_FUNCTION
#define fdb_future_is_error(x) FDB_REMOVED_FUNCTION
#endif
#if FDB_API_VERSION < 14
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_keyvalue_array(
FDBFuture* f, FDBKeyValue const** out_kv, int* out_count );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_keyvalue_array(FDBFuture* f,
FDBKeyValue const** out_kv,
int* out_count);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get(
FDBTransaction* tr, uint8_t const* key_name, int key_name_length );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_key(
FDBTransaction* tr, uint8_t const* key_name, int key_name_length,
fdb_bool_t or_equal, int offset );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_key(FDBTransaction* tr,
uint8_t const* key_name,
int key_name_length,
fdb_bool_t or_equal,
int offset);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_setup_network( const char* local_address );
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_setup_network(const char* local_address);
DLLEXPORT void fdb_transaction_set_option(
FDBTransaction* tr, FDBTransactionOption option );
DLLEXPORT void fdb_transaction_set_option(FDBTransaction* tr, FDBTransactionOption option);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_range(
FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name,
int end_key_name_length, int limit );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_range(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
uint8_t const* end_key_name,
int end_key_name_length,
int limit);
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_range_selector(
FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, fdb_bool_t begin_or_equal,
int begin_offset, uint8_t const* end_key_name,
int end_key_name_length, fdb_bool_t end_or_equal, int end_offset,
int limit );
DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_range_selector(FDBTransaction* tr,
uint8_t const* begin_key_name,
int begin_key_name_length,
fdb_bool_t begin_or_equal,
int begin_offset,
uint8_t const* end_key_name,
int end_key_name_length,
fdb_bool_t end_or_equal,
int end_offset,
int limit);
#else
#define fdb_transaction_get_range_selector(tr,bkn,bknl,boe,bo,ekn,eknl,eoe,eo,lim) FDB_REMOVED_FUNCTION
#define fdb_transaction_get_range_selector(tr, bkn, bknl, boe, bo, ekn, eknl, eoe, eo, lim) FDB_REMOVED_FUNCTION
#endif
#ifdef __cplusplus

354
bindings/c/test/mako/mako.c
View File

@ -101,8 +101,13 @@ int commit_transaction(FDBTransaction* transaction) {
return FDB_SUCCESS;
}
void update_op_lat_stats(struct timespec* start, struct timespec* end, int op, mako_stats_t* stats,
lat_block_t* block[], int* elem_size, bool* is_memory_allocated) {
void update_op_lat_stats(struct timespec* start,
struct timespec* end,
int op,
mako_stats_t* stats,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
uint64_t latencyus;
latencyus = (((uint64_t)end->tv_sec * 1000000000 + end->tv_nsec) -
@ -116,7 +121,8 @@ void update_op_lat_stats(struct timespec* start, struct timespec* end, int op, m
if (latencyus > stats->latency_us_max[op]) {
stats->latency_us_max[op] = latencyus;
}
if (!is_memory_allocated[op]) return;
if (!is_memory_allocated[op])
return;
if (elem_size[op] < stats->latency_samples[op]) {
elem_size[op] = elem_size[op] + LAT_BLOCK_SIZE;
lat_block_t* temp_block = (lat_block_t*)malloc(sizeof(lat_block_t));
@ -157,11 +163,13 @@ int cleanup(FDBTransaction* transaction, mako_args_t* args) {
endstr[4] = 0xff;
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start);
fdb_transaction_clear_range(transaction, (uint8_t*)beginstr, 5, (uint8_t*)endstr, 5);
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit;
if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
fdb_transaction_reset(transaction);
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_end);
fprintf(printme, "INFO: Clear range: %6.3f sec\n",
fprintf(printme,
"INFO: Clear range: %6.3f sec\n",
((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) /
1000000000);
return 0;
@ -172,8 +180,15 @@ failExit:
}
/* populate database */
int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int thread_id, int thread_tps,
mako_stats_t* stats, lat_block_t* block[], int* elem_size, bool* is_memory_allocated) {
int populate(FDBTransaction* transaction,
mako_args_t* args,
int worker_id,
int thread_id,
int thread_tps,
mako_stats_t* stats,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
int i;
struct timespec timer_start, timer_end;
struct timespec timer_prev, timer_now; /* for throttling */
@ -188,7 +203,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
int tracetimer = 0;
keystr = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr) return -1;
if (!keystr)
return -1;
valstr = (char*)malloc(sizeof(char) * args->value_length + 1);
if (!valstr) {
free(keystr);
@ -226,8 +242,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
fdb_error_t err;
tracetimer = 0;
fprintf(debugme, "DEBUG: txn tracing %s\n", keystr);
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
(uint8_t*)keystr, strlen(keystr));
err = fdb_transaction_set_option(
transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, (uint8_t*)keystr, strlen(keystr));
if (err) {
fprintf(
stderr,
@ -236,7 +252,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
}
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0);
if (err) {
fprintf(stderr, "ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n",
fprintf(stderr,
"ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n",
fdb_get_error(err));
}
}
@ -259,14 +276,20 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_start_commit);
}
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit;
if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
/* xact latency stats */
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size,
is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size,
update_op_lat_stats(
&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start,
&timer_per_xact_end,
OP_TRANSACTION,
stats,
block,
elem_size,
is_memory_allocated);
}
@ -283,21 +306,26 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_start_commit);
}
if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit;
if (commit_transaction(transaction) != FDB_SUCCESS)
goto failExit;
/* xact latency stats */
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size,
is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size,
is_memory_allocated);
update_op_lat_stats(
&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, is_memory_allocated);
update_op_lat_stats(
&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, is_memory_allocated);
}
clock_gettime(CLOCK_MONOTONIC, &timer_end);
stats->xacts++;
fprintf(debugme, "DEBUG: Populated %d rows (%d-%d): %6.3f sec\n", end - begin, begin, end,
fprintf(debugme,
"DEBUG: Populated %d rows (%d-%d): %6.3f sec\n",
end - begin,
begin,
end,
((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) /
1000000000);
@ -306,8 +334,10 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int
return 0;
failExit:
if (keystr) free(keystr);
if (valstr) free(valstr);
if (keystr)
free(keystr);
if (valstr)
free(valstr);
fprintf(stderr, "ERROR: FDB failure in populate()\n");
return -1;
}
@ -365,17 +395,28 @@ int run_op_get(FDBTransaction* transaction, char* keystr, char* valstr, int snap
return FDB_SUCCESS;
}
int run_op_getrange(FDBTransaction* transaction, char* keystr, char* keystr2, char* valstr, int snapshot, int reverse, FDBStreamingMode streaming_mode) {
int run_op_getrange(FDBTransaction* transaction,
char* keystr,
char* keystr2,
char* valstr,
int snapshot,
int reverse,
FDBStreamingMode streaming_mode) {
FDBFuture* f;
fdb_error_t err;
FDBKeyValue const* out_kv;
int out_count;
int out_more;
f = fdb_transaction_get_range(transaction, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)),
FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1, 0 /* limit */,
0 /* target_bytes */, streaming_mode /* FDBStreamingMode */,
0 /* iteration */, snapshot, reverse /* reverse */);
f = fdb_transaction_get_range(transaction,
FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)),
FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1,
0 /* limit */,
0 /* target_bytes */,
streaming_mode /* FDBStreamingMode */,
0 /* iteration */,
snapshot,
reverse /* reverse */);
fdb_wait_and_handle_error(fdb_transaction_get_range, f, transaction);
err = fdb_future_get_keyvalue_array(f, &out_kv, &out_count, &out_more);
@ -428,8 +469,15 @@ int run_op_clearrange(FDBTransaction* transaction, char* keystr, char* keystr2)
}
/* run one transaction */
int run_one_transaction(FDBTransaction* transaction, mako_args_t* args, mako_stats_t* stats, char* keystr,
char* keystr2, char* valstr, lat_block_t* block[], int* elem_size, bool* is_memory_allocated) {
int run_one_transaction(FDBTransaction* transaction,
mako_args_t* args,
mako_stats_t* stats,
char* keystr,
char* keystr2,
char* valstr,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
int i;
int count;
int rc;
@ -488,13 +536,25 @@ retryTxn:
rc = run_op_get(transaction, keystr, valstr, 0);
break;
case OP_GETRANGE:
rc = run_op_getrange(transaction, keystr, keystr2, valstr, 0, args->txnspec.ops[i][OP_REVERSE], args->streaming_mode);
rc = run_op_getrange(transaction,
keystr,
keystr2,
valstr,
0,
args->txnspec.ops[i][OP_REVERSE],
args->streaming_mode);
break;
case OP_SGET:
rc = run_op_get(transaction, keystr, valstr, 1);
break;
case OP_SGETRANGE:
rc = run_op_getrange(transaction, keystr, keystr2, valstr, 1, args->txnspec.ops[i][OP_REVERSE], args->streaming_mode);
rc = run_op_getrange(transaction,
keystr,
keystr2,
valstr,
1,
args->txnspec.ops[i][OP_REVERSE],
args->streaming_mode);
break;
case OP_UPDATE:
randstr(valstr, args->value_length + 1);
@ -512,10 +572,13 @@ retryTxn:
randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */
randstr(valstr, args->value_length + 1);
for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) {
sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]),
sprintf(keystr + KEYPREFIXLEN + randstrlen,
"%0.*d",
digits(args->txnspec.ops[i][OP_RANGE]),
rangei);
rc = run_op_insert(transaction, keystr, valstr);
if (rc != FDB_SUCCESS) break;
if (rc != FDB_SUCCESS)
break;
}
docommit = 1;
break;
@ -539,10 +602,20 @@ retryTxn:
stats->ops[OP_TRANSACTION]++;
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block,
elem_size, is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats,
block, elem_size, is_memory_allocated);
update_op_lat_stats(&timer_start_commit,
&timer_per_xact_end,
OP_COMMIT,
stats,
block,
elem_size,
is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start,
&timer_per_xact_end,
OP_TRANSACTION,
stats,
block,
elem_size,
is_memory_allocated);
}
} else {
/* error */
@ -572,7 +645,9 @@ retryTxn:
randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */
randstr(valstr, args->value_length + 1);
for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) {
sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]),
sprintf(keystr + KEYPREFIXLEN + randstrlen,
"%0.*d",
digits(args->txnspec.ops[i][OP_RANGE]),
rangei);
if (rangei == 0) {
strcpy(keystr2, keystr);
@ -598,10 +673,20 @@ retryTxn:
stats->ops[OP_TRANSACTION]++;
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block,
elem_size, is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats,
block, elem_size, is_memory_allocated);
update_op_lat_stats(&timer_start_commit,
&timer_per_xact_end,
OP_COMMIT,
stats,
block,
elem_size,
is_memory_allocated);
update_op_lat_stats(&timer_per_xact_start,
&timer_per_xact_end,
OP_TRANSACTION,
stats,
block,
elem_size,
is_memory_allocated);
}
} else {
/* error */
@ -666,8 +751,8 @@ retryTxn:
stats->ops[OP_COMMIT]++;
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size,
is_memory_allocated);
update_op_lat_stats(
&timer_start_commit, &timer_per_xact_end, OP_COMMIT, stats, block, elem_size, is_memory_allocated);
}
} else {
/* error */
@ -688,8 +773,8 @@ retryTxn:
stats->ops[OP_TRANSACTION]++;
if (stats->xacts % args->sampling == 0) {
clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end);
update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size,
is_memory_allocated);
update_op_lat_stats(
&timer_per_xact_start, &timer_per_xact_end, OP_TRANSACTION, stats, block, elem_size, is_memory_allocated);
}
stats->xacts++;
@ -699,9 +784,18 @@ retryTxn:
return 0;
}
int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps, volatile double* throttle_factor,
int thread_iters, volatile int* signal, mako_stats_t* stats, int dotrace, int dotagging, lat_block_t* block[],
int* elem_size, bool* is_memory_allocated) {
int run_workload(FDBTransaction* transaction,
mako_args_t* args,
int thread_tps,
volatile double* throttle_factor,
int thread_iters,
volatile int* signal,
mako_stats_t* stats,
int dotrace,
int dotagging,
lat_block_t* block[],
int* elem_size,
bool* is_memory_allocated) {
int xacts = 0;
int64_t total_xacts = 0;
int rc = 0;
@ -714,13 +808,14 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
int tracetimer = 0;
char* tagstr;
if (thread_tps < 0) return 0;
if (thread_tps < 0)
return 0;
if (dotrace) {
traceid = (char*)malloc(32);
}
if(dotagging) {
if (dotagging) {
tagstr = (char*)calloc(16, 1);
memcpy(tagstr, KEYPREFIX, KEYPREFIXLEN);
memcpy(tagstr + KEYPREFIXLEN, args->txntagging_prefix, TAGPREFIXLENGTH_MAX);
@ -729,7 +824,8 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
current_tps = (int)((double)thread_tps * *throttle_factor);
keystr = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr) return -1;
if (!keystr)
return -1;
keystr2 = (char*)malloc(sizeof(char) * args->key_length + 1);
if (!keystr2) {
free(keystr);
@ -770,11 +866,13 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
tracetimer = 0;
snprintf(traceid, 32, "makotrace%019lld", total_xacts);
fprintf(debugme, "DEBUG: txn tracing %s\n", traceid);
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
(uint8_t*)traceid, strlen(traceid));
err = fdb_transaction_set_option(transaction,
FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER,
(uint8_t*)traceid,
strlen(traceid));
if (err) {
fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n",
fdb_get_error(err));
fprintf(
stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fdb_get_error(err));
}
err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0);
if (err) {
@ -783,7 +881,6 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
}
}
} else {
if (thread_tps > 0) {
/* 1 second not passed, throttle */
@ -796,16 +893,15 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
/* enable transaction tagging */
if (dotagging > 0) {
sprintf(tagstr + KEYPREFIXLEN + TAGPREFIXLENGTH_MAX, "%03d", urand(0, args->txntagging - 1));
fdb_error_t err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_AUTO_THROTTLE_TAG,
(uint8_t*)tagstr, 16);
fdb_error_t err =
fdb_transaction_set_option(transaction, FDB_TR_OPTION_AUTO_THROTTLE_TAG, (uint8_t*)tagstr, 16);
if (err) {
fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n",
fdb_get_error(err));
fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fdb_get_error(err));
}
}
rc = run_one_transaction(transaction, args, stats, keystr, keystr2, valstr, block, elem_size,
is_memory_allocated);
rc = run_one_transaction(
transaction, args, stats, keystr, keystr2, valstr, block, elem_size, is_memory_allocated);
if (rc) {
/* FIXME: run_one_transaction should return something meaningful */
fprintf(annoyme, "ERROR: run_one_transaction failed (%d)\n", rc);
@ -829,7 +925,7 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps,
if (dotrace) {
free(traceid);
}
if(dotagging) {
if (dotagging) {
free(tagstr);
}
@ -924,8 +1020,13 @@ void* worker_thread(void* thread_args) {
stats->latency_samples[op] = 0;
}
fprintf(debugme, "DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n", worker_id, args->num_processes, thread_id,
args->num_threads, (unsigned int)pthread_self());
fprintf(debugme,
"DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n",
worker_id,
args->num_processes,
thread_id,
args->num_threads,
(unsigned int)pthread_self());
if (args->tpsmax) {
thread_tps = compute_thread_tps(args->tpsmax, worker_id, thread_id, args->num_processes, args->num_threads);
@ -965,8 +1066,18 @@ void* worker_thread(void* thread_args) {
/* run the workload */
else if (args->mode == MODE_RUN) {
rc = run_workload(transaction, args, thread_tps, throttle_factor, thread_iters,
signal, stats, dotrace, dotagging, block, elem_size, is_memory_allocated);
rc = run_workload(transaction,
args,
thread_tps,
throttle_factor,
thread_iters,
signal,
stats,
dotrace,
dotagging,
block,
elem_size,
is_memory_allocated);
if (rc < 0) {
fprintf(stderr, "ERROR: run_workload failed\n");
}
@ -991,7 +1102,8 @@ void* worker_thread(void* thread_args) {
temp_block = temp_block->next_block;
}
size = stats->latency_samples[op] % LAT_BLOCK_SIZE;
if (size != 0) fwrite(&temp_block->data, sizeof(uint64_t) * size, 1, fp);
if (size != 0)
fwrite(&temp_block->data, sizeof(uint64_t) * size, 1, fp);
} else {
while (temp_block) {
fwrite(&temp_block->data, sizeof(uint64_t) * LAT_BLOCK_SIZE, 1, fp);
@ -1047,7 +1159,6 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
return -1;
}
/* enable flatbuffers if specified */
if (args->flatbuffers) {
#ifdef FDB_NET_OPTION_USE_FLATBUFFERS
@ -1063,7 +1174,9 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
/* enable tracing if specified */
if (args->trace) {
fprintf(debugme, "DEBUG: Enable Tracing in %s (%s)\n", (args->traceformat == 0) ? "XML" : "JSON",
fprintf(debugme,
"DEBUG: Enable Tracing in %s (%s)\n",
(args->traceformat == 0) ? "XML" : "JSON",
(args->tracepath[0] == '\0') ? "current directory" : args->tracepath);
err = fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE, (uint8_t*)args->tracepath, strlen(args->tracepath));
if (err) {
@ -1095,11 +1208,10 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
fprintf(debugme, "DEBUG: fdb_setup_network\n");
err = fdb_setup_network();
if (err) {
fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err));
return -1;
fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err));
return -1;
}
/* Each worker process will have its own network thread */
fprintf(debugme, "DEBUG: creating network thread\n");
rc = pthread_create(&network_thread, NULL, fdb_network_thread, (void*)args);
@ -1181,8 +1293,10 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm, pi
}
failExit:
if (worker_threads) free(worker_threads);
if (thread_args) free(thread_args);
if (worker_threads)
free(worker_threads);
if (thread_args)
free(thread_args);
/* clean up database and cluster */
fdb_database_destroy(process.database);
@ -1208,7 +1322,8 @@ failExit:
/* initialize the parameters with default values */
int init_args(mako_args_t* args) {
int i;
if (!args) return -1;
if (!args)
return -1;
memset(args, 0, sizeof(mako_args_t)); /* zero-out everything */
args->api_version = fdb_get_max_api_version();
args->json = 0;
@ -1394,8 +1509,11 @@ void usage() {
printf("%-24s %s\n", " --tracepath=PATH", "Set trace file path");
printf("%-24s %s\n", " --trace_format <xml|json>", "Set trace format (Default: json)");
printf("%-24s %s\n", " --txntrace=sec", "Specify transaction tracing interval (Default: 0)");
printf("%-24s %s\n", " --txntagging", "Specify the number of different transaction tag (Default: 0, max = 1000)");
printf("%-24s %s\n", " --txntagging_prefix", "Specify the prefix of transaction tag - mako${txntagging_prefix} (Default: '')");
printf(
"%-24s %s\n", " --txntagging", "Specify the number of different transaction tag (Default: 0, max = 1000)");
printf("%-24s %s\n",
" --txntagging_prefix",
"Specify the prefix of transaction tag - mako${txntagging_prefix} (Default: '')");
printf("%-24s %s\n", " --knobs=KNOBS", "Set client knobs");
printf("%-24s %s\n", " --flatbuffers", "Use flatbuffers");
printf("%-24s %s\n", " --streaming", "Streaming mode: all (default), iterator, small, medium, large, serial");
@ -1430,7 +1548,7 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
{ "knobs", required_argument, NULL, ARG_KNOBS },
{ "tracepath", required_argument, NULL, ARG_TRACEPATH },
{ "trace_format", required_argument, NULL, ARG_TRACEFORMAT },
{ "streaming", required_argument, NULL, ARG_STREAMING_MODE },
{ "streaming", required_argument, NULL, ARG_STREAMING_MODE },
{ "txntrace", required_argument, NULL, ARG_TXNTRACE },
/* no args */
{ "help", no_argument, NULL, 'h' },
@ -1440,13 +1558,14 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
{ "flatbuffers", no_argument, NULL, ARG_FLATBUFFERS },
{ "trace", no_argument, NULL, ARG_TRACE },
{ "txntagging", required_argument, NULL, ARG_TXNTAGGING },
{ "txntagging_prefix", required_argument, NULL, ARG_TXNTAGGINGPREFIX},
{ "txntagging_prefix", required_argument, NULL, ARG_TXNTAGGINGPREFIX },
{ "version", no_argument, NULL, ARG_VERSION },
{ NULL, 0, NULL, 0 }
};
idx = 0;
c = getopt_long(argc, argv, short_options, long_options, &idx);
if (c < 0) break;
if (c < 0)
break;
switch (c) {
case '?':
case 'h':
@ -1475,7 +1594,8 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
break;
case 'x':
rc = parse_transaction(args, optarg);
if (rc < 0) return -1;
if (rc < 0)
return -1;
break;
case 'v':
args->verbose = atoi(optarg);
@ -1577,19 +1697,18 @@ int parse_args(int argc, char* argv[], mako_args_t* args) {
case ARG_TXNTAGGING:
args->txntagging = atoi(optarg);
if(args->txntagging > 1000) {
if (args->txntagging > 1000) {
args->txntagging = 1000;
}
break;
case ARG_TXNTAGGINGPREFIX: {
if(strlen(optarg) > TAGPREFIXLENGTH_MAX) {
if (strlen(optarg) > TAGPREFIXLENGTH_MAX) {
fprintf(stderr, "Error: the length of txntagging_prefix is larger than %d\n", TAGPREFIXLENGTH_MAX);
exit(0);
}
memcpy(args->txntagging_prefix, optarg, strlen(optarg));
break;
}
}
}
@ -1649,9 +1768,9 @@ int validate_args(mako_args_t* args) {
fprintf(stderr, "ERROR: Must specify either seconds or iteration\n");
return -1;
}
if(args->txntagging < 0) {
fprintf(stderr, "ERROR: --txntagging must be a non-negative integer\n");
return -1;
if (args->txntagging < 0) {
fprintf(stderr, "ERROR: --txntagging must be a non-negative integer\n");
return -1;
}
}
return 0;
@ -1724,7 +1843,8 @@ void print_stats_header(mako_args_t* args, bool show_commit, bool is_first_heade
/* header */
if (is_first_header_empty)
for (i = 0; i <= STATS_TITLE_WIDTH; i++) printf(" ");
for (i = 0; i <= STATS_TITLE_WIDTH; i++)
printf(" ");
for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0) {
switch (op) {
@ -1768,7 +1888,8 @@ void print_stats_header(mako_args_t* args, bool show_commit, bool is_first_heade
}
}
if (show_commit) printf("%" STR(STATS_FIELD_WIDTH) "s ", "COMMIT");
if (show_commit)
printf("%" STR(STATS_FIELD_WIDTH) "s ", "COMMIT");
if (show_op_stats) {
printf("%" STR(STATS_FIELD_WIDTH) "s\n", "TRANSACTION");
} else {
@ -1776,37 +1897,46 @@ void print_stats_header(mako_args_t* args, bool show_commit, bool is_first_heade
printf("%" STR(STATS_FIELD_WIDTH) "s\n", "Conflicts/s");
}
for (i = 0; i < STATS_TITLE_WIDTH; i++) printf("=");
for (i = 0; i < STATS_TITLE_WIDTH; i++)
printf("=");
printf(" ");
for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0) {
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" ");
}
}
/* COMMIT */
if (show_commit) {
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" ");
}
if (show_op_stats) {
/* TRANSACTION */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" ");
} else {
/* TPS */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
printf(" ");
/* Conflicts */
for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("=");
for (i = 0; i < STATS_FIELD_WIDTH; i++)
printf("=");
}
printf("\n");
}
void print_report(mako_args_t* args, mako_stats_t* stats, struct timespec* timer_now, struct timespec* timer_start,
void print_report(mako_args_t* args,
mako_stats_t* stats,
struct timespec* timer_now,
struct timespec* timer_start,
pid_t* pid_main) {
int i, j, k, op, index;
uint64_t totalxacts = 0;
@ -2069,13 +2199,18 @@ void print_report(mako_args_t* args, mako_stats_t* stats, struct timespec* timer
for (op = 0; op < MAX_OP; op++) {
if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_TRANSACTION) {
if (lat_total[op]) free(dataPoints[op]);
if (lat_total[op])
free(dataPoints[op]);
}
}
}
int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double* throttle_factor, volatile int* signal,
volatile int* stopcount, pid_t* pid_main) {
int stats_process_main(mako_args_t* args,
mako_stats_t* stats,
volatile double* throttle_factor,
volatile int* signal,
volatile int* stopcount,
pid_t* pid_main) {
struct timespec timer_start, timer_prev, timer_now;
double sin_factor;
@ -2084,7 +2219,8 @@ int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double*
usleep(10000); /* 10ms */
}
if (args->verbose >= VERBOSE_DEFAULT) print_stats_header(args, false, true, false);
if (args->verbose >= VERBOSE_DEFAULT)
print_stats_header(args, false, true, false);
clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start);
timer_prev.tv_sec = timer_start.tv_sec;
@ -2126,7 +2262,8 @@ int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double*
}
}
if (args->verbose >= VERBOSE_DEFAULT) print_stats(args, stats, &timer_now, &timer_prev);
if (args->verbose >= VERBOSE_DEFAULT)
print_stats(args, stats, &timer_now, &timer_prev);
timer_prev.tv_sec = timer_now.tv_sec;
timer_prev.tv_nsec = timer_now.tv_nsec;
}
@ -2172,7 +2309,8 @@ int main(int argc, char* argv[]) {
}
rc = validate_args(&args);
if (rc < 0) return -1;
if (rc < 0)
return -1;
if (args.mode == MODE_CLEAN) {
/* cleanup will be done from a single thread */
@ -2338,9 +2476,11 @@ int main(int argc, char* argv[]) {
failExit:
if (worker_pids) free(worker_pids);
if (worker_pids)
free(worker_pids);
if (shm != MAP_FAILED) munmap(shm, shmsize);
if (shm != MAP_FAILED)
munmap(shm, shmsize);
if (shmfd) {
close(shmfd);

336
bindings/c/test/performance_test.c
View File

@ -31,38 +31,40 @@ int numKeys = 1000000;
int keySize = 16;
uint8_t** keys = NULL;
int valueSize = 100;
uint8_t *valueStr = NULL;
uint8_t* valueStr = NULL;
fdb_error_t waitError(FDBFuture *f) {
fdb_error_t waitError(FDBFuture* f) {
fdb_error_t blockError = fdb_future_block_until_ready(f);
if(!blockError) {
if (!blockError) {
return fdb_future_get_error(f);
} else {
return blockError;
}
}
struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) {
FDBTransaction *tr = NULL;
struct RunResult run(struct ResultSet* rs,
FDBDatabase* db,
struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) {
FDBTransaction* tr = NULL;
fdb_error_t e = fdb_database_create_transaction(db, &tr);
checkError(e, "create transaction", rs);
while(1) {
while (1) {
struct RunResult r = func(rs, tr);
e = r.e;
if(!e) {
FDBFuture *f = fdb_transaction_commit(tr);
if (!e) {
FDBFuture* f = fdb_transaction_commit(tr);
e = waitError(f);
fdb_future_destroy(f);
}
if(e) {
FDBFuture *f = fdb_transaction_on_error(tr, e);
if (e) {
FDBFuture* f = fdb_transaction_on_error(tr, e);
fdb_error_t retryE = waitError(f);
fdb_future_destroy(f);
if (retryE) {
fdb_transaction_destroy(tr);
return (struct RunResult) {0, retryE};
return (struct RunResult){ 0, retryE };
}
} else {
fdb_transaction_destroy(tr);
@ -73,19 +75,22 @@ struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*f
return RES(0, 4100); // internal_error ; we should never get here
}
int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) {
int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*),
FDBDatabase* db,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25;
int *results = malloc(sizeof(int)*numRuns);
int* results = malloc(sizeof(int) * numRuns);
int i = 0;
for(; i < numRuns; ++i) {
for (; i < numRuns; ++i) {
struct RunResult res = run(rs, db, testFxn);
if(res.e) {
if (res.e) {
logError(res.e, kpiName, rs);
free(results);
return 0;
}
results[i] = res.res;
if(results[i] < 0) {
if (results[i] < 0) {
free(results);
return -1;
}
@ -99,19 +104,22 @@ int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDB
return result;
}
int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) {
int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*),
FDBDatabase* db,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25;
int *results = malloc(sizeof(int)*numRuns);
int* results = malloc(sizeof(int) * numRuns);
int i = 0;
for(; i < numRuns; ++i) {
for (; i < numRuns; ++i) {
struct RunResult res = testFxn(rs, db);
if(res.e) {
if (res.e) {
logError(res.e, kpiName, rs);
free(results);
return 0;
}
results[i] = res.res;
if(results[i] < 0) {
if (results[i] < 0) {
free(results);
return -1;
}
@ -125,139 +133,144 @@ int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBD
return result;
}
struct RunResult clearAll(struct ResultSet *rs, FDBTransaction *tr) {
struct RunResult clearAll(struct ResultSet* rs, FDBTransaction* tr) {
fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1);
return RES(0, 0);
}
uint32_t start = 0;
uint32_t stop = 0;
struct RunResult insertRange(struct ResultSet *rs, FDBTransaction *tr) {
struct RunResult insertRange(struct ResultSet* rs, FDBTransaction* tr) {
int i;
for(i = start; i < stop; i++) {
for (i = start; i < stop; i++) {
fdb_transaction_set(tr, keys[i], keySize, valueStr, valueSize);
}
return RES(0, 0);
}
void insertData(struct ResultSet *rs, FDBDatabase *db) {
void insertData(struct ResultSet* rs, FDBDatabase* db) {
checkError(run(rs, db, &clearAll).e, "clearing database", rs);
// TODO: Do this asynchronously.
start = 0;
while(start < numKeys) {
while (start < numKeys) {
stop = start + 1000;
if(stop > numKeys) stop = numKeys;
if (stop > numKeys)
stop = numKeys;
checkError(run(rs, db, &insertRange).e, "inserting data range", rs);
start = stop;
}
}
fdb_error_t setRetryLimit(struct ResultSet *rs, FDBTransaction *tr, uint64_t limit) {
fdb_error_t setRetryLimit(struct ResultSet* rs, FDBTransaction* tr, uint64_t limit) {
return fdb_transaction_set_option(tr, FDB_TR_OPTION_RETRY_LIMIT, (const uint8_t*)&limit, sizeof(uint64_t));
}
uint32_t FUTURE_LATENCY_COUNT = 100000;
const char *FUTURE_LATENCY_KPI = "C future throughput (local client)";
struct RunResult futureLatency(struct ResultSet *rs, FDBTransaction *tr) {
const char* FUTURE_LATENCY_KPI = "C future throughput (local client)";
struct RunResult futureLatency(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
FDBFuture *f = fdb_transaction_get_read_version(tr);
FDBFuture* f = fdb_transaction_get_read_version(tr);
e = waitError(f);
fdb_future_destroy(f);
maybeLogError(e, "getting initial read version", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
double start = getTime();
int i;
for(i = 0; i < FUTURE_LATENCY_COUNT; i++) {
FDBFuture *f = fdb_transaction_get_read_version(tr);
for (i = 0; i < FUTURE_LATENCY_COUNT; i++) {
FDBFuture* f = fdb_transaction_get_read_version(tr);
e = waitError(f);
fdb_future_destroy(f);
maybeLogError(e, "getting read version", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
}
double end = getTime();
return RES(FUTURE_LATENCY_COUNT/(end - start), 0);
return RES(FUTURE_LATENCY_COUNT / (end - start), 0);
}
uint32_t CLEAR_COUNT = 100000;
const char *CLEAR_KPI = "C clear throughput (local client)";
struct RunResult clear(struct ResultSet *rs, FDBTransaction *tr) {
const char* CLEAR_KPI = "C clear throughput (local client)";
struct RunResult clear(struct ResultSet* rs, FDBTransaction* tr) {
double start = getTime();
int i;
for(i = 0; i < CLEAR_COUNT; i++) {
for (i = 0; i < CLEAR_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_clear(tr, keys[k], keySize);
}
double end = getTime();
fdb_transaction_reset(tr); // Don't actually clear things.
return RES(CLEAR_COUNT/(end - start), 0);
return RES(CLEAR_COUNT / (end - start), 0);
}
uint32_t CLEAR_RANGE_COUNT = 100000;
const char *CLEAR_RANGE_KPI = "C clear range throughput (local client)";
struct RunResult clearRange(struct ResultSet *rs, FDBTransaction *tr) {
const char* CLEAR_RANGE_KPI = "C clear range throughput (local client)";
struct RunResult clearRange(struct ResultSet* rs, FDBTransaction* tr) {
double start = getTime();
int i;
for(i = 0; i < CLEAR_RANGE_COUNT; i++) {
for (i = 0; i < CLEAR_RANGE_COUNT; i++) {
int k = ((uint64_t)rand()) % (numKeys - 1);
fdb_transaction_clear_range(tr, keys[k], keySize, keys[k+1], keySize);
fdb_transaction_clear_range(tr, keys[k], keySize, keys[k + 1], keySize);
}
double end = getTime();
fdb_transaction_reset(tr); // Don't actually clear things.
return RES(CLEAR_RANGE_COUNT/(end - start), 0);
return RES(CLEAR_RANGE_COUNT / (end - start), 0);
}
uint32_t SET_COUNT = 100000;
const char *SET_KPI = "C set throughput (local client)";
struct RunResult set(struct ResultSet *rs, FDBTransaction *tr) {
const char* SET_KPI = "C set throughput (local client)";
struct RunResult set(struct ResultSet* rs, FDBTransaction* tr) {
double start = getTime();
int i;
for(i = 0; i < SET_COUNT; i++) {
for (i = 0; i < SET_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
}
double end = getTime();
fdb_transaction_reset(tr); // Don't actually set things.
return RES(SET_COUNT/(end - start), 0);
return RES(SET_COUNT / (end - start), 0);
}
uint32_t PARALLEL_GET_COUNT = 10000;
const char *PARALLEL_GET_KPI = "C parallel get throughput (local client)";
struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) {
const char* PARALLEL_GET_KPI = "C parallel get throughput (local client)";
struct RunResult parallelGet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT);
FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT);
double start = getTime();
int i;
for(i = 0; i < PARALLEL_GET_COUNT; i++) {
for (i = 0; i < PARALLEL_GET_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys;
futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0);
}
fdb_bool_t present;
uint8_t const *outValue;
uint8_t const* outValue;
int outValueLength;
for(i = 0; i < PARALLEL_GET_COUNT; i++) {
for (i = 0; i < PARALLEL_GET_COUNT; i++) {
e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs);
if(e) {
if (e) {
fdb_future_destroy(futures[i]);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if(e) {
e = maybeLogError(
fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if (e) {
fdb_future_destroy(futures[i]);
return RES(0, e);
}
@ -268,39 +281,41 @@ struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
free(futures);
return RES(PARALLEL_GET_COUNT/(end - start), 0);
return RES(PARALLEL_GET_COUNT / (end - start), 0);
}
uint32_t ALTERNATING_GET_SET_COUNT = 2000;
const char *ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)";
struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) {
const char* ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)";
struct RunResult alternatingGetSet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT);
FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT);
double start = getTime();
int i;
for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
for (i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0);
}
fdb_bool_t present;
uint8_t const *outValue;
uint8_t const* outValue;
int outValueLength;
for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
for (i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs);
if(e) {
if (e) {
fdb_future_destroy(futures[i]);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if(e) {
e = maybeLogError(
fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
if (e) {
fdb_future_destroy(futures[i]);
return RES(0, e);
}
@ -311,38 +326,39 @@ struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
free(futures);
return RES(ALTERNATING_GET_SET_COUNT/(end - start), 0);
return RES(ALTERNATING_GET_SET_COUNT / (end - start), 0);
}
uint32_t SERIAL_GET_COUNT = 2000;
const char *SERIAL_GET_KPI = "C serial get throughput (local client)";
struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
const char* SERIAL_GET_KPI = "C serial get throughput (local client)";
struct RunResult serialGet(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
int i;
uint32_t *keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT);
uint32_t* keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT);
if(SERIAL_GET_COUNT > numKeys/2) {
for(i = 0; i < SERIAL_GET_COUNT; i++) {
if (SERIAL_GET_COUNT > numKeys / 2) {
for (i = 0; i < SERIAL_GET_COUNT; i++) {
keyIndices[i] = ((uint64_t)rand()) % numKeys;
}
} else {
for(i = 0; i < SERIAL_GET_COUNT; i++) {
while(1) {
for (i = 0; i < SERIAL_GET_COUNT; i++) {
while (1) {
// Yes, this is a linear scan. This happens outside
// the part we are measuring.
uint32_t index = ((uint64_t)rand()) % numKeys;
int j;
fdb_bool_t found = 0;
for(j = 0; j < i; j++) {
if(keyIndices[j] == index) {
for (j = 0; j < i; j++) {
if (keyIndices[j] == index) {
found = 1;
break;
}
}
if(!found) {
if (!found) {
keyIndices[i] = index;
break;
}
@ -353,13 +369,13 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
double start = getTime();
fdb_bool_t present;
uint8_t const *outValue;
uint8_t const* outValue;
int outValueLength;
for(i = 0; i < SERIAL_GET_COUNT; i++) {
FDBFuture *f = fdb_transaction_get(tr, keys[keyIndices[i]], keySize, 0);
for (i = 0; i < SERIAL_GET_COUNT; i++) {
FDBFuture* f = fdb_transaction_get(tr, keys[keyIndices[i]], keySize, 0);
fdb_error_t e = maybeLogError(fdb_future_block_until_ready(f), "getting key in serial", rs);
if(e) {
if (e) {
free(keyIndices);
fdb_future_destroy(f);
return RES(0, e);
@ -367,7 +383,7 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs);
fdb_future_destroy(f);
if(e) {
if (e) {
free(keyIndices);
return RES(0, e);
}
@ -376,66 +392,87 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
free(keyIndices);
return RES(SERIAL_GET_COUNT/(end - start), 0);
return RES(SERIAL_GET_COUNT / (end - start), 0);
}
uint32_t GET_RANGE_COUNT = 100000;
const char *GET_RANGE_KPI = "C get range throughput (local client)";
struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) {
const char* GET_RANGE_KPI = "C get range throughput (local client)";
struct RunResult getRange(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
uint32_t startKey = ((uint64_t)rand()) % (numKeys - GET_RANGE_COUNT - 1);
double start = getTime();
const FDBKeyValue *outKv;
const FDBKeyValue* outKv;
int outCount;
fdb_bool_t outMore = 1;
int totalOut = 0;
int iteration = 0;
FDBFuture *f = fdb_transaction_get_range(tr,
keys[startKey], keySize, 1, 0,
keys[startKey + GET_RANGE_COUNT], keySize, 1, 0,
0, 0,
FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0);
FDBFuture* f = fdb_transaction_get_range(tr,
keys[startKey],
keySize,
1,
0,
keys[startKey + GET_RANGE_COUNT],
keySize,
1,
0,
0,
0,
FDB_STREAMING_MODE_WANT_ALL,
++iteration,
0,
0);
while(outMore) {
while (outMore) {
e = maybeLogError(fdb_future_block_until_ready(f), "getting range", rs);
if(e) {
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading range array", rs);
if(e) {
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
totalOut += outCount;
if(outMore) {
FDBFuture *f2 = fdb_transaction_get_range(tr,
outKv[outCount - 1].key, outKv[outCount - 1].key_length, 1, 1,
keys[startKey + GET_RANGE_COUNT], keySize, 1, 0,
0, 0,
FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0);
if (outMore) {
FDBFuture* f2 = fdb_transaction_get_range(tr,
outKv[outCount - 1].key,
outKv[outCount - 1].key_length,
1,
1,
keys[startKey + GET_RANGE_COUNT],
keySize,
1,
0,
0,
0,
FDB_STREAMING_MODE_WANT_ALL,
++iteration,
0,
0);
fdb_future_destroy(f);
f = f2;
}
}
if(totalOut != GET_RANGE_COUNT) {
char *msg = (char*)malloc((sizeof(char)) * 200);
if (totalOut != GET_RANGE_COUNT) {
char* msg = (char*)malloc((sizeof(char)) * 200);
sprintf(msg, "verifying out count (%d != %d)", totalOut, GET_RANGE_COUNT);
logError(4100, msg, rs);
free(msg);
fdb_future_destroy(f);
return RES(0, 4100);
}
if(outMore) {
if (outMore) {
logError(4100, "verifying no more in range", rs);
fdb_future_destroy(f);
return RES(0, 4100);
@ -444,84 +481,84 @@ struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
return RES(GET_RANGE_COUNT/(end - start), 0);
return RES(GET_RANGE_COUNT / (end - start), 0);
}
uint32_t GET_KEY_COUNT = 2000;
const char *GET_KEY_KPI = "C get key throughput (local client)";
struct RunResult getKey(struct ResultSet *rs, FDBTransaction *tr) {
const char* GET_KEY_KPI = "C get key throughput (local client)";
struct RunResult getKey(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
double start = getTime();
fdb_bool_t present;
uint8_t const *outValue;
uint8_t const* outValue;
int outValueLength;
int i;
for(i = 0; i < GET_KEY_COUNT; i++) {
for (i = 0; i < GET_KEY_COUNT; i++) {
int key = ((uint64_t)rand()) % numKeys;
int offset = (((uint64_t)rand()) % 21) - 10;
FDBFuture *f = fdb_transaction_get_key(tr, keys[key], keySize, 1, offset, 0);
FDBFuture* f = fdb_transaction_get_key(tr, keys[key], keySize, 1, offset, 0);
e = maybeLogError(fdb_future_block_until_ready(f), "waiting for get key", rs);
if(e) {
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs);
fdb_future_destroy(f);
if(e) {
if (e) {
return RES(0, e);
}
}
double end = getTime();
return RES(GET_KEY_COUNT/(end - start), 0);
return RES(GET_KEY_COUNT / (end - start), 0);
}
uint32_t GET_SINGLE_KEY_RANGE_COUNT = 2000;
const char *GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)";
struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) {
const char* GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)";
struct RunResult getSingleKeyRange(struct ResultSet* rs, FDBTransaction* tr) {
fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
if(e) return RES(0, e);
if (e)
return RES(0, e);
double start = getTime();
const FDBKeyValue *outKv;
const FDBKeyValue* outKv;
int outCount;
fdb_bool_t outMore;
int i;
for(i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) {
for (i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) {
int key = ((uint64_t)rand()) % (numKeys - 1);
FDBFuture *f = fdb_transaction_get_range(tr,
keys[key], keySize, 1, 0,
keys[key + 1], keySize, 1, 0,
2, 0,
FDB_STREAMING_MODE_EXACT, 1, 0, 0);
FDBFuture* f = fdb_transaction_get_range(
tr, keys[key], keySize, 1, 0, keys[key + 1], keySize, 1, 0, 2, 0, FDB_STREAMING_MODE_EXACT, 1, 0, 0);
e = maybeLogError(fdb_future_block_until_ready(f), "waiting for single key range", rs);
if(e) {
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs);
if(e) {
e = maybeLogError(
fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs);
if (e) {
fdb_future_destroy(f);
return RES(0, e);
}
if(outCount != 1) {
if (outCount != 1) {
logError(4100, "more than one key returned in single key range read", rs);
fdb_future_destroy(f);
return RES(0, 4100);
}
if(outMore) {
if (outMore) {
logError(4100, "more keys to read in single key range read", rs);
fdb_future_destroy(f);
return RES(0, 4100);
@ -532,33 +569,34 @@ struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) {
double end = getTime();
return RES(GET_SINGLE_KEY_RANGE_COUNT/(end - start), 0);
return RES(GET_SINGLE_KEY_RANGE_COUNT / (end - start), 0);
}
struct RunResult singleKey(struct ResultSet *rs, FDBTransaction *tr) {
struct RunResult singleKey(struct ResultSet* rs, FDBTransaction* tr) {
int k = ((uint64_t)rand()) % numKeys;
fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
return RES(0, 0);
}
uint32_t WRITE_TRANSACTION_COUNT = 1000;
const char *WRITE_TRANSACTION_KPI = "C write_transaction throughput (local client)";
struct RunResult writeTransaction(struct ResultSet *rs, FDBDatabase *db) {
const char* WRITE_TRANSACTION_KPI = "C write_transaction throughput (local client)";
struct RunResult writeTransaction(struct ResultSet* rs, FDBDatabase* db) {
double start = getTime();
int i;
for(i = 0; i < WRITE_TRANSACTION_COUNT; i++) {
for (i = 0; i < WRITE_TRANSACTION_COUNT; i++) {
struct RunResult res = run(rs, db, &singleKey);
if(res.e) return res;
if (res.e)
return res;
}
double end = getTime();
return RES(WRITE_TRANSACTION_COUNT/(end - start), 0);
return RES(WRITE_TRANSACTION_COUNT / (end - start), 0);
}
void runTests(struct ResultSet *rs) {
FDBDatabase *db = openDatabase(rs, &netThread);
void runTests(struct ResultSet* rs) {
FDBDatabase* db = openDatabase(rs, &netThread);
printf("Loading database...\n");
insertData(rs, db);
@ -600,15 +638,15 @@ void runTests(struct ResultSet *rs) {
fdb_stop_network();
}
int main(int argc, char **argv) {
int main(int argc, char** argv) {
srand(time(NULL));
struct ResultSet *rs = newResultSet();
struct ResultSet* rs = newResultSet();
checkError(fdb_select_api_version(700), "select API version", rs);
printf("Running performance test at client version: %s\n", fdb_get_client_version());
valueStr = (uint8_t*)malloc((sizeof(uint8_t))*valueSize);
valueStr = (uint8_t*)malloc((sizeof(uint8_t)) * valueSize);
int i;
for(i = 0; i < valueSize; i++) {
for (i = 0; i < valueSize; i++) {
valueStr[i] = (uint8_t)'x';
}

156
bindings/c/test/ryw_benchmark.c
View File

@ -34,23 +34,26 @@ int numKeys = 10000;
int keySize = 16;
uint8_t** keys;
void insertData(FDBTransaction *tr) {
void insertData(FDBTransaction* tr) {
fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1);
uint8_t *v = (uint8_t*)"foo";
uint8_t* v = (uint8_t*)"foo";
uint32_t i;
for(i = 0; i <= numKeys; ++i) {
for (i = 0; i <= numKeys; ++i) {
fdb_transaction_set(tr, keys[i], keySize, v, 3);
}
}
int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction *tr, struct ResultSet *rs, const char *kpiName) {
int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*),
FDBTransaction* tr,
struct ResultSet* rs,
const char* kpiName) {
int numRuns = 25;
int *results = malloc(sizeof(int)*numRuns);
int* results = malloc(sizeof(int) * numRuns);
int i = 0;
for(; i < numRuns; ++i) {
for (; i < numRuns; ++i) {
results[i] = testFxn(tr, rs);
if(results[i] < 0) {
if (results[i] < 0) {
free(results);
return -1;
}
@ -64,17 +67,19 @@ int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction *
return result;
}
int getSingle(FDBTransaction *tr, struct ResultSet *rs) {
int getSingle(FDBTransaction* tr, struct ResultSet* rs) {
int present;
uint8_t const *value;
uint8_t const* value;
int length;
int i;
double start = getTime();
for(i = 0; i < numKeys; ++i) {
FDBFuture *f = fdb_transaction_get(tr, keys[5001], keySize, 0);
if(getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs)) return -1;
if(getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs)) return -1;
for (i = 0; i < numKeys; ++i) {
FDBFuture* f = fdb_transaction_get(tr, keys[5001], keySize, 0);
if (getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs))
return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs))
return -1;
fdb_future_destroy(f);
}
double end = getTime();
@ -82,17 +87,19 @@ int getSingle(FDBTransaction *tr, struct ResultSet *rs) {
return numKeys / (end - start);
}
int getManySequential(FDBTransaction *tr, struct ResultSet *rs) {
int getManySequential(FDBTransaction* tr, struct ResultSet* rs) {
int present;
uint8_t const *value;
uint8_t const* value;
int length;
int i;
double start = getTime();
for(i = 0; i < numKeys; ++i) {
FDBFuture *f = fdb_transaction_get(tr, keys[i], keySize, 0);
if(getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs)) return -1;
if(getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs)) return -1;
for (i = 0; i < numKeys; ++i) {
FDBFuture* f = fdb_transaction_get(tr, keys[i], keySize, 0);
if (getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs))
return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs))
return -1;
fdb_future_destroy(f);
}
double end = getTime();
@ -100,20 +107,30 @@ int getManySequential(FDBTransaction *tr, struct ResultSet *rs) {
return numKeys / (end - start);
}
int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) {
int getRangeBasic(FDBTransaction* tr, struct ResultSet* rs) {
int count;
const FDBKeyValue *kvs;
const FDBKeyValue* kvs;
int more;
int i;
double start = getTime();
for(i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
for (i = 0; i < 100; ++i) {
FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs)) return -1;
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs))
return -1;
if (getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs))
return -1;
if(count != numKeys) {
if (count != numKeys) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys);
addError(rs, "GetRangeBasic bad count");
return -1;
@ -124,26 +141,37 @@ int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) {
return 100 * numKeys / (end - start);
}
int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) {
int singleClearGetRange(FDBTransaction* tr, struct ResultSet* rs) {
int count;
const FDBKeyValue *kvs;
const FDBKeyValue* kvs;
int more;
int i;
for(i = 0; i < numKeys; i+=2) {
for (i = 0; i < numKeys; i += 2) {
fdb_transaction_clear(tr, keys[i], keySize);
}
double start = getTime();
for(i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
for (i = 0; i < 100; ++i) {
FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs)) return -1;
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs))
return -1;
if (getError(
fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs))
return -1;
fdb_future_destroy(f);
if(count != numKeys/2) {
if (count != numKeys / 2) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys);
addError(rs, "SingleClearGetRange bad count");
return -1;
@ -155,27 +183,38 @@ int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) {
return 100 * numKeys / 2 / (end - start);
}
int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) {
int clearRangeGetRange(FDBTransaction* tr, struct ResultSet* rs) {
int count;
const FDBKeyValue *kvs;
const FDBKeyValue* kvs;
int more;
int i;
for(i = 0; i < numKeys; i+=4) {
fdb_transaction_clear_range(tr, keys[i], keySize, keys[i+1], keySize);
for (i = 0; i < numKeys; i += 4) {
fdb_transaction_clear_range(tr, keys[i], keySize, keys[i + 1], keySize);
}
double start = getTime();
for(i = 0; i < 100; ++i) {
FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
for (i = 0; i < 100; ++i) {
FDBFuture* f = fdb_transaction_get_range(tr,
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize),
FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize),
numKeys,
0,
0,
1,
0,
0);
if(getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs)) return -1;
if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs)) return -1;
if (getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs))
return -1;
if (getError(
fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs))
return -1;
fdb_future_destroy(f);
if(count != numKeys*3/4) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys*3/4);
if (count != numKeys * 3 / 4) {
fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys * 3 / 4);
addError(rs, "ClearRangeGetRange bad count");
return -1;
}
@ -186,13 +225,13 @@ int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) {
return 100 * numKeys * 3 / 4 / (end - start);
}
int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
int interleavedSetsGets(FDBTransaction* tr, struct ResultSet* rs) {
int present;
uint8_t const *value;
uint8_t const* value;
int length;
int i;
uint8_t *k = (uint8_t*)"foo";
uint8_t* k = (uint8_t*)"foo";
uint8_t v[10];
int num = 1;
@ -200,10 +239,12 @@ int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
sprintf((char*)v, "%d", num);
fdb_transaction_set(tr, k, 3, v, strlen((char*)v));
for(i = 0; i < 10000; ++i) {
FDBFuture *f = fdb_transaction_get(tr, k, 3, 0);
if(getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs)) return -1;
if(getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs)) return -1;
for (i = 0; i < 10000; ++i) {
FDBFuture* f = fdb_transaction_get(tr, k, 3, 0);
if (getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs))
return -1;
if (getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs))
return -1;
fdb_future_destroy(f);
sprintf((char*)v, "%d", ++num);
@ -214,13 +255,13 @@ int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
return 10000 / (end - start);
}
void runTests(struct ResultSet *rs) {
FDBDatabase *db = openDatabase(rs, &netThread);
void runTests(struct ResultSet* rs) {
FDBDatabase* db = openDatabase(rs, &netThread);
FDBTransaction *tr;
FDBTransaction* tr;
checkError(fdb_database_create_transaction(db, &tr), "create transaction", rs);
FDBFuture *f = fdb_transaction_get_read_version(tr);
FDBFuture* f = fdb_transaction_get_read_version(tr);
checkError(fdb_future_block_until_ready(f), "block for read version", rs);
int64_t version;
@ -241,9 +282,9 @@ void runTests(struct ResultSet *rs) {
fdb_stop_network();
}
int main(int argc, char **argv) {
int main(int argc, char** argv) {
srand(time(NULL));
struct ResultSet *rs = newResultSet();
struct ResultSet* rs = newResultSet();
checkError(fdb_select_api_version(700), "select API version", rs);
printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version());
@ -255,4 +296,3 @@ int main(int argc, char **argv) {
return 0;
}

109
bindings/c/test/test.h
View File

@ -38,27 +38,27 @@
double getTime() {
static struct timeval tv;
gettimeofday(&tv, NULL);
return tv.tv_usec/1000000.0 + tv.tv_sec;
return tv.tv_usec / 1000000.0 + tv.tv_sec;
}
void writeKey(uint8_t **dest, int key, int keySize) {
*dest = (uint8_t*)malloc((sizeof(uint8_t))*keySize);
void writeKey(uint8_t** dest, int key, int keySize) {
*dest = (uint8_t*)malloc((sizeof(uint8_t)) * keySize);
sprintf((char*)*dest, "%0*d", keySize, key);
}
uint8_t **generateKeys(int numKeys, int keySize) {
uint8_t **keys = (uint8_t**)malloc(sizeof(uint8_t*)*(numKeys+1));
uint8_t** generateKeys(int numKeys, int keySize) {
uint8_t** keys = (uint8_t**)malloc(sizeof(uint8_t*) * (numKeys + 1));
uint32_t i;
for(i = 0; i <= numKeys; ++i) {
for (i = 0; i <= numKeys; ++i) {
writeKey(keys + i, i, keySize);
}
return keys;
}
void freeKeys(uint8_t **keys, int numKeys) {
void freeKeys(uint8_t** keys, int numKeys) {
uint32_t i;
for(i = 0; i < numKeys; i++) {
for (i = 0; i < numKeys; i++) {
free(keys[i]);
}
free(keys);
@ -68,38 +68,39 @@ int cmpfunc(const void* a, const void* b) {
return (*(int*)a - *(int*)b);
}
int median(int *values, int length) {
int median(int* values, int length) {
qsort(values, length, sizeof(int), cmpfunc);
return values[length/2];
return values[length / 2];
}
struct RunResult {
int res;
fdb_error_t e;
};
#define RES(x, y) (struct RunResult) { x, y }
#define RES(x, y) \
(struct RunResult) { x, y }
struct Kpi {
const char *name;
const char* name;
int value;
const char *units;
const char* units;
struct Kpi *next;
struct Kpi* next;
};
struct Error {
char *message;
char* message;
struct Error *next;
struct Error* next;
};
struct ResultSet {
struct Kpi *kpis;
struct Error *errors;
struct Kpi* kpis;
struct Error* errors;
};
struct ResultSet* newResultSet() {
struct ResultSet *rs = malloc(sizeof(struct ResultSet));
struct ResultSet* rs = malloc(sizeof(struct ResultSet));
rs->kpis = NULL;
rs->errors = NULL;
@ -107,8 +108,8 @@ struct ResultSet* newResultSet() {
return rs;
}
void addKpi(struct ResultSet *rs, const char *name, int value, const char *units) {
struct Kpi *k = malloc(sizeof(struct Kpi));
void addKpi(struct ResultSet* rs, const char* name, int value, const char* units) {
struct Kpi* k = malloc(sizeof(struct Kpi));
k->name = name;
k->value = value;
k->units = units;
@ -116,20 +117,20 @@ void addKpi(struct ResultSet *rs, const char *name, int value, const char *units
rs->kpis = k;
}
void addError(struct ResultSet *rs, const char *message) {
struct Error *e = malloc(sizeof(struct Error));
e->message = (char*)malloc(strlen(message)+1);
void addError(struct ResultSet* rs, const char* message) {
struct Error* e = malloc(sizeof(struct Error));
e->message = (char*)malloc(strlen(message) + 1);
strcpy(e->message, message);
e->next = rs->errors;
rs->errors = e;
}
void writeResultSet(struct ResultSet *rs) {
void writeResultSet(struct ResultSet* rs) {
uint64_t id = ((uint64_t)rand() << 32) + rand();
char name[100];
sprintf(name, "fdb-c_result-%" SCNu64 ".json", id);
FILE *fp = fopen(name, "w");
if(!fp) {
FILE* fp = fopen(name, "w");
if (!fp) {
fprintf(stderr, "Could not open results file %s\n", name);
exit(1);
}
@ -137,10 +138,10 @@ void writeResultSet(struct ResultSet *rs) {
fprintf(fp, "{\n");
fprintf(fp, "\t\"kpis\": {\n");
struct Kpi *k = rs->kpis;
while(k != NULL) {
struct Kpi* k = rs->kpis;
while (k != NULL) {
fprintf(fp, "\t\t\"%s\": { \"units\": \"%s\", \"value\": %d }", k->name, k->units, k->value);
if(k->next != NULL) {
if (k->next != NULL) {
fprintf(fp, ",");
}
fprintf(fp, "\n");
@ -150,10 +151,10 @@ void writeResultSet(struct ResultSet *rs) {
fprintf(fp, "\t},\n");
fprintf(fp, "\t\"errors\": [\n");
struct Error *e = rs->errors;
while(e != NULL) {
struct Error* e = rs->errors;
while (e != NULL) {
fprintf(fp, "\t\t\"%s\"", e->message);
if(e->next != NULL) {
if (e->next != NULL) {
fprintf(fp, ",");
}
fprintf(fp, "\n");
@ -166,17 +167,17 @@ void writeResultSet(struct ResultSet *rs) {
fclose(fp);
}
void freeResultSet(struct ResultSet *rs) {
struct Kpi *k = rs->kpis;
while(k != NULL) {
struct Kpi *next = k->next;
void freeResultSet(struct ResultSet* rs) {
struct Kpi* k = rs->kpis;
while (k != NULL) {
struct Kpi* next = k->next;
free(k);
k = next;
}
struct Error *e = rs->errors;
while(e != NULL) {
struct Error *next = e->next;
struct Error* e = rs->errors;
while (e != NULL) {
struct Error* next = e->next;
free(e->message);
free(e);
e = next;
@ -185,12 +186,12 @@ void freeResultSet(struct ResultSet *rs) {
free(rs);
}
fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet *rs) {
if(err) {
char *msg = (char*)malloc(strlen(context) + 100);
fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet* rs) {
if (err) {
char* msg = (char*)malloc(strlen(context) + 100);
sprintf(msg, "Error in %s: %s", context, fdb_get_error(err));
fprintf(stderr, "%s\n", msg);
if(rs != NULL) {
if (rs != NULL) {
addError(rs, msg);
}
@ -200,9 +201,9 @@ fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet *rs)
return err;
}
void checkError(fdb_error_t err, const char* context, struct ResultSet *rs) {
if(getError(err, context, rs)) {
if(rs != NULL) {
void checkError(fdb_error_t err, const char* context, struct ResultSet* rs) {
if (getError(err, context, rs)) {
if (rs != NULL) {
writeResultSet(rs);
freeResultSet(rs);
}
@ -210,11 +211,11 @@ void checkError(fdb_error_t err, const char* context, struct ResultSet *rs) {
}
}
fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet *rs) {
char *msg = (char*)malloc(strlen(context) + 100);
fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet* rs) {
char* msg = (char*)malloc(strlen(context) + 100);
sprintf(msg, "Error in %s: %s", context, fdb_get_error(err));
fprintf(stderr, "%s\n", msg);
if(rs != NULL) {
if (rs != NULL) {
addError(rs, msg);
}
@ -222,8 +223,8 @@ fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet *rs)
return err;
}
fdb_error_t maybeLogError(fdb_error_t err, const char* context, struct ResultSet *rs) {
if(err && !fdb_error_predicate( FDB_ERROR_PREDICATE_RETRYABLE, err ) ) {
fdb_error_t maybeLogError(fdb_error_t err, const char* context, struct ResultSet* rs) {
if (err && !fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, err)) {
return logError(err, context, rs);
}
return err;
@ -234,11 +235,11 @@ void* runNetwork() {
return NULL;
}
FDBDatabase* openDatabase(struct ResultSet *rs, pthread_t *netThread) {
FDBDatabase* openDatabase(struct ResultSet* rs, pthread_t* netThread) {
checkError(fdb_setup_network(), "setup network", rs);
pthread_create(netThread, NULL, (void*)(&runNetwork), NULL);
FDBDatabase *db;
FDBDatabase* db;
checkError(fdb_create_database(NULL, &db), "create database", rs);
return db;

16
bindings/c/test/txn_size_test.c
View File

@ -31,14 +31,14 @@ pthread_t netThread;
const int numKeys = 100;
uint8_t** keys = NULL;
#define KEY_SIZE 16
#define KEY_SIZE 16
#define VALUE_SIZE 100
uint8_t valueStr[VALUE_SIZE];
fdb_error_t getSize(struct ResultSet* rs, FDBTransaction* tr, int64_t* out_size) {
fdb_error_t e;
FDBFuture* future = fdb_transaction_get_approximate_size(tr);
e = maybeLogError(fdb_future_block_until_ready(future), "waiting for get future", rs);
if (e) {
fdb_future_destroy(future);
@ -55,11 +55,11 @@ fdb_error_t getSize(struct ResultSet* rs, FDBTransaction* tr, int64_t* out_size)
return 0;
}
void runTests(struct ResultSet *rs) {
void runTests(struct ResultSet* rs) {
int64_t sizes[numKeys];
int i = 0, j = 0;
FDBDatabase *db = openDatabase(rs, &netThread);
FDBTransaction *tr = NULL;
FDBDatabase* db = openDatabase(rs, &netThread);
FDBTransaction* tr = NULL;
fdb_error_t e = fdb_database_create_transaction(db, &tr);
checkError(e, "create transaction", rs);
memset(sizes, 0, numKeys * sizeof(uint32_t));
@ -82,7 +82,7 @@ void runTests(struct ResultSet *rs) {
printf("size %d: %u\n", i, sizes[i]);
i++;
fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i+1], KEY_SIZE);
fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i + 1], KEY_SIZE);
e = getSize(rs, tr, sizes + i);
checkError(e, "transaction get size", rs);
printf("size %d: %u\n", i, sizes[i]);
@ -94,9 +94,9 @@ void runTests(struct ResultSet *rs) {
printf("Test passed!\n");
}
int main(int argc, char **argv) {
int main(int argc, char** argv) {
srand(time(NULL));
struct ResultSet *rs = newResultSet();
struct ResultSet* rs = newResultSet();
checkError(fdb_select_api_version(700), "select API version", rs);
printf("Running performance test at client version: %s\n", fdb_get_client_version());

173
bindings/c/test/unit/fdb_api.cpp
View File

@ -27,82 +27,80 @@ namespace fdb {
// Future
Future::~Future() {
fdb_future_destroy(future_);
fdb_future_destroy(future_);
}
bool Future::is_ready() {
return fdb_future_is_ready(future_);
return fdb_future_is_ready(future_);
}
[[nodiscard]] fdb_error_t Future::block_until_ready() {
return fdb_future_block_until_ready(future_);
return fdb_future_block_until_ready(future_);
}
[[nodiscard]] fdb_error_t Future::set_callback(FDBCallback callback,
void* callback_parameter) {
return fdb_future_set_callback(future_, callback, callback_parameter);
[[nodiscard]] fdb_error_t Future::set_callback(FDBCallback callback, void* callback_parameter) {
return fdb_future_set_callback(future_, callback, callback_parameter);
}
[[nodiscard]] fdb_error_t Future::get_error() {
return fdb_future_get_error(future_);
return fdb_future_get_error(future_);
}
void Future::release_memory() {
fdb_future_release_memory(future_);
fdb_future_release_memory(future_);
}
void Future::cancel() {
fdb_future_cancel(future_);
fdb_future_cancel(future_);
}
// Int64Future
[[nodiscard]] fdb_error_t Int64Future::get(int64_t* out) {
return fdb_future_get_int64(future_, out);
return fdb_future_get_int64(future_, out);
}
// ValueFuture
[[nodiscard]] fdb_error_t ValueFuture::get(fdb_bool_t* out_present,
const uint8_t** out_value,
int* out_value_length) {
return fdb_future_get_value(future_, out_present, out_value,
out_value_length);
[[nodiscard]] fdb_error_t ValueFuture::get(fdb_bool_t* out_present, const uint8_t** out_value, int* out_value_length) {
return fdb_future_get_value(future_, out_present, out_value, out_value_length);
}
// KeyFuture
[[nodiscard]] fdb_error_t KeyFuture::get(const uint8_t** out_key,
int* out_key_length) {
return fdb_future_get_key(future_, out_key, out_key_length);
[[nodiscard]] fdb_error_t KeyFuture::get(const uint8_t** out_key, int* out_key_length) {
return fdb_future_get_key(future_, out_key, out_key_length);
}
// StringArrayFuture
[[nodiscard]] fdb_error_t StringArrayFuture::get(const char*** out_strings,
int* out_count) {
return fdb_future_get_string_array(future_, out_strings, out_count);
[[nodiscard]] fdb_error_t StringArrayFuture::get(const char*** out_strings, int* out_count) {
return fdb_future_get_string_array(future_, out_strings, out_count);
}
// KeyValueArrayFuture
[[nodiscard]] fdb_error_t KeyValueArrayFuture::get(const FDBKeyValue** out_kv,
int* out_count,
fdb_bool_t* out_more) {
return fdb_future_get_keyvalue_array(future_, out_kv, out_count, out_more);
[[nodiscard]] fdb_error_t KeyValueArrayFuture::get(const FDBKeyValue** out_kv, int* out_count, fdb_bool_t* out_more) {
return fdb_future_get_keyvalue_array(future_, out_kv, out_count, out_more);
}
// Database
Int64Future Database::reboot_worker(FDBDatabase* db, const uint8_t* address, int address_length, fdb_bool_t check,
int duration) {
Int64Future Database::reboot_worker(FDBDatabase* db,
const uint8_t* address,
int address_length,
fdb_bool_t check,
int duration) {
return Int64Future(fdb_database_reboot_worker(db, address, address_length, check, duration));
}
EmptyFuture Database::force_recovery_with_data_loss(FDBDatabase *db, const uint8_t *dcid, int dcid_length) {
return EmptyFuture(fdb_database_force_recovery_with_data_loss(db, dcid, dcid_length));
EmptyFuture Database::force_recovery_with_data_loss(FDBDatabase* db, const uint8_t* dcid, int dcid_length) {
return EmptyFuture(fdb_database_force_recovery_with_data_loss(db, dcid, dcid_length));
}
EmptyFuture Database::create_snapshot(FDBDatabase* db, const uint8_t* uid, int uid_length, const uint8_t* snap_command,
EmptyFuture Database::create_snapshot(FDBDatabase* db,
const uint8_t* uid,
int uid_length,
const uint8_t* snap_command,
int snap_command_length) {
return EmptyFuture(fdb_database_create_snapshot(db, uid, uid_length, snap_command, snap_command_length));
}
@ -110,61 +108,58 @@ EmptyFuture Database::create_snapshot(FDBDatabase* db, const uint8_t* uid, int u
// Transaction
Transaction::Transaction(FDBDatabase* db) {
if (fdb_error_t err = fdb_database_create_transaction(db, &tr_)) {
std::cerr << fdb_get_error(err) << std::endl;
std::abort();
}
if (fdb_error_t err = fdb_database_create_transaction(db, &tr_)) {
std::cerr << fdb_get_error(err) << std::endl;
std::abort();
}
}
Transaction::~Transaction() {
fdb_transaction_destroy(tr_);
fdb_transaction_destroy(tr_);
}
void Transaction::reset() {
fdb_transaction_reset(tr_);
fdb_transaction_reset(tr_);
}
void Transaction::cancel() {
fdb_transaction_cancel(tr_);
fdb_transaction_cancel(tr_);
}
[[nodiscard]] fdb_error_t Transaction::set_option(FDBTransactionOption option,
const uint8_t* value,
int value_length) {
return fdb_transaction_set_option(tr_, option, value, value_length);
[[nodiscard]] fdb_error_t Transaction::set_option(FDBTransactionOption option, const uint8_t* value, int value_length) {
return fdb_transaction_set_option(tr_, option, value, value_length);
}
void Transaction::set_read_version(int64_t version) {
fdb_transaction_set_read_version(tr_, version);
fdb_transaction_set_read_version(tr_, version);
}
Int64Future Transaction::get_read_version() {
return Int64Future(fdb_transaction_get_read_version(tr_));
return Int64Future(fdb_transaction_get_read_version(tr_));
}
Int64Future Transaction::get_approximate_size() {
return Int64Future(fdb_transaction_get_approximate_size(tr_));
return Int64Future(fdb_transaction_get_approximate_size(tr_));
}
KeyFuture Transaction::get_versionstamp() {
return KeyFuture(fdb_transaction_get_versionstamp(tr_));
return KeyFuture(fdb_transaction_get_versionstamp(tr_));
}
ValueFuture Transaction::get(std::string_view key, fdb_bool_t snapshot) {
return ValueFuture(fdb_transaction_get(tr_, (const uint8_t*)key.data(),
key.size(), snapshot));
return ValueFuture(fdb_transaction_get(tr_, (const uint8_t*)key.data(), key.size(), snapshot));
}
KeyFuture Transaction::get_key(const uint8_t* key_name, int key_name_length,
fdb_bool_t or_equal, int offset,
KeyFuture Transaction::get_key(const uint8_t* key_name,
int key_name_length,
fdb_bool_t or_equal,
int offset,
fdb_bool_t snapshot) {
return KeyFuture(fdb_transaction_get_key(tr_, key_name, key_name_length,
or_equal, offset, snapshot));
return KeyFuture(fdb_transaction_get_key(tr_, key_name, key_name_length, or_equal, offset, snapshot));
}
StringArrayFuture Transaction::get_addresses_for_key(std::string_view key) {
return StringArrayFuture(fdb_transaction_get_addresses_for_key(tr_,
(const uint8_t*)key.data(), key.size()));
return StringArrayFuture(fdb_transaction_get_addresses_for_key(tr_, (const uint8_t*)key.data(), key.size()));
}
KeyValueArrayFuture Transaction::get_range(const uint8_t* begin_key_name,
@ -174,71 +169,71 @@ KeyValueArrayFuture Transaction::get_range(const uint8_t* begin_key_name,
const uint8_t* end_key_name,
int end_key_name_length,
fdb_bool_t end_or_equal,
int end_offset, int limit,
int end_offset,
int limit,
int target_bytes,
FDBStreamingMode mode,
int iteration, fdb_bool_t snapshot,
int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse) {
return KeyValueArrayFuture(fdb_transaction_get_range(tr_, begin_key_name,
begin_key_name_length,
begin_or_equal,
begin_offset,
end_key_name,
end_key_name_length,
end_or_equal,
end_offset,
limit, target_bytes,
mode, iteration,
snapshot, reverse));
return KeyValueArrayFuture(fdb_transaction_get_range(tr_,
begin_key_name,
begin_key_name_length,
begin_or_equal,
begin_offset,
end_key_name,
end_key_name_length,
end_or_equal,
end_offset,
limit,
target_bytes,
mode,
iteration,
snapshot,
reverse));
}
EmptyFuture Transaction::watch(std::string_view key) {
return EmptyFuture(fdb_transaction_watch(tr_, (const uint8_t*)key.data(), key.size()));
return EmptyFuture(fdb_transaction_watch(tr_, (const uint8_t*)key.data(), key.size()));
}
EmptyFuture Transaction::commit() {
return EmptyFuture(fdb_transaction_commit(tr_));
return EmptyFuture(fdb_transaction_commit(tr_));
}
EmptyFuture Transaction::on_error(fdb_error_t err) {
return EmptyFuture(fdb_transaction_on_error(tr_, err));
return EmptyFuture(fdb_transaction_on_error(tr_, err));
}
void Transaction::clear(std::string_view key) {
return fdb_transaction_clear(tr_, (const uint8_t*)key.data(), key.size());
return fdb_transaction_clear(tr_, (const uint8_t*)key.data(), key.size());
}
void Transaction::clear_range(std::string_view begin_key,
std::string_view end_key) {
fdb_transaction_clear_range(tr_, (const uint8_t*)begin_key.data(),
begin_key.size(), (const uint8_t*)end_key.data(),
end_key.size());
void Transaction::clear_range(std::string_view begin_key, std::string_view end_key) {
fdb_transaction_clear_range(
tr_, (const uint8_t*)begin_key.data(), begin_key.size(), (const uint8_t*)end_key.data(), end_key.size());
}
void Transaction::set(std::string_view key, std::string_view value) {
fdb_transaction_set(tr_, (const uint8_t*)key.data(), key.size(),
(const uint8_t*)value.data(), value.size());
fdb_transaction_set(tr_, (const uint8_t*)key.data(), key.size(), (const uint8_t*)value.data(), value.size());
}
void Transaction::atomic_op(std::string_view key, const uint8_t* param,
int param_length, FDBMutationType operationType) {
return fdb_transaction_atomic_op(tr_, (const uint8_t*)key.data(), key.size(),
param, param_length, operationType);
void Transaction::atomic_op(std::string_view key,
const uint8_t* param,
int param_length,
FDBMutationType operationType) {
return fdb_transaction_atomic_op(tr_, (const uint8_t*)key.data(), key.size(), param, param_length, operationType);
}
[[nodiscard]] fdb_error_t Transaction::get_committed_version(int64_t* out_version) {
return fdb_transaction_get_committed_version(tr_, out_version);
return fdb_transaction_get_committed_version(tr_, out_version);
}
fdb_error_t Transaction::add_conflict_range(std::string_view begin_key,
std::string_view end_key,
FDBConflictRangeType type) {
return fdb_transaction_add_conflict_range(tr_,
(const uint8_t*)begin_key.data(),
begin_key.size(),
(const uint8_t*)end_key.data(),
end_key.size(),
type);
return fdb_transaction_add_conflict_range(
tr_, (const uint8_t*)begin_key.data(), begin_key.size(), (const uint8_t*)end_key.data(), end_key.size(), type);
}
} // namespace fdb
} // namespace fdb

286
bindings/c/test/unit/fdb_api.hpp
View File

@ -50,203 +50,207 @@ namespace fdb {
// Wrapper parent class to manage memory of an FDBFuture pointer. Cleans up
// FDBFuture when this instance goes out of scope.
class Future {
public:
virtual ~Future() = 0;
public:
virtual ~Future() = 0;
// Wrapper around fdb_future_is_ready.
bool is_ready();
// Wrapper around fdb_future_block_until_ready.
fdb_error_t block_until_ready();
// Wrapper around fdb_future_set_callback.
fdb_error_t set_callback(FDBCallback callback, void* callback_parameter);
// Wrapper around fdb_future_get_error.
fdb_error_t get_error();
// Wrapper around fdb_future_release_memory.
void release_memory();
// Wrapper around fdb_future_cancel.
void cancel();
// Wrapper around fdb_future_is_ready.
bool is_ready();
// Wrapper around fdb_future_block_until_ready.
fdb_error_t block_until_ready();
// Wrapper around fdb_future_set_callback.
fdb_error_t set_callback(FDBCallback callback, void* callback_parameter);
// Wrapper around fdb_future_get_error.
fdb_error_t get_error();
// Wrapper around fdb_future_release_memory.
void release_memory();
// Wrapper around fdb_future_cancel.
void cancel();
// Conversion operator to allow Future instances to work interchangeably as
// an FDBFuture object.
// operator FDBFuture* () const {
// return future_;
// }
// Conversion operator to allow Future instances to work interchangeably as
// an FDBFuture object.
// operator FDBFuture* () const {
// return future_;
// }
protected:
Future(FDBFuture *f) : future_(f) {}
FDBFuture* future_;
protected:
Future(FDBFuture* f) : future_(f) {}
FDBFuture* future_;
};
class Int64Future : public Future {
public:
// Call this function instead of fdb_future_get_int64 when using the
// Int64Future type. It's behavior is identical to fdb_future_get_int64.
fdb_error_t get(int64_t* out);
private:
friend class Transaction;
friend class Database;
Int64Future(FDBFuture* f) : Future(f) {}
public:
// Call this function instead of fdb_future_get_int64 when using the
// Int64Future type. It's behavior is identical to fdb_future_get_int64.
fdb_error_t get(int64_t* out);
private:
friend class Transaction;
friend class Database;
Int64Future(FDBFuture* f) : Future(f) {}
};
class KeyFuture : public Future {
public:
// Call this function instead of fdb_future_get_key when using the KeyFuture
// type. It's behavior is identical to fdb_future_get_key.
fdb_error_t get(const uint8_t** out_key, int* out_key_length);
public:
// Call this function instead of fdb_future_get_key when using the KeyFuture
// type. It's behavior is identical to fdb_future_get_key.
fdb_error_t get(const uint8_t** out_key, int* out_key_length);
private:
friend class Transaction;
KeyFuture(FDBFuture* f) : Future(f) {}
private:
friend class Transaction;
KeyFuture(FDBFuture* f) : Future(f) {}
};
class ValueFuture : public Future {
public:
// Call this function instead of fdb_future_get_value when using the
// ValueFuture type. It's behavior is identical to fdb_future_get_value.
fdb_error_t get(fdb_bool_t* out_present, const uint8_t** out_value,
int* out_value_length);
public:
// Call this function instead of fdb_future_get_value when using the
// ValueFuture type. It's behavior is identical to fdb_future_get_value.
fdb_error_t get(fdb_bool_t* out_present, const uint8_t** out_value, int* out_value_length);
private:
friend class Transaction;
ValueFuture(FDBFuture* f) : Future(f) {}
private:
friend class Transaction;
ValueFuture(FDBFuture* f) : Future(f) {}
};
class StringArrayFuture : public Future {
public:
// Call this function instead of fdb_future_get_string_array when using the
// StringArrayFuture type. It's behavior is identical to
// fdb_future_get_string_array.
fdb_error_t get(const char*** out_strings, int* out_count);
public:
// Call this function instead of fdb_future_get_string_array when using the
// StringArrayFuture type. It's behavior is identical to
// fdb_future_get_string_array.
fdb_error_t get(const char*** out_strings, int* out_count);
private:
friend class Transaction;
StringArrayFuture(FDBFuture* f) : Future(f) {}
private:
friend class Transaction;
StringArrayFuture(FDBFuture* f) : Future(f) {}
};
class KeyValueArrayFuture : public Future {
public:
// Call this function instead of fdb_future_get_keyvalue_array when using
// the KeyValueArrayFuture type. It's behavior is identical to
// fdb_future_get_keyvalue_array.
fdb_error_t get(const FDBKeyValue** out_kv, int* out_count,
fdb_bool_t* out_more);
public:
// Call this function instead of fdb_future_get_keyvalue_array when using
// the KeyValueArrayFuture type. It's behavior is identical to
// fdb_future_get_keyvalue_array.
fdb_error_t get(const FDBKeyValue** out_kv, int* out_count, fdb_bool_t* out_more);
private:
friend class Transaction;
KeyValueArrayFuture(FDBFuture* f) : Future(f) {}
private:
friend class Transaction;
KeyValueArrayFuture(FDBFuture* f) : Future(f) {}
};
class EmptyFuture : public Future {
private:
friend class Transaction;
friend class Database;
EmptyFuture(FDBFuture* f) : Future(f) {}
private:
friend class Transaction;
friend class Database;
EmptyFuture(FDBFuture* f) : Future(f) {}
};
// Wrapper around FDBDatabase, providing database-level API
class Database final {
public:
static Int64Future reboot_worker(FDBDatabase* db, const uint8_t* address, int address_length, fdb_bool_t check,
int duration);
static EmptyFuture force_recovery_with_data_loss(FDBDatabase* db, const uint8_t* dcid, int dcid_length);
static EmptyFuture create_snapshot(FDBDatabase* db, const uint8_t* uid, int uid_length, const uint8_t* snap_command,
int snap_command_length);
static Int64Future reboot_worker(FDBDatabase* db,
const uint8_t* address,
int address_length,
fdb_bool_t check,
int duration);
static EmptyFuture force_recovery_with_data_loss(FDBDatabase* db, const uint8_t* dcid, int dcid_length);
static EmptyFuture create_snapshot(FDBDatabase* db,
const uint8_t* uid,
int uid_length,
const uint8_t* snap_command,
int snap_command_length);
};
// Wrapper around FDBTransaction, providing the same set of calls as the C API.
// Handles cleanup of memory, removing the need to call
// fdb_transaction_destroy.
class Transaction final {
public:
// Given an FDBDatabase, initializes a new transaction.
Transaction(FDBDatabase* db);
~Transaction();
public:
// Given an FDBDatabase, initializes a new transaction.
Transaction(FDBDatabase* db);
~Transaction();
// Wrapper around fdb_transaction_reset.
void reset();
// Wrapper around fdb_transaction_reset.
void reset();
// Wrapper around fdb_transaction_cancel.
void cancel();
// Wrapper around fdb_transaction_cancel.
void cancel();
// Wrapper around fdb_transaction_set_option.
fdb_error_t set_option(FDBTransactionOption option, const uint8_t* value,
int value_length);
// Wrapper around fdb_transaction_set_option.
fdb_error_t set_option(FDBTransactionOption option, const uint8_t* value, int value_length);
// Wrapper around fdb_transaction_set_read_version.
void set_read_version(int64_t version);
// Wrapper around fdb_transaction_set_read_version.
void set_read_version(int64_t version);
// Returns a future which will be set to the transaction read version.
Int64Future get_read_version();
// Returns a future which will be set to the transaction read version.
Int64Future get_read_version();
// Returns a future which will be set to the approximate transaction size so far.
Int64Future get_approximate_size();
// Returns a future which will be set to the approximate transaction size so far.
Int64Future get_approximate_size();
// Returns a future which will be set to the versionstamp which was used by
// any versionstamp operations in the transaction.
KeyFuture get_versionstamp();
// Returns a future which will be set to the versionstamp which was used by
// any versionstamp operations in the transaction.
KeyFuture get_versionstamp();
// Returns a future which will be set to the value of `key` in the database.
ValueFuture get(std::string_view key, fdb_bool_t snapshot);
// Returns a future which will be set to the value of `key` in the database.
ValueFuture get(std::string_view key, fdb_bool_t snapshot);
// Returns a future which will be set to the key in the database matching the
// passed key selector.
KeyFuture get_key(const uint8_t* key_name, int key_name_length,
fdb_bool_t or_equal, int offset, fdb_bool_t snapshot);
// Returns a future which will be set to the key in the database matching the
// passed key selector.
KeyFuture get_key(const uint8_t* key_name,
int key_name_length,
fdb_bool_t or_equal,
int offset,
fdb_bool_t snapshot);
// Returns a future which will be set to an array of strings.
StringArrayFuture get_addresses_for_key(std::string_view key);
// Returns a future which will be set to an array of strings.
StringArrayFuture get_addresses_for_key(std::string_view key);
// Returns a future which will be set to an FDBKeyValue array.
KeyValueArrayFuture get_range(const uint8_t* begin_key_name,
int begin_key_name_length,
fdb_bool_t begin_or_equal, int begin_offset,
const uint8_t* end_key_name,
int end_key_name_length,
fdb_bool_t end_or_equal, int end_offset,
int limit, int target_bytes,
FDBStreamingMode mode, int iteration,
fdb_bool_t snapshot, fdb_bool_t reverse);
// Returns a future which will be set to an FDBKeyValue array.
KeyValueArrayFuture get_range(const uint8_t* begin_key_name,
int begin_key_name_length,
fdb_bool_t begin_or_equal,
int begin_offset,
const uint8_t* end_key_name,
int end_key_name_length,
fdb_bool_t end_or_equal,
int end_offset,
int limit,
int target_bytes,
FDBStreamingMode mode,
int iteration,
fdb_bool_t snapshot,
fdb_bool_t reverse);
// Wrapper around fdb_transaction_watch. Returns a future representing an
// empty value.
EmptyFuture watch(std::string_view key);
// Wrapper around fdb_transaction_watch. Returns a future representing an
// empty value.
EmptyFuture watch(std::string_view key);
// Wrapper around fdb_transaction_commit. Returns a future representing an
// empty value.
EmptyFuture commit();
// Wrapper around fdb_transaction_commit. Returns a future representing an
// empty value.
EmptyFuture commit();
// Wrapper around fdb_transaction_on_error. Returns a future representing an
// empty value.
EmptyFuture on_error(fdb_error_t err);
// Wrapper around fdb_transaction_on_error. Returns a future representing an
// empty value.
EmptyFuture on_error(fdb_error_t err);
// Wrapper around fdb_transaction_clear.
void clear(std::string_view key);
// Wrapper around fdb_transaction_clear.
void clear(std::string_view key);
// Wrapper around fdb_transaction_clear_range.
void clear_range(std::string_view begin_key, std::string_view end_key);
// Wrapper around fdb_transaction_clear_range.
void clear_range(std::string_view begin_key, std::string_view end_key);
// Wrapper around fdb_transaction_set.
void set(std::string_view key, std::string_view value);
// Wrapper around fdb_transaction_set.
void set(std::string_view key, std::string_view value);
// Wrapper around fdb_transaction_atomic_op.
void atomic_op(std::string_view key, const uint8_t* param, int param_length,
FDBMutationType operationType);
// Wrapper around fdb_transaction_atomic_op.
void atomic_op(std::string_view key, const uint8_t* param, int param_length, FDBMutationType operationType);
// Wrapper around fdb_transaction_get_committed_version.
fdb_error_t get_committed_version(int64_t* out_version);
// Wrapper around fdb_transaction_get_committed_version.
fdb_error_t get_committed_version(int64_t* out_version);
// Wrapper around fdb_transaction_add_conflict_range.
fdb_error_t add_conflict_range(std::string_view begin_key,
std::string_view end_key,
FDBConflictRangeType type);
// Wrapper around fdb_transaction_add_conflict_range.
fdb_error_t add_conflict_range(std::string_view begin_key, std::string_view end_key, FDBConflictRangeType type);
private:
FDBTransaction* tr_;
private:
FDBTransaction* tr_;
};
} // namespace fdb
} // namespace fdb

66
bindings/c/test/unit/setup_tests.cpp
View File

@ -29,47 +29,47 @@
#include "doctest.h"
void fdb_check(fdb_error_t e) {
if (e) {
std::cerr << fdb_get_error(e) << std::endl;
std::abort();
}
if (e) {
std::cerr << fdb_get_error(e) << std::endl;
std::abort();
}
}
TEST_CASE("setup") {
fdb_error_t err;
// Version passed here must be <= FDB_API_VERSION
err = fdb_select_api_version(9000);
CHECK(err);
fdb_error_t err;
// Version passed here must be <= FDB_API_VERSION
err = fdb_select_api_version(9000);
CHECK(err);
// Select current API version
fdb_check(fdb_select_api_version(700));
// Select current API version
fdb_check(fdb_select_api_version(700));
// Error to call again after a successful return
err = fdb_select_api_version(700);
CHECK(err);
// Error to call again after a successful return
err = fdb_select_api_version(700);
CHECK(err);
CHECK(fdb_get_max_api_version() >= 700);
CHECK(fdb_get_max_api_version() >= 700);
fdb_check(fdb_setup_network());
// Calling a second time should fail
err = fdb_setup_network();
CHECK(err);
fdb_check(fdb_setup_network());
// Calling a second time should fail
err = fdb_setup_network();
CHECK(err);
struct Context {
bool called = false;
};
Context context;
fdb_check(fdb_add_network_thread_completion_hook(
[](void *param) {
auto *context = static_cast<Context *>(param);
context->called = true;
},
&context));
struct Context {
bool called = false;
};
Context context;
fdb_check(fdb_add_network_thread_completion_hook(
[](void* param) {
auto* context = static_cast<Context*>(param);
context->called = true;
},
&context));
std::thread network_thread{&fdb_run_network};
std::thread network_thread{ &fdb_run_network };
CHECK(!context.called);
fdb_check(fdb_stop_network());
network_thread.join();
CHECK(context.called);
CHECK(!context.called);
fdb_check(fdb_stop_network());
network_thread.join();
CHECK(context.called);
}

3169
bindings/c/test/unit/unit_tests.cpp
View File

File diff suppressed because it is too large Load Diff

22
bindings/c/test/workloads/SimpleWorkload.cpp
View File

@ -104,7 +104,10 @@ struct SimpleWorkload : FDBWorkload {
unsigned long from, to, lastTx = 0;
std::unordered_map<State, ActorCallback> callbacks;
PopulateActor(const Callback& promise, SimpleWorkload& self, FDBDatabase* db, unsigned long from,
PopulateActor(const Callback& promise,
SimpleWorkload& self,
FDBDatabase* db,
unsigned long from,
unsigned long to)
: ActorBase(promise, self, db), from(from), to(to) {
error = fdb_database_create_transaction(db, &tx);
@ -130,8 +133,11 @@ struct SimpleWorkload : FDBWorkload {
for (; from < to && ops < self.insertsPerTx; ++ops, ++from) {
std::string value = std::to_string(from);
std::string key = KEY_PREFIX + value;
fdb_transaction_set(tx, reinterpret_cast<const uint8_t*>(key.c_str()), key.size(),
reinterpret_cast<const uint8_t*>(value.c_str()), value.size());
fdb_transaction_set(tx,
reinterpret_cast<const uint8_t*>(key.c_str()),
key.size(),
reinterpret_cast<const uint8_t*>(value.c_str()),
value.size());
}
lastTx = ops;
auto commit_future = fdb_transaction_commit(tx);
@ -154,7 +160,8 @@ struct SimpleWorkload : FDBWorkload {
run();
},
[this](fdb_error_t error) {
self.context->trace(FDBSeverity::Error, "AssertionFailure",
self.context->trace(FDBSeverity::Error,
"AssertionFailure",
{ { "Reason", "tx.onError failed" },
{ "Error", std::string(fdb_get_error(error)) } });
self.success = false;
@ -230,7 +237,8 @@ struct SimpleWorkload : FDBWorkload {
get();
},
[this](fdb_error_t) {
self.context->trace(FDBSeverity::Error, "AssertionFailure",
self.context->trace(FDBSeverity::Error,
"AssertionFailure",
{ { "Reason", "tx.onError failed" },
{ "Error", std::string(fdb_get_error(error)) } });
self.success = false;
@ -260,8 +268,8 @@ struct SimpleWorkload : FDBWorkload {
runFor = context->getOption("runFor", 10.0);
auto err = fdb_select_api_version(700);
if (err) {
context->trace(FDBSeverity::Info, "SelectAPIVersionFailed",
{ { "Error", std::string(fdb_get_error(err)) } });
context->trace(
FDBSeverity::Info, "SelectAPIVersionFailed", { { "Error", std::string(fdb_get_error(err)) } });
}
return true;
}

20
bindings/c/test/workloads/workloads.cpp
View File

@ -23,19 +23,19 @@
FDBWorkloadFactoryImpl::~FDBWorkloadFactoryImpl() {}
std::map<std::string, IFDBWorkloadFactory*>& FDBWorkloadFactoryImpl::factories() {
static std::map<std::string, IFDBWorkloadFactory*> _factories;
return _factories;
static std::map<std::string, IFDBWorkloadFactory*> _factories;
return _factories;
}
std::shared_ptr<FDBWorkload> FDBWorkloadFactoryImpl::create(const std::string &name) {
auto res = factories().find(name);
if (res == factories().end()) {
return nullptr;
}
return res->second->create();
std::shared_ptr<FDBWorkload> FDBWorkloadFactoryImpl::create(const std::string& name) {
auto res = factories().find(name);
if (res == factories().end()) {
return nullptr;
}
return res->second->create();
}
FDBWorkloadFactory* workloadFactory(FDBLogger*) {
static FDBWorkloadFactoryImpl impl;
return &impl;
static FDBWorkloadFactoryImpl impl;
return &impl;
}

10
bindings/c/test/workloads/workloads.h
View File

@ -33,15 +33,11 @@ struct FDBWorkloadFactoryImpl : FDBWorkloadFactory {
std::shared_ptr<FDBWorkload> create(const std::string& name) override;
};
template<class WorkloadType>
template <class WorkloadType>
struct FDBWorkloadFactoryT : IFDBWorkloadFactory {
explicit FDBWorkloadFactoryT(const std::string& name) {
FDBWorkloadFactoryImpl::factories()[name] = this;
}
explicit FDBWorkloadFactoryT(const std::string& name) { FDBWorkloadFactoryImpl::factories()[name] = this; }
std::shared_ptr<FDBWorkload> create() override {
return std::make_shared<WorkloadType>();
}
std::shared_ptr<FDBWorkload> create() override { return std::make_shared<WorkloadType>(); }
};
extern "C" DLLEXPORT FDBWorkloadFactory* workloadFactory(FDBLogger*);

901
bindings/flow/DirectoryLayer.actor.cpp
View File

@ -22,486 +22,541 @@
#include "DirectoryPartition.h"
namespace FDB {
const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = {1,0,0,0,0,0,0,0};
const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca");
const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer");
const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version");
const int64_t DirectoryLayer::SUB_DIR_KEY = 0;
const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = { 1, 0, 0, 0, 0, 0, 0, 0 };
const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca");
const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer");
const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version");
const int64_t DirectoryLayer::SUB_DIR_KEY = 0;
const uint32_t DirectoryLayer::VERSION[3] = {1, 0, 0};
const uint32_t DirectoryLayer::VERSION[3] = { 1, 0, 0 };
const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = LiteralStringRef("\xfe");
const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX);
const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace();
const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition");
const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = LiteralStringRef("\xfe");
const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX);
const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace();
const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition");
DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes) :
nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes),
rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY))
{ }
DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes)
: nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes),
rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY)) {}
Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const {
return nodeSubspace.get(prefix);
}
template<class T>
Optional<Subspace> DirectoryLayer::nodeWithPrefix(Optional<T> const& prefix) const {
if(!prefix.present()) {
return Optional<Subspace>();
}
return nodeWithPrefix(prefix.get());
}
ACTOR Future<DirectoryLayer::Node> find(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
state int pathIndex = 0;
state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path);
for(; pathIndex != path.size(); ++pathIndex) {
ASSERT(node.subspace.present());
Optional<FDBStandalone<ValueRef>> val = wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key()));
node.path.push_back(path[pathIndex]);
node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path);
DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
node = _node;
if(!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) {
return node;
}
}
if(!node.loadedMetadata) {
DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
node = _node;
}
return node;
}
IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const {
Path path;
path.reserve(this->path.size() + subpath.size());
path.insert(path.end(), this->path.begin(), this->path.end());
path.insert(path.end(), subpath.begin(), subpath.end());
return path;
}
Reference<DirectorySubspace> DirectoryLayer::contentsOfNode(Subspace const& node, Path const& path, Standalone<StringRef> const& layer) {
Standalone<StringRef> prefix = nodeSubspace.unpack(node.key()).getString(0);
if(layer == PARTITION_LAYER) {
return Reference<DirectorySubspace>(new DirectoryPartition(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this)));
}
else {
return Reference<DirectorySubspace>(new DirectorySubspace(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this), layer));
}
}
Reference<DirectorySubspace> DirectoryLayer::openInternal(Standalone<StringRef> const& layer, Node const& existingNode, bool allowOpen) {
if (!allowOpen) {
throw directory_already_exists();
}
if(layer.size() > 0 && layer != existingNode.layer) {
throw mismatched_layer();
}
return existingNode.getContents();
}
Future<Reference<DirectorySubspace>> DirectoryLayer::open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), false, true);
}
void DirectoryLayer::initializeDirectory(Reference<Transaction> const& tr) const {
tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12));
}
ACTOR Future<Void> checkVersionInternal(const DirectoryLayer* dirLayer, Reference<Transaction> tr, bool writeAccess) {
Optional<FDBStandalone<ValueRef>> versionBytes = wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY)));
if(!versionBytes.present()) {
if(writeAccess) {
dirLayer->initializeDirectory(tr);
}
return Void();
}
else {
if(versionBytes.get().size() != 12) {
throw invalid_directory_layer_metadata();
}
if(((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) {
throw incompatible_directory_version();
}
else if(((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) {
throw incompatible_directory_version();
}
}
return Void();
}
Future<Void> DirectoryLayer::checkVersion(Reference<Transaction> const& tr, bool writeAccess) const {
return checkVersionInternal(this, tr, writeAccess);
}
ACTOR Future<Standalone<StringRef>> getPrefix(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Optional<Standalone<StringRef>> prefix) {
if(!prefix.present()) {
Standalone<StringRef> allocated = wait(dirLayer->allocator.allocate(tr));
state Standalone<StringRef> finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key());
FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1));
if(result.size() > 0) {
throw directory_prefix_not_empty();
}
return finalPrefix;
}
return prefix.get();
}
ACTOR Future<Optional<Subspace>> nodeContainingKey(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Standalone<StringRef> key, bool snapshot) {
if(key.startsWith(dirLayer->nodeSubspace.key())) {
return dirLayer->rootNode;
}
KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key)));
FDBStandalone<RangeResultRef> result = wait(tr->getRange(range, 1, snapshot, true));
if(result.size() > 0) {
Standalone<StringRef> prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0);
if(key.startsWith(prevPrefix)) {
return dirLayer->nodeWithPrefix(prevPrefix);
}
}
Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const {
return nodeSubspace.get(prefix);
}
template <class T>
Optional<Subspace> DirectoryLayer::nodeWithPrefix(Optional<T> const& prefix) const {
if (!prefix.present()) {
return Optional<Subspace>();
}
ACTOR Future<bool> isPrefixFree(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Standalone<StringRef> prefix, bool snapshot){
if(!prefix.size()) {
return false;
}
return nodeWithPrefix(prefix.get());
}
Optional<Subspace> node = wait(nodeContainingKey(dirLayer, tr, prefix, snapshot));
if(node.present()) {
return false;
}
ACTOR Future<DirectoryLayer::Node> find(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
state int pathIndex = 0;
state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path);
FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot));
return !result.size();
for (; pathIndex != path.size(); ++pathIndex) {
ASSERT(node.subspace.present());
Optional<FDBStandalone<ValueRef>> val =
wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key()));
}
node.path.push_back(path[pathIndex]);
node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path);
ACTOR Future<Subspace> getParentNode(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
if(path.size() > 1) {
Reference<DirectorySubspace> parent = wait(dirLayer->createOrOpenInternal(tr, IDirectory::Path(path.begin(), path.end() - 1), StringRef(), Optional<Standalone<StringRef>>(), true, true));
return dirLayer->nodeWithPrefix(parent->key());
}
else {
return dirLayer->rootNode;
DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
node = _node;
if (!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) {
return node;
}
}
ACTOR Future<Reference<DirectorySubspace>> createInternal(
Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path,
Standalone<StringRef> layer, Optional<Standalone<StringRef>> prefix, bool allowCreate)
{
if(!allowCreate) {
throw directory_does_not_exist();
}
wait(dirLayer->checkVersion(tr, true));
state Standalone<StringRef> newPrefix = wait(getPrefix(dirLayer, tr, prefix));
bool isFree = wait(isPrefixFree(dirLayer, tr, newPrefix, !prefix.present()));
if(!isFree) {
throw directory_prefix_in_use();
}
Subspace parentNode = wait(getParentNode(dirLayer, tr, path));
Subspace node = dirLayer->nodeWithPrefix(newPrefix);
tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix);
tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer);
return dirLayer->contentsOfNode(node, path, layer);
if (!node.loadedMetadata) {
DirectoryLayer::Node _node = wait(node.loadMetadata(tr));
node = _node;
}
ACTOR Future<Reference<DirectorySubspace>> _createOrOpenInternal(
Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path,
Standalone<StringRef> layer, Optional<Standalone<StringRef>> prefix, bool allowCreate, bool allowOpen)
{
ASSERT(!prefix.present() || allowCreate);
wait(dirLayer->checkVersion(tr, false));
return node;
}
if(prefix.present() && !dirLayer->allowManualPrefixes) {
if(!dirLayer->getPath().size()) {
throw manual_prefixes_not_enabled();
}
else {
throw prefix_in_partition();
}
IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const {
Path path;
path.reserve(this->path.size() + subpath.size());
path.insert(path.end(), this->path.begin(), this->path.end());
path.insert(path.end(), subpath.begin(), subpath.end());
return path;
}
Reference<DirectorySubspace> DirectoryLayer::contentsOfNode(Subspace const& node,
Path const& path,
Standalone<StringRef> const& layer) {
Standalone<StringRef> prefix = nodeSubspace.unpack(node.key()).getString(0);
if (layer == PARTITION_LAYER) {
return Reference<DirectorySubspace>(
new DirectoryPartition(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this)));
} else {
return Reference<DirectorySubspace>(
new DirectorySubspace(toAbsolutePath(path), prefix, Reference<DirectoryLayer>::addRef(this), layer));
}
}
Reference<DirectorySubspace> DirectoryLayer::openInternal(Standalone<StringRef> const& layer,
Node const& existingNode,
bool allowOpen) {
if (!allowOpen) {
throw directory_already_exists();
}
if (layer.size() > 0 && layer != existingNode.layer) {
throw mismatched_layer();
}
return existingNode.getContents();
}
Future<Reference<DirectorySubspace>> DirectoryLayer::open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), false, true);
}
void DirectoryLayer::initializeDirectory(Reference<Transaction> const& tr) const {
tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12));
}
ACTOR Future<Void> checkVersionInternal(const DirectoryLayer* dirLayer, Reference<Transaction> tr, bool writeAccess) {
Optional<FDBStandalone<ValueRef>> versionBytes =
wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY)));
if (!versionBytes.present()) {
if (writeAccess) {
dirLayer->initializeDirectory(tr);
}
return Void();
} else {
if (versionBytes.get().size() != 12) {
throw invalid_directory_layer_metadata();
}
if (((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) {
throw incompatible_directory_version();
} else if (((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) {
throw incompatible_directory_version();
}
}
return Void();
}
Future<Void> DirectoryLayer::checkVersion(Reference<Transaction> const& tr, bool writeAccess) const {
return checkVersionInternal(this, tr, writeAccess);
}
ACTOR Future<Standalone<StringRef>> getPrefix(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Optional<Standalone<StringRef>> prefix) {
if (!prefix.present()) {
Standalone<StringRef> allocated = wait(dirLayer->allocator.allocate(tr));
state Standalone<StringRef> finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key());
FDBStandalone<RangeResultRef> result = wait(tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1));
if (result.size() > 0) {
throw directory_prefix_not_empty();
}
if(!path.size()){
throw cannot_open_root_directory();
}
return finalPrefix;
}
state DirectoryLayer::Node existingNode = wait(find(dirLayer, tr, path));
if(existingNode.exists()) {
if(existingNode.isInPartition()) {
IDirectory::Path subpath = existingNode.getPartitionSubpath();
Reference<DirectorySubspace> dirSpace = wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(tr, subpath, layer, prefix, allowCreate, allowOpen));
return dirSpace;
}
return dirLayer->openInternal(layer, existingNode, allowOpen);
return prefix.get();
}
ACTOR Future<Optional<Subspace>> nodeContainingKey(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Standalone<StringRef> key,
bool snapshot) {
if (key.startsWith(dirLayer->nodeSubspace.key())) {
return dirLayer->rootNode;
}
KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key)));
FDBStandalone<RangeResultRef> result = wait(tr->getRange(range, 1, snapshot, true));
if (result.size() > 0) {
Standalone<StringRef> prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0);
if (key.startsWith(prevPrefix)) {
return dirLayer->nodeWithPrefix(prevPrefix);
}
else {
Reference<DirectorySubspace> dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate));
}
return Optional<Subspace>();
}
ACTOR Future<bool> isPrefixFree(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
Standalone<StringRef> prefix,
bool snapshot) {
if (!prefix.size()) {
return false;
}
Optional<Subspace> node = wait(nodeContainingKey(dirLayer, tr, prefix, snapshot));
if (node.present()) {
return false;
}
FDBStandalone<RangeResultRef> result = wait(tr->getRange(
KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot));
return !result.size();
}
ACTOR Future<Subspace> getParentNode(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
if (path.size() > 1) {
Reference<DirectorySubspace> parent =
wait(dirLayer->createOrOpenInternal(tr,
IDirectory::Path(path.begin(), path.end() - 1),
StringRef(),
Optional<Standalone<StringRef>>(),
true,
true));
return dirLayer->nodeWithPrefix(parent->key());
} else {
return dirLayer->rootNode;
}
}
ACTOR Future<Reference<DirectorySubspace>> createInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
Standalone<StringRef> layer,
Optional<Standalone<StringRef>> prefix,
bool allowCreate) {
if (!allowCreate) {
throw directory_does_not_exist();
}
wait(dirLayer->checkVersion(tr, true));
state Standalone<StringRef> newPrefix = wait(getPrefix(dirLayer, tr, prefix));
bool isFree = wait(isPrefixFree(dirLayer, tr, newPrefix, !prefix.present()));
if (!isFree) {
throw directory_prefix_in_use();
}
Subspace parentNode = wait(getParentNode(dirLayer, tr, path));
Subspace node = dirLayer->nodeWithPrefix(newPrefix);
tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix);
tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer);
return dirLayer->contentsOfNode(node, path, layer);
}
ACTOR Future<Reference<DirectorySubspace>> _createOrOpenInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
Standalone<StringRef> layer,
Optional<Standalone<StringRef>> prefix,
bool allowCreate,
bool allowOpen) {
ASSERT(!prefix.present() || allowCreate);
wait(dirLayer->checkVersion(tr, false));
if (prefix.present() && !dirLayer->allowManualPrefixes) {
if (!dirLayer->getPath().size()) {
throw manual_prefixes_not_enabled();
} else {
throw prefix_in_partition();
}
}
if (!path.size()) {
throw cannot_open_root_directory();
}
state DirectoryLayer::Node existingNode = wait(find(dirLayer, tr, path));
if (existingNode.exists()) {
if (existingNode.isInPartition()) {
IDirectory::Path subpath = existingNode.getPartitionSubpath();
Reference<DirectorySubspace> dirSpace =
wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(
tr, subpath, layer, prefix, allowCreate, allowOpen));
return dirSpace;
}
return dirLayer->openInternal(layer, existingNode, allowOpen);
} else {
Reference<DirectorySubspace> dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate));
return dirSpace;
}
}
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpenInternal(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix,
bool allowCreate,
bool allowOpen) {
return _createOrOpenInternal(
Reference<DirectoryLayer>::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::create(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix) {
return createOrOpenInternal(tr, path, layer, prefix, true, false);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), true, true);
}
ACTOR Future<Standalone<VectorRef<StringRef>>> listInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if (!node.exists()) {
throw directory_does_not_exist();
}
if (node.isInPartition(true)) {
Standalone<VectorRef<StringRef>> partitionList =
wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath()));
return partitionList;
}
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpenInternal(
Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix, bool allowCreate, bool allowOpen)
{
return _createOrOpenInternal(Reference<DirectoryLayer>::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen);
}
state Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY);
state Key begin = subdir.range().begin;
state Standalone<VectorRef<StringRef>> subdirectories;
Future<Reference<DirectorySubspace>> DirectoryLayer::create(
Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix)
{
return createOrOpenInternal(tr, path, layer, prefix, true, false);
}
loop {
FDBStandalone<RangeResultRef> subdirRange = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
Future<Reference<DirectorySubspace>> DirectoryLayer::createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
return createOrOpenInternal(tr, path, layer, Optional<Standalone<StringRef>>(), true, true);
}
ACTOR Future<Standalone<VectorRef<StringRef>>> listInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if(!node.exists()) {
throw directory_does_not_exist();
}
if(node.isInPartition(true)) {
Standalone<VectorRef<StringRef>> partitionList = wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath()));
return partitionList;
for (int i = 0; i < subdirRange.size(); ++i) {
subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0));
}
state Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY);
state Key begin = subdir.range().begin;
state Standalone<VectorRef<StringRef>> subdirectories;
loop {
FDBStandalone<RangeResultRef> subdirRange = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
for(int i = 0; i < subdirRange.size(); ++i) {
subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0));
}
if(!subdirRange.more) {
return subdirectories;
}
begin = keyAfter(subdirRange.back().key);
if (!subdirRange.more) {
return subdirectories;
}
}
Future<Standalone<VectorRef<StringRef>>> DirectoryLayer::list(Reference<Transaction> const& tr, Path const& path) {
return listInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
begin = keyAfter(subdirRange.back().key);
}
}
bool pathsEqual(IDirectory::Path const& path1, IDirectory::Path const& path2, size_t maxElementsToCheck = std::numeric_limits<size_t>::max()) {
if(std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) {
Future<Standalone<VectorRef<StringRef>>> DirectoryLayer::list(Reference<Transaction> const& tr, Path const& path) {
return listInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
}
bool pathsEqual(IDirectory::Path const& path1,
IDirectory::Path const& path2,
size_t maxElementsToCheck = std::numeric_limits<size_t>::max()) {
if (std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) {
return false;
}
for (int i = 0; i < path1.size() && i < maxElementsToCheck; ++i) {
if (path1[i] != path2[i]) {
return false;
}
for(int i = 0; i < path1.size() && i < maxElementsToCheck; ++i) {
if(path1[i] != path2[i]) {
return false;
}
}
return true;
}
ACTOR Future<Void> removeFromParent(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
ASSERT(path.size() >= 1);
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1)));
if(parentNode.subspace.present()) {
tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key());
}
return true;
}
return Void();
ACTOR Future<Void> removeFromParent(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
ASSERT(path.size() >= 1);
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1)));
if (parentNode.subspace.present()) {
tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key());
}
ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path oldPath, IDirectory::Path newPath) {
wait(dirLayer->checkVersion(tr, true));
return Void();
}
if(oldPath.size() <= newPath.size()) {
if(pathsEqual(oldPath, newPath, oldPath.size())) {
throw invalid_destination_directory();
}
ACTOR Future<Reference<DirectorySubspace>> moveInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path oldPath,
IDirectory::Path newPath) {
wait(dirLayer->checkVersion(tr, true));
if (oldPath.size() <= newPath.size()) {
if (pathsEqual(oldPath, newPath, oldPath.size())) {
throw invalid_destination_directory();
}
std::vector<Future<DirectoryLayer::Node>> futures;
futures.push_back(find(dirLayer, tr, oldPath));
futures.push_back(find(dirLayer, tr, newPath));
std::vector<DirectoryLayer::Node> nodes = wait(getAll(futures));
state DirectoryLayer::Node oldNode = nodes[0];
state DirectoryLayer::Node newNode = nodes[1];
if(!oldNode.exists()) {
throw directory_does_not_exist();
}
if(oldNode.isInPartition() || newNode.isInPartition()) {
if(!oldNode.isInPartition() || !newNode.isInPartition() || !pathsEqual(oldNode.path, newNode.path)) {
throw cannot_move_directory_between_partitions();
}
Reference<DirectorySubspace> partitionMove = wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath()));
return partitionMove;
}
if(newNode.exists() || newPath.empty()) {
throw directory_already_exists();
}
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1)));
if(!parentNode.exists()) {
throw parent_directory_does_not_exist();
}
tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(), dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0));
wait(removeFromParent(dirLayer, tr, oldPath));
return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) {
return moveInternal(Reference<DirectoryLayer>::addRef(this), tr, oldPath, newPath);
std::vector<Future<DirectoryLayer::Node>> futures;
futures.push_back(find(dirLayer, tr, oldPath));
futures.push_back(find(dirLayer, tr, newPath));
std::vector<DirectoryLayer::Node> nodes = wait(getAll(futures));
state DirectoryLayer::Node oldNode = nodes[0];
state DirectoryLayer::Node newNode = nodes[1];
if (!oldNode.exists()) {
throw directory_does_not_exist();
}
Future<Reference<DirectorySubspace>> DirectoryLayer::moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) {
if (oldNode.isInPartition() || newNode.isInPartition()) {
if (!oldNode.isInPartition() || !newNode.isInPartition() || !pathsEqual(oldNode.path, newNode.path)) {
throw cannot_move_directory_between_partitions();
}
Reference<DirectorySubspace> partitionMove =
wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath()));
return partitionMove;
}
if (newNode.exists() || newPath.empty()) {
throw directory_already_exists();
}
DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1)));
if (!parentNode.exists()) {
throw parent_directory_does_not_exist();
}
tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(),
dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0));
wait(removeFromParent(dirLayer, tr, oldPath));
return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) {
return moveInternal(Reference<DirectoryLayer>::addRef(this), tr, oldPath, newPath);
}
Future<Reference<DirectorySubspace>> DirectoryLayer::moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) {
throw cannot_modify_root_directory();
}
Future<Void> removeRecursive(Reference<DirectoryLayer> const&, Reference<Transaction> const&, Subspace const&);
ACTOR Future<Void> removeRecursive(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Subspace nodeSub) {
state Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY);
state Key begin = subdir.range().begin;
state std::vector<Future<Void>> futures;
loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
for (int i = 0; i < range.size(); ++i) {
Subspace subNode = dirLayer->nodeWithPrefix(range[i].value);
futures.push_back(removeRecursive(dirLayer, tr, subNode));
}
if (!range.more) {
break;
}
begin = keyAfter(range.back().key);
}
// waits are done concurrently
wait(waitForAll(futures));
Standalone<StringRef> nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0);
tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix)));
tr->clear(nodeSub.range());
return Void();
}
Future<bool> removeInternal(Reference<DirectoryLayer> const&,
Reference<Transaction> const&,
IDirectory::Path const&,
bool const&);
ACTOR Future<bool> removeInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path,
bool failOnNonexistent) {
wait(dirLayer->checkVersion(tr, true));
if (path.empty()) {
throw cannot_modify_root_directory();
}
Future<Void> removeRecursive(Reference<DirectoryLayer> const&, Reference<Transaction> const&, Subspace const&);
ACTOR Future<Void> removeRecursive(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, Subspace nodeSub) {
state Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY);
state Key begin = subdir.range().begin;
state std::vector<Future<Void>> futures;
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end)));
for (int i = 0; i < range.size(); ++i) {
Subspace subNode = dirLayer->nodeWithPrefix(range[i].value);
futures.push_back(removeRecursive(dirLayer, tr, subNode));
}
if(!range.more) {
break;
}
begin = keyAfter(range.back().key);
}
// waits are done concurrently
wait(waitForAll(futures));
Standalone<StringRef> nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0);
tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix)));
tr->clear(nodeSub.range());
return Void();
}
Future<bool> removeInternal(Reference<DirectoryLayer> const&, Reference<Transaction> const&, IDirectory::Path const&, bool const&);
ACTOR Future<bool> removeInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path, bool failOnNonexistent) {
wait(dirLayer->checkVersion(tr, true));
if(path.empty()) {
throw cannot_modify_root_directory();
}
state DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if(!node.exists()) {
if(failOnNonexistent) {
throw directory_does_not_exist();
}
else {
return false;
}
}
if(node.isInPartition()) {
bool recurse = wait(removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent));
return recurse;
}
state std::vector<Future<Void>> futures;
futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get()));
futures.push_back(removeFromParent(dirLayer, tr, path));
wait(waitForAll(futures));
return true;
}
Future<Void> DirectoryLayer::remove(Reference<Transaction> const& tr, Path const& path) {
return success(removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, true));
}
Future<bool> DirectoryLayer::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
return removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, false);
}
ACTOR Future<bool> existsInternal(Reference<DirectoryLayer> dirLayer, Reference<Transaction> tr, IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if(!node.exists()) {
if (!node.exists()) {
if (failOnNonexistent) {
throw directory_does_not_exist();
} else {
return false;
}
if(node.isInPartition()) {
bool exists = wait(node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath()));
return exists;
}
return true;
}
Future<bool> DirectoryLayer::exists(Reference<Transaction> const& tr, Path const& path) {
return existsInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
if (node.isInPartition()) {
bool recurse = wait(
removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent));
return recurse;
}
Reference<DirectoryLayer> DirectoryLayer::getDirectoryLayer() {
return Reference<DirectoryLayer>::addRef(this);
}
state std::vector<Future<Void>> futures;
futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get()));
futures.push_back(removeFromParent(dirLayer, tr, path));
const Standalone<StringRef> DirectoryLayer::getLayer() const {
return StringRef();
}
wait(waitForAll(futures));
const IDirectory::Path DirectoryLayer::getPath() const {
return path;
}
return true;
}
Future<Void> DirectoryLayer::remove(Reference<Transaction> const& tr, Path const& path) {
return success(removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, true));
}
Future<bool> DirectoryLayer::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
return removeInternal(Reference<DirectoryLayer>::addRef(this), tr, path, false);
}
ACTOR Future<bool> existsInternal(Reference<DirectoryLayer> dirLayer,
Reference<Transaction> tr,
IDirectory::Path path) {
wait(dirLayer->checkVersion(tr, false));
DirectoryLayer::Node node = wait(find(dirLayer, tr, path));
if (!node.exists()) {
return false;
}
if (node.isInPartition()) {
bool exists = wait(node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath()));
return exists;
}
return true;
}
Future<bool> DirectoryLayer::exists(Reference<Transaction> const& tr, Path const& path) {
return existsInternal(Reference<DirectoryLayer>::addRef(this), tr, path);
}
Reference<DirectoryLayer> DirectoryLayer::getDirectoryLayer() {
return Reference<DirectoryLayer>::addRef(this);
}
const Standalone<StringRef> DirectoryLayer::getLayer() const {
return StringRef();
}
const IDirectory::Path DirectoryLayer::getPath() const {
return path;
}
} // namespace FDB

154
bindings/flow/DirectoryLayer.h Executable file → Normal file
View File

@ -28,84 +28,108 @@
#include "HighContentionAllocator.h"
namespace FDB {
class DirectoryLayer : public IDirectory {
public:
DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE, Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE, bool allowManualPrefixes = false);
class DirectoryLayer : public IDirectory {
public:
DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE,
Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE,
bool allowManualPrefixes = false);
Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(), Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<Reference<DirectorySubspace>> create(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath);
Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath);
Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path());
Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path());
Reference<DirectoryLayer> getDirectoryLayer();
const Standalone<StringRef> getLayer() const;
const Path getPath() const;
Reference<DirectoryLayer> getDirectoryLayer();
const Standalone<StringRef> getLayer() const;
const Path getPath() const;
static const Subspace DEFAULT_NODE_SUBSPACE;
static const Subspace DEFAULT_CONTENT_SUBSPACE;
static const StringRef PARTITION_LAYER;
static const Subspace DEFAULT_NODE_SUBSPACE;
static const Subspace DEFAULT_CONTENT_SUBSPACE;
static const StringRef PARTITION_LAYER;
//private:
static const uint8_t LITTLE_ENDIAN_LONG_ONE[8];
static const StringRef HIGH_CONTENTION_KEY;
static const StringRef LAYER_KEY;
static const StringRef VERSION_KEY;
static const int64_t SUB_DIR_KEY;
static const uint32_t VERSION[3];
static const StringRef DEFAULT_NODE_SUBSPACE_PREFIX;
// private:
static const uint8_t LITTLE_ENDIAN_LONG_ONE[8];
static const StringRef HIGH_CONTENTION_KEY;
static const StringRef LAYER_KEY;
static const StringRef VERSION_KEY;
static const int64_t SUB_DIR_KEY;
static const uint32_t VERSION[3];
static const StringRef DEFAULT_NODE_SUBSPACE_PREFIX;
struct Node {
Node() {}
Node(Reference<DirectoryLayer> const& directoryLayer, Optional<Subspace> const& subspace, Path const& path, Path const& targetPath);
struct Node {
Node() {}
Node(Reference<DirectoryLayer> const& directoryLayer,
Optional<Subspace> const& subspace,
Path const& path,
Path const& targetPath);
bool exists() const;
bool exists() const;
Future<Node> loadMetadata(Reference<Transaction> tr);
void ensureMetadataLoaded() const;
Future<Node> loadMetadata(Reference<Transaction> tr);
void ensureMetadataLoaded() const;
bool isInPartition(bool includeEmptySubpath = false) const;
Path getPartitionSubpath() const;
Reference<DirectorySubspace> getContents() const;
Reference<DirectoryLayer> directoryLayer;
Optional<Subspace> subspace;
Path path;
Path targetPath;
Standalone<StringRef> layer;
bool loadedMetadata;
};
Reference<DirectorySubspace> openInternal(Standalone<StringRef> const& layer, Node const& existingNode, bool allowOpen);
Future<Reference<DirectorySubspace>> createOrOpenInternal(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer, Optional<Standalone<StringRef>> const& prefix, bool allowCreate, bool allowOpen);
void initializeDirectory(Reference<Transaction> const& tr) const;
Future<Void> checkVersion(Reference<Transaction> const& tr, bool writeAccess) const;
template <class T>
Optional<Subspace> nodeWithPrefix(Optional<T> const& prefix) const;
Subspace nodeWithPrefix(StringRef const& prefix) const;
Reference<DirectorySubspace> contentsOfNode(Subspace const& node, Path const& path, Standalone<StringRef> const& layer);
Path toAbsolutePath(Path const& subpath) const;
Subspace rootNode;
Subspace nodeSubspace;
Subspace contentSubspace;
HighContentionAllocator allocator;
bool allowManualPrefixes;
bool isInPartition(bool includeEmptySubpath = false) const;
Path getPartitionSubpath() const;
Reference<DirectorySubspace> getContents() const;
Reference<DirectoryLayer> directoryLayer;
Optional<Subspace> subspace;
Path path;
Path targetPath;
Standalone<StringRef> layer;
bool loadedMetadata;
};
}
Reference<DirectorySubspace> openInternal(Standalone<StringRef> const& layer,
Node const& existingNode,
bool allowOpen);
Future<Reference<DirectorySubspace>> createOrOpenInternal(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix,
bool allowCreate,
bool allowOpen);
void initializeDirectory(Reference<Transaction> const& tr) const;
Future<Void> checkVersion(Reference<Transaction> const& tr, bool writeAccess) const;
template <class T>
Optional<Subspace> nodeWithPrefix(Optional<T> const& prefix) const;
Subspace nodeWithPrefix(StringRef const& prefix) const;
Reference<DirectorySubspace> contentsOfNode(Subspace const& node,
Path const& path,
Standalone<StringRef> const& layer);
Path toAbsolutePath(Path const& subpath) const;
Subspace rootNode;
Subspace nodeSubspace;
Subspace contentSubspace;
HighContentionAllocator allocator;
bool allowManualPrefixes;
Path path;
};
} // namespace FDB
#endif

50
bindings/flow/DirectoryPartition.h
View File

@ -28,34 +28,38 @@
#include "DirectoryLayer.h"
namespace FDB {
class DirectoryPartition : public DirectorySubspace {
class DirectoryPartition : public DirectorySubspace {
public:
DirectoryPartition(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> parentDirectoryLayer)
: DirectorySubspace(path, prefix, Reference<DirectoryLayer>(new DirectoryLayer(Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)), Subspace(prefix))), DirectoryLayer::PARTITION_LAYER),
parentDirectoryLayer(parentDirectoryLayer)
{
this->directoryLayer->path = path;
}
virtual ~DirectoryPartition() {}
public:
DirectoryPartition(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> parentDirectoryLayer)
: DirectorySubspace(path,
prefix,
Reference<DirectoryLayer>(new DirectoryLayer(
Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)),
Subspace(prefix))),
DirectoryLayer::PARTITION_LAYER),
parentDirectoryLayer(parentDirectoryLayer) {
this->directoryLayer->path = path;
}
virtual ~DirectoryPartition() {}
virtual Key key() const { throw cannot_use_partition_as_subspace(); }
virtual bool contains(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); }
virtual Key key() const { throw cannot_use_partition_as_subspace(); }
virtual bool contains(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); }
virtual Key pack(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); }
virtual Tuple unpack(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); }
virtual KeyRange range(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); }
virtual Key pack(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); }
virtual Tuple unpack(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); }
virtual KeyRange range(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); }
virtual Subspace subspace(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); }
virtual Subspace get(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); }
virtual Subspace subspace(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); }
virtual Subspace get(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); }
protected:
Reference<DirectoryLayer> parentDirectoryLayer;
protected:
Reference<DirectoryLayer> parentDirectoryLayer;
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const {
return path.empty() ? parentDirectoryLayer : directoryLayer;
}
};
}
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const {
return path.empty() ? parentDirectoryLayer : directoryLayer;
}
};
} // namespace FDB
#endif

165
bindings/flow/DirectorySubspace.cpp Executable file → Normal file
View File

@ -21,89 +21,100 @@
#include "DirectorySubspace.h"
namespace FDB {
DirectorySubspace::DirectorySubspace(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> directoryLayer, Standalone<StringRef> const& layer)
: Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) { }
DirectorySubspace::DirectorySubspace(Path const& path,
StringRef const& prefix,
Reference<DirectoryLayer> directoryLayer,
Standalone<StringRef> const& layer)
: Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) {}
Future<Reference<DirectorySubspace>> DirectorySubspace::create(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix) {
return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix);
}
Future<Reference<DirectorySubspace>> DirectorySubspace::create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer,
Optional<Standalone<StringRef>> const& prefix)
{
return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix);
Future<Reference<DirectorySubspace>> DirectorySubspace::open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return directoryLayer->open(tr, getPartitionSubpath(path), layer);
}
Future<Reference<DirectorySubspace>> DirectorySubspace::createOrOpen(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer) {
return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer);
}
Future<bool> DirectorySubspace::exists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->exists(tr, getPartitionSubpath(path, directoryLayer));
}
Future<Standalone<VectorRef<StringRef>>> DirectorySubspace::list(Reference<Transaction> const& tr, Path const& path) {
return directoryLayer->list(tr, getPartitionSubpath(path));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) {
return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(Path());
Path directoryLayerPath = directoryLayer->getPath();
if (directoryLayerPath.size() > newAbsolutePath.size()) {
return cannot_move_directory_between_partitions();
}
Future<Reference<DirectorySubspace>> DirectorySubspace::open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
return directoryLayer->open(tr, getPartitionSubpath(path), layer);
}
Future<Reference<DirectorySubspace>> DirectorySubspace::createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer) {
return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer);
}
Future<bool> DirectorySubspace::exists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->exists(tr, getPartitionSubpath(path, directoryLayer));
}
Future<Standalone<VectorRef<StringRef>>> DirectorySubspace::list(Reference<Transaction> const& tr, Path const& path) {
return directoryLayer->list(tr, getPartitionSubpath(path));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) {
return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath));
}
Future<Reference<DirectorySubspace>> DirectorySubspace::moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(Path());
Path directoryLayerPath = directoryLayer->getPath();
if(directoryLayerPath.size() > newAbsolutePath.size()) {
for (int i = 0; i < directoryLayerPath.size(); ++i) {
if (directoryLayerPath[i] != newAbsolutePath[i]) {
return cannot_move_directory_between_partitions();
}
for(int i = 0; i < directoryLayerPath.size(); ++i) {
if(directoryLayerPath[i] != newAbsolutePath[i]) {
return cannot_move_directory_between_partitions();
}
}
Path newRelativePath(newAbsolutePath.begin() + directoryLayerPath.size(), newAbsolutePath.end());
return directoryLayer->move(tr, getPartitionSubpath(Path(), directoryLayer), newRelativePath);
}
Future<Void> DirectorySubspace::remove(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->remove(tr, getPartitionSubpath(path, directoryLayer));
}
Future<bool> DirectorySubspace::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->removeIfExists(tr, getPartitionSubpath(path, directoryLayer));
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayer() {
return directoryLayer;
}
const Standalone<StringRef> DirectorySubspace::getLayer() const {
return layer;
}
const IDirectory::Path DirectorySubspace::getPath() const {
return path;
}
IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path, Reference<DirectoryLayer> directoryLayer) const {
if(!directoryLayer) {
directoryLayer = this->directoryLayer;
}
Path newPath(this->path.begin() + directoryLayer->getPath().size(), this->path.end());
newPath.insert(newPath.end(), path.begin(), path.end());
return newPath;
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayerForPath(Path const& path) const {
return directoryLayer;
}
Path newRelativePath(newAbsolutePath.begin() + directoryLayerPath.size(), newAbsolutePath.end());
return directoryLayer->move(tr, getPartitionSubpath(Path(), directoryLayer), newRelativePath);
}
Future<Void> DirectorySubspace::remove(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->remove(tr, getPartitionSubpath(path, directoryLayer));
}
Future<bool> DirectorySubspace::removeIfExists(Reference<Transaction> const& tr, Path const& path) {
Reference<DirectoryLayer> directoryLayer = getDirectoryLayerForPath(path);
return directoryLayer->removeIfExists(tr, getPartitionSubpath(path, directoryLayer));
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayer() {
return directoryLayer;
}
const Standalone<StringRef> DirectorySubspace::getLayer() const {
return layer;
}
const IDirectory::Path DirectorySubspace::getPath() const {
return path;
}
IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path,
Reference<DirectoryLayer> directoryLayer) const {
if (!directoryLayer) {
directoryLayer = this->directoryLayer;
}
Path newPath(this->path.begin() + directoryLayer->getPath().size(), this->path.end());
newPath.insert(newPath.end(), path.begin(), path.end());
return newPath;
}
Reference<DirectoryLayer> DirectorySubspace::getDirectoryLayerForPath(Path const& path) const {
return directoryLayer;
}
} // namespace FDB

64
bindings/flow/DirectorySubspace.h Executable file → Normal file
View File

@ -28,39 +28,53 @@
#include "Subspace.h"
namespace FDB {
class DirectorySubspace : public IDirectory, public Subspace {
class DirectorySubspace : public IDirectory, public Subspace {
public:
DirectorySubspace(Path const& path, StringRef const& prefix, Reference<DirectoryLayer> directorLayer, Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual ~DirectorySubspace() {}
public:
DirectorySubspace(Path const& path,
StringRef const& prefix,
Reference<DirectoryLayer> directorLayer,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual ~DirectorySubspace() {}
virtual Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
virtual Future<Reference<DirectorySubspace>> create(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>());
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<Reference<DirectorySubspace>> createOrOpen(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>());
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath);
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath);
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath);
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path());
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path());
virtual Reference<DirectoryLayer> getDirectoryLayer();
virtual const Standalone<StringRef> getLayer() const;
virtual const Path getPath() const;
virtual Reference<DirectoryLayer> getDirectoryLayer();
virtual const Standalone<StringRef> getLayer() const;
virtual const Path getPath() const;
protected:
Reference<DirectoryLayer> directoryLayer;
Path path;
Standalone<StringRef> layer;
protected:
Reference<DirectoryLayer> directoryLayer;
Path path;
Standalone<StringRef> layer;
virtual Path getPartitionSubpath(Path const& path, Reference<DirectoryLayer> directoryLayer = Reference<DirectoryLayer>()) const;
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const;
};
}
virtual Path getPartitionSubpath(Path const& path,
Reference<DirectoryLayer> directoryLayer = Reference<DirectoryLayer>()) const;
virtual Reference<DirectoryLayer> getDirectoryLayerForPath(Path const& path) const;
};
} // namespace FDB
#endif

576
bindings/flow/FDBLoanerTypes.h Executable file → Normal file
View File

@ -22,292 +22,304 @@
#define FDB_FLOW_LOANER_TYPES_H
namespace FDB {
typedef StringRef KeyRef;
typedef StringRef ValueRef;
typedef StringRef KeyRef;
typedef StringRef ValueRef;
typedef int64_t Version;
typedef int64_t Version;
typedef Standalone<KeyRef> Key;
typedef Standalone<ValueRef> Value;
typedef Standalone<KeyRef> Key;
typedef Standalone<ValueRef> Value;
inline Key keyAfter( const KeyRef& key ) {
if(key == LiteralStringRef("\xff\xff"))
return key;
inline Key keyAfter(const KeyRef& key) {
if (key == LiteralStringRef("\xff\xff"))
return key;
Standalone<StringRef> r;
uint8_t* s = new (r.arena()) uint8_t[ key.size() + 1 ];
memcpy(s, key.begin(), key.size() );
s[key.size()] = 0;
((StringRef&) r) = StringRef( s, key.size() + 1 );
return r;
}
inline KeyRef keyAfter( const KeyRef& key, Arena& arena ) {
if(key == LiteralStringRef("\xff\xff"))
return key;
uint8_t* t = new ( arena ) uint8_t[ key.size()+1 ];
memcpy(t, key.begin(), key.size() );
t[key.size()] = 0;
return KeyRef(t,key.size()+1);
}
struct KeySelectorRef {
KeyRef key; // Find the last item less than key
bool orEqual; // (or equal to key, if this is true)
int offset; // and then move forward this many items (or backward if negative)
KeySelectorRef() {}
KeySelectorRef( const KeyRef& key, bool orEqual, int offset ) : key(key), orEqual(orEqual), offset(offset) {}
KeySelectorRef( Arena& arena, const KeySelectorRef& copyFrom ) : key(arena,copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {}
int expectedSize() const { return key.expectedSize(); }
// std::string toString() const {
// if (offset > 0) {
// if (orEqual) return format("firstGreaterThan(%s)%+d", printable(key).c_str(), offset-1);
// else return format("firstGreaterOrEqual(%s)%+d", printable(key).c_str(), offset-1);
// } else {
// if (orEqual) return format("lastLessOrEqual(%s)%+d", printable(key).c_str(), offset);
// else return format("lastLessThan(%s)%+d", printable(key).c_str(), offset);
// }
// }
bool isBackward() const { return !orEqual && offset<=0; } // True if the resolution of the KeySelector depends only on keys less than key
bool isFirstGreaterOrEqual() const { return !orEqual && offset==1; }
bool isFirstGreaterThan() const { return orEqual && offset==1; }
bool isLastLessOrEqual() const { return orEqual && offset==0; }
// True iff, regardless of the contents of the database, lhs must resolve to a key > rhs
bool isDefinitelyGreater( KeyRef const& k ) {
return offset >= 1 && ( isFirstGreaterOrEqual() ? key > k : key >= k );
}
// True iff, regardless of the contents of the database, lhs must resolve to a key < rhs
bool isDefinitelyLess( KeyRef const& k ) {
return offset <= 0 && ( isLastLessOrEqual() ? key < k : key <= k );
}
template <class Ar>
void serialize( Ar& ar ) {
serializer(ar, key, orEqual, offset);
}
};
inline bool operator == (const KeySelectorRef& lhs, const KeySelectorRef& rhs) { return lhs.key == rhs.key && lhs.orEqual==rhs.orEqual && lhs.offset==rhs.offset; }
inline KeySelectorRef lastLessThan( const KeyRef& k ) {
return KeySelectorRef( k, false, 0 );
}
inline KeySelectorRef lastLessOrEqual( const KeyRef& k ) {
return KeySelectorRef( k, true, 0 );
}
inline KeySelectorRef firstGreaterThan( const KeyRef& k ) {
return KeySelectorRef( k, true, +1 );
}
inline KeySelectorRef firstGreaterOrEqual( const KeyRef& k ) {
return KeySelectorRef( k, false, +1 );
}
inline KeySelectorRef operator + (const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset+off);
}
inline KeySelectorRef operator - (const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset-off);
}
typedef Standalone<KeySelectorRef> KeySelector;
struct KeyValueRef {
KeyRef key;
ValueRef value;
KeyValueRef() {}
KeyValueRef( const KeyRef& key, const ValueRef& value ) : key(key), value(value) {}
KeyValueRef( Arena& a, const KeyValueRef& copyFrom ) : key(a, copyFrom.key), value(a, copyFrom.value) {}
bool operator == ( const KeyValueRef& r ) const { return key == r.key && value == r.value; }
int expectedSize() const { return key.expectedSize() + value.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) { serializer(ar, key, value); }
struct OrderByKey {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const {
return a.key < b.key;
}
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a < b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key < b;
}
};
struct OrderByKeyBack {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const {
return a.key > b.key;
}
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a > b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key > b;
}
};
};
typedef Standalone<KeyValueRef> KeyValue;
struct RangeResultRef : VectorRef<KeyValueRef> {
bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the limits requested)
// False implies that no such values remain
Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or beginning if reverse) of the range
// which was read to produce these results. This is guarenteed to be less than the requested range.
bool readToBegin;
bool readThroughEnd;
RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {}
RangeResultRef( Arena& p, const RangeResultRef& toCopy ) : more( toCopy.more ), readToBegin( toCopy.readToBegin ), readThroughEnd( toCopy.readThroughEnd ), readThrough( toCopy.readThrough.present() ? KeyRef( p, toCopy.readThrough.get() ) : Optional<KeyRef>() ), VectorRef<KeyValueRef>( p, toCopy ) {}
RangeResultRef( const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>() ) : VectorRef<KeyValueRef>( value ), more( more ), readThrough( readThrough ), readToBegin( false ), readThroughEnd( false ) {}
RangeResultRef( bool readToBegin, bool readThroughEnd ) : more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) { }
template <class Ar>
void serialize( Ar& ar ) {
serializer(ar, ((VectorRef<KeyValueRef>&)*this), more, readThrough, readToBegin, readThroughEnd);
}
};
struct GetRangeLimits {
enum { ROW_LIMIT_UNLIMITED = -1, BYTE_LIMIT_UNLIMITED = -1 };
int rows;
int minRows;
int bytes;
GetRangeLimits() : rows( ROW_LIMIT_UNLIMITED ), minRows(1), bytes( BYTE_LIMIT_UNLIMITED ) {}
explicit GetRangeLimits( int rowLimit ) : rows( rowLimit ), minRows(1), bytes( BYTE_LIMIT_UNLIMITED ) {}
GetRangeLimits( int rowLimit, int byteLimit ) : rows( rowLimit ), minRows(1), bytes( byteLimit ) {}
void decrement( VectorRef<KeyValueRef> const& data );
void decrement( KeyValueRef const& data );
// True if either the row or byte limit has been reached
bool isReached();
// True if data would cause the row or byte limit to be reached
bool reachedBy( VectorRef<KeyValueRef> const& data );
bool hasByteLimit();
bool hasRowLimit();
bool hasSatisfiedMinRows();
bool isValid() { return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED)
&& (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED)
&& minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED); }
};
struct KeyRangeRef {
const KeyRef begin, end;
KeyRangeRef() {}
KeyRangeRef( const KeyRef& begin, const KeyRef& end ) : begin(begin), end(end) {
if( begin > end ) {
throw inverted_range();
}
}
KeyRangeRef( Arena& a, const KeyRangeRef& copyFrom ) : begin(a, copyFrom.begin), end(a, copyFrom.end) {}
bool operator == ( const KeyRangeRef& r ) const { return begin == r.begin && end == r.end; }
bool operator != ( const KeyRangeRef& r ) const { return begin != r.begin || end != r.end; }
bool contains( const KeyRef& key ) const { return begin <= key && key < end; }
bool contains( const KeyRangeRef& keys ) const { return begin <= keys.begin && keys.end <= end; }
bool intersects( const KeyRangeRef& keys ) const { return begin < keys.end && keys.begin < end; }
bool empty() const { return begin == end; }
Standalone<KeyRangeRef> withPrefix( const StringRef& prefix ) const {
return KeyRangeRef( begin.withPrefix(prefix), end.withPrefix(prefix) );
}
const KeyRangeRef& operator = (const KeyRangeRef& rhs) {
const_cast<KeyRef&>(begin) = rhs.begin;
const_cast<KeyRef&>(end) = rhs.end;
return *this;
}
int expectedSize() const { return begin.expectedSize() + end.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) {
serializer(ar, const_cast<KeyRef&>(begin), const_cast<KeyRef&>(end));
if( begin > end ) {
throw inverted_range();
};
}
struct ArbitraryOrder {
bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const {
if (a.begin < b.begin) return true;
if (a.begin > b.begin) return false;
return a.end < b.end;
}
};
};
inline KeyRangeRef operator & (const KeyRangeRef& lhs, const KeyRangeRef& rhs) {
KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end);
if (e < b)
return KeyRangeRef();
return KeyRangeRef(b,e);
}
typedef Standalone<KeyRangeRef> KeyRange;
template <class T>
static std::string describe(T const& item) {
return item.toString();
}
template <class K, class V>
static std::string describe(std::map<K, V> const& items, int max_items = -1) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto it = items.begin(); it != items.end(); it++) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1) s += ",";
s += describe(it->first) + "=>" + describe(it->second);
}
return s;
}
template <class T>
static std::string describeList(T const& items, int max_items) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto const& item : items) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1) s += ",";
s += describe(item);
}
return s;
}
template <class T>
static std::string describe(std::vector<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T>
static std::string describe(std::set<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T1, class T2>
static std::string describe(std::pair<T1, T2> const& pair) {
return "first: " + describe(pair.first) + " second: " + describe(pair.second);
}
Standalone<StringRef> r;
uint8_t* s = new (r.arena()) uint8_t[key.size() + 1];
memcpy(s, key.begin(), key.size());
s[key.size()] = 0;
((StringRef&)r) = StringRef(s, key.size() + 1);
return r;
}
inline KeyRef keyAfter(const KeyRef& key, Arena& arena) {
if (key == LiteralStringRef("\xff\xff"))
return key;
uint8_t* t = new (arena) uint8_t[key.size() + 1];
memcpy(t, key.begin(), key.size());
t[key.size()] = 0;
return KeyRef(t, key.size() + 1);
}
struct KeySelectorRef {
KeyRef key; // Find the last item less than key
bool orEqual; // (or equal to key, if this is true)
int offset; // and then move forward this many items (or backward if negative)
KeySelectorRef() {}
KeySelectorRef(const KeyRef& key, bool orEqual, int offset) : key(key), orEqual(orEqual), offset(offset) {}
KeySelectorRef(Arena& arena, const KeySelectorRef& copyFrom)
: key(arena, copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {}
int expectedSize() const { return key.expectedSize(); }
// std::string toString() const {
// if (offset > 0) {
// if (orEqual) return format("firstGreaterThan(%s)%+d", printable(key).c_str(), offset-1);
// else return format("firstGreaterOrEqual(%s)%+d", printable(key).c_str(), offset-1);
// } else {
// if (orEqual) return format("lastLessOrEqual(%s)%+d", printable(key).c_str(), offset);
// else return format("lastLessThan(%s)%+d", printable(key).c_str(), offset);
// }
// }
bool isBackward() const {
return !orEqual && offset <= 0;
} // True if the resolution of the KeySelector depends only on keys less than key
bool isFirstGreaterOrEqual() const { return !orEqual && offset == 1; }
bool isFirstGreaterThan() const { return orEqual && offset == 1; }
bool isLastLessOrEqual() const { return orEqual && offset == 0; }
// True iff, regardless of the contents of the database, lhs must resolve to a key > rhs
bool isDefinitelyGreater(KeyRef const& k) { return offset >= 1 && (isFirstGreaterOrEqual() ? key > k : key >= k); }
// True iff, regardless of the contents of the database, lhs must resolve to a key < rhs
bool isDefinitelyLess(KeyRef const& k) { return offset <= 0 && (isLastLessOrEqual() ? key < k : key <= k); }
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, key, orEqual, offset);
}
};
inline bool operator==(const KeySelectorRef& lhs, const KeySelectorRef& rhs) {
return lhs.key == rhs.key && lhs.orEqual == rhs.orEqual && lhs.offset == rhs.offset;
}
inline KeySelectorRef lastLessThan(const KeyRef& k) {
return KeySelectorRef(k, false, 0);
}
inline KeySelectorRef lastLessOrEqual(const KeyRef& k) {
return KeySelectorRef(k, true, 0);
}
inline KeySelectorRef firstGreaterThan(const KeyRef& k) {
return KeySelectorRef(k, true, +1);
}
inline KeySelectorRef firstGreaterOrEqual(const KeyRef& k) {
return KeySelectorRef(k, false, +1);
}
inline KeySelectorRef operator+(const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset + off);
}
inline KeySelectorRef operator-(const KeySelectorRef& s, int off) {
return KeySelectorRef(s.key, s.orEqual, s.offset - off);
}
typedef Standalone<KeySelectorRef> KeySelector;
struct KeyValueRef {
KeyRef key;
ValueRef value;
KeyValueRef() {}
KeyValueRef(const KeyRef& key, const ValueRef& value) : key(key), value(value) {}
KeyValueRef(Arena& a, const KeyValueRef& copyFrom) : key(a, copyFrom.key), value(a, copyFrom.value) {}
bool operator==(const KeyValueRef& r) const { return key == r.key && value == r.value; }
int expectedSize() const { return key.expectedSize() + value.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) {
serializer(ar, key, value);
}
struct OrderByKey {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key < b.key; }
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a < b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key < b;
}
};
struct OrderByKeyBack {
bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key > b.key; }
template <class T>
bool operator()(T const& a, KeyValueRef const& b) const {
return a > b.key;
}
template <class T>
bool operator()(KeyValueRef const& a, T const& b) const {
return a.key > b;
}
};
};
typedef Standalone<KeyValueRef> KeyValue;
struct RangeResultRef : VectorRef<KeyValueRef> {
bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the
// limits requested)
// False implies that no such values remain
Optional<KeyRef> readThrough; // Only present when 'more' is true. When present, this value represent the end (or
// beginning if reverse) of the range
// which was read to produce these results. This is guarenteed to be less than the requested range.
bool readToBegin;
bool readThroughEnd;
RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {}
RangeResultRef(Arena& p, const RangeResultRef& toCopy)
: more(toCopy.more), readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd),
readThrough(toCopy.readThrough.present() ? KeyRef(p, toCopy.readThrough.get()) : Optional<KeyRef>()),
VectorRef<KeyValueRef>(p, toCopy) {}
RangeResultRef(const VectorRef<KeyValueRef>& value, bool more, Optional<KeyRef> readThrough = Optional<KeyRef>())
: VectorRef<KeyValueRef>(value), more(more), readThrough(readThrough), readToBegin(false), readThroughEnd(false) {
}
RangeResultRef(bool readToBegin, bool readThroughEnd)
: more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, ((VectorRef<KeyValueRef>&)*this), more, readThrough, readToBegin, readThroughEnd);
}
};
struct GetRangeLimits {
enum { ROW_LIMIT_UNLIMITED = -1, BYTE_LIMIT_UNLIMITED = -1 };
int rows;
int minRows;
int bytes;
GetRangeLimits() : rows(ROW_LIMIT_UNLIMITED), minRows(1), bytes(BYTE_LIMIT_UNLIMITED) {}
explicit GetRangeLimits(int rowLimit) : rows(rowLimit), minRows(1), bytes(BYTE_LIMIT_UNLIMITED) {}
GetRangeLimits(int rowLimit, int byteLimit) : rows(rowLimit), minRows(1), bytes(byteLimit) {}
void decrement(VectorRef<KeyValueRef> const& data);
void decrement(KeyValueRef const& data);
// True if either the row or byte limit has been reached
bool isReached();
// True if data would cause the row or byte limit to be reached
bool reachedBy(VectorRef<KeyValueRef> const& data);
bool hasByteLimit();
bool hasRowLimit();
bool hasSatisfiedMinRows();
bool isValid() {
return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED) && (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED) &&
minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED);
}
};
struct KeyRangeRef {
const KeyRef begin, end;
KeyRangeRef() {}
KeyRangeRef(const KeyRef& begin, const KeyRef& end) : begin(begin), end(end) {
if (begin > end) {
throw inverted_range();
}
}
KeyRangeRef(Arena& a, const KeyRangeRef& copyFrom) : begin(a, copyFrom.begin), end(a, copyFrom.end) {}
bool operator==(const KeyRangeRef& r) const { return begin == r.begin && end == r.end; }
bool operator!=(const KeyRangeRef& r) const { return begin != r.begin || end != r.end; }
bool contains(const KeyRef& key) const { return begin <= key && key < end; }
bool contains(const KeyRangeRef& keys) const { return begin <= keys.begin && keys.end <= end; }
bool intersects(const KeyRangeRef& keys) const { return begin < keys.end && keys.begin < end; }
bool empty() const { return begin == end; }
Standalone<KeyRangeRef> withPrefix(const StringRef& prefix) const {
return KeyRangeRef(begin.withPrefix(prefix), end.withPrefix(prefix));
}
const KeyRangeRef& operator=(const KeyRangeRef& rhs) {
const_cast<KeyRef&>(begin) = rhs.begin;
const_cast<KeyRef&>(end) = rhs.end;
return *this;
}
int expectedSize() const { return begin.expectedSize() + end.expectedSize(); }
template <class Ar>
force_inline void serialize(Ar& ar) {
serializer(ar, const_cast<KeyRef&>(begin), const_cast<KeyRef&>(end));
if (begin > end) {
throw inverted_range();
};
}
struct ArbitraryOrder {
bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const {
if (a.begin < b.begin)
return true;
if (a.begin > b.begin)
return false;
return a.end < b.end;
}
};
};
inline KeyRangeRef operator&(const KeyRangeRef& lhs, const KeyRangeRef& rhs) {
KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end);
if (e < b)
return KeyRangeRef();
return KeyRangeRef(b, e);
}
typedef Standalone<KeyRangeRef> KeyRange;
template <class T>
static std::string describe(T const& item) {
return item.toString();
}
template <class K, class V>
static std::string describe(std::map<K, V> const& items, int max_items = -1) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto it = items.begin(); it != items.end(); it++) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1)
s += ",";
s += describe(it->first) + "=>" + describe(it->second);
}
return s;
}
template <class T>
static std::string describeList(T const& items, int max_items) {
if (!items.size())
return "[no items]";
std::string s;
int count = 0;
for (auto const& item : items) {
if (++count > max_items && max_items >= 0)
break;
if (count > 1)
s += ",";
s += describe(item);
}
return s;
}
template <class T>
static std::string describe(std::vector<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T>
static std::string describe(std::set<T> const& items, int max_items = -1) {
return describeList(items, max_items);
}
template <class T1, class T2>
static std::string describe(std::pair<T1, T2> const& pair) {
return "first: " + describe(pair.first) + " second: " + describe(pair.second);
}
} // namespace FDB
#endif /* FDB_LOANER_TYPES_H */

152
bindings/flow/HighContentionAllocator.actor.cpp
View File

@ -21,90 +21,90 @@
#include "HighContentionAllocator.h"
namespace FDB {
ACTOR Future<Standalone<StringRef>> _allocate(Reference<Transaction> tr, Subspace counters, Subspace recent){
state int64_t start = 0;
state int64_t window = 0;
ACTOR Future<Standalone<StringRef>> _allocate(Reference<Transaction> tr, Subspace counters, Subspace recent) {
state int64_t start = 0;
state int64_t window = 0;
loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(counters.range(), 1, true, true));
if (range.size() > 0) {
start = counters.unpack(range[0].key).getInt(0);
}
state bool windowAdvanced = false;
loop {
// if thread safety is needed, this should be locked {
if (windowAdvanced) {
tr->clear(KeyRangeRef(counters.key(), counters.get(start).key()));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->clear(KeyRangeRef(recent.key(), recent.get(start).key()));
}
int64_t inc = 1;
tr->atomicOp(counters.get(start).key(), StringRef((uint8_t*)&inc, 8), FDB_MUTATION_TYPE_ADD);
Future<Optional<FDBStandalone<ValueRef>>> countFuture = tr->get(counters.get(start).key(), true);
// }
Optional<FDBStandalone<ValueRef>> countValue = wait(countFuture);
int64_t count = 0;
if (countValue.present()) {
if (countValue.get().size() != 8) {
throw invalid_directory_layer_metadata();
}
count = *(int64_t*)countValue.get().begin();
}
window = HighContentionAllocator::windowSize(start);
if (count * 2 < window) {
break;
}
start += window;
windowAdvanced = true;
}
loop {
FDBStandalone<RangeResultRef> range = wait(tr->getRange(counters.range(), 1, true, true));
state int64_t candidate = deterministicRandom()->randomInt(start, start + window);
if(range.size() > 0) {
start = counters.unpack(range[0].key).getInt(0);
// if thread safety is needed, this should be locked {
state Future<FDBStandalone<RangeResultRef>> latestCounter = tr->getRange(counters.range(), 1, true, true);
state Future<Optional<FDBStandalone<ValueRef>>> candidateValue = tr->get(recent.get(candidate).key());
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->set(recent.get(candidate).key(), ValueRef());
// }
wait(success(latestCounter) && success(candidateValue));
int64_t currentWindowStart = 0;
if (latestCounter.get().size() > 0) {
currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0);
}
state bool windowAdvanced = false;
loop {
// if thread safety is needed, this should be locked {
if(windowAdvanced) {
tr->clear(KeyRangeRef(counters.key(), counters.get(start).key()));
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->clear(KeyRangeRef(recent.key(), recent.get(start).key()));
}
int64_t inc = 1;
tr->atomicOp(counters.get(start).key(), StringRef((uint8_t*)&inc, 8), FDB_MUTATION_TYPE_ADD);
Future<Optional<FDBStandalone<ValueRef>>> countFuture = tr->get(counters.get(start).key(), true);
// }
Optional<FDBStandalone<ValueRef>> countValue = wait(countFuture);
int64_t count = 0;
if(countValue.present()) {
if(countValue.get().size() != 8) {
throw invalid_directory_layer_metadata();
}
count = *(int64_t*)countValue.get().begin();
}
window = HighContentionAllocator::windowSize(start);
if(count * 2 < window) {
break;
}
start += window;
windowAdvanced = true;
if (currentWindowStart > start) {
break;
}
loop {
state int64_t candidate = deterministicRandom()->randomInt(start, start + window);
// if thread safety is needed, this should be locked {
state Future<FDBStandalone<RangeResultRef>> latestCounter = tr->getRange(counters.range(), 1, true, true);
state Future<Optional<FDBStandalone<ValueRef>>> candidateValue = tr->get(recent.get(candidate).key());
tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->set(recent.get(candidate).key(), ValueRef());
// }
wait(success(latestCounter) && success(candidateValue));
int64_t currentWindowStart = 0;
if(latestCounter.get().size() > 0) {
currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0);
}
if(currentWindowStart > start) {
break;
}
if(!candidateValue.get().present()) {
tr->addWriteConflictKey(recent.get(candidate).key());
return Tuple().append(candidate).pack();
}
if (!candidateValue.get().present()) {
tr->addWriteConflictKey(recent.get(candidate).key());
return Tuple().append(candidate).pack();
}
}
}
Future<Standalone<StringRef>> HighContentionAllocator::allocate(Reference<Transaction> const& tr) const {
return _allocate(tr, counters, recent);
}
int64_t HighContentionAllocator::windowSize(int64_t start) {
if (start < 255) {
return 64;
}
if (start < 65535) {
return 1024;
}
return 8192;
}
}
Future<Standalone<StringRef>> HighContentionAllocator::allocate(Reference<Transaction> const& tr) const {
return _allocate(tr, counters, recent);
}
int64_t HighContentionAllocator::windowSize(int64_t start) {
if (start < 255) {
return 64;
}
if (start < 65535) {
return 1024;
}
return 8192;
}
} // namespace FDB

21
bindings/flow/HighContentionAllocator.h
View File

@ -26,16 +26,17 @@
#include "Subspace.h"
namespace FDB {
class HighContentionAllocator {
public:
HighContentionAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {}
Future<Standalone<StringRef>> allocate(Reference<Transaction> const& tr) const;
class HighContentionAllocator {
public:
HighContentionAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {}
Future<Standalone<StringRef>> allocate(Reference<Transaction> const& tr) const;
static int64_t windowSize(int64_t start);
private:
Subspace counters;
Subspace recent;
};
}
static int64_t windowSize(int64_t start);
private:
Subspace counters;
Subspace recent;
};
} // namespace FDB
#endif

54
bindings/flow/IDirectory.h Executable file → Normal file
View File

@ -27,34 +27,46 @@
#include "bindings/flow/fdb_flow.h"
namespace FDB {
class DirectoryLayer;
class DirectorySubspace;
class DirectoryLayer;
class DirectorySubspace;
class IDirectory : public ReferenceCounted<IDirectory> {
public:
typedef std::vector<Standalone<StringRef>> Path;
class IDirectory : public ReferenceCounted<IDirectory> {
public:
typedef std::vector<Standalone<StringRef>> Path;
virtual Future<Reference<DirectorySubspace>> create(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()) = 0;
virtual Future<Reference<DirectorySubspace>> create(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>(),
Optional<Standalone<StringRef>> const& prefix = Optional<Standalone<StringRef>>()) = 0;
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<Reference<DirectorySubspace>> createOrOpen(Reference<Transaction> const& tr, Path const& path, Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<Reference<DirectorySubspace>> open(Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<Reference<DirectorySubspace>> createOrOpen(
Reference<Transaction> const& tr,
Path const& path,
Standalone<StringRef> const& layer = Standalone<StringRef>()) = 0;
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<bool> exists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<Standalone<VectorRef<StringRef>>> list(Reference<Transaction> const& tr,
Path const& path = Path()) = 0;
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr, Path const& oldPath, Path const& newPath) = 0;
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr, Path const& newAbsolutePath) = 0;
virtual Future<Reference<DirectorySubspace>> move(Reference<Transaction> const& tr,
Path const& oldPath,
Path const& newPath) = 0;
virtual Future<Reference<DirectorySubspace>> moveTo(Reference<Transaction> const& tr,
Path const& newAbsolutePath) = 0;
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<Void> remove(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Future<bool> removeIfExists(Reference<Transaction> const& tr, Path const& path = Path()) = 0;
virtual Reference<DirectoryLayer> getDirectoryLayer() = 0;
virtual const Standalone<StringRef> getLayer() const = 0;
virtual const Path getPath() const = 0;
virtual Reference<DirectoryLayer> getDirectoryLayer() = 0;
virtual const Standalone<StringRef> getLayer() const = 0;
virtual const Path getPath() const = 0;
virtual ~IDirectory() {};
};
}
virtual ~IDirectory(){};
};
} // namespace FDB
#endif

73
bindings/flow/Node.actor.cpp Executable file → Normal file
View File

@ -21,50 +21,49 @@
#include "DirectoryLayer.h"
namespace FDB {
DirectoryLayer::Node::Node(Reference<DirectoryLayer> const& directoryLayer, Optional<Subspace> const& subspace, IDirectory::Path const& path, IDirectory::Path const& targetPath)
: directoryLayer(directoryLayer),
subspace(subspace),
path(path),
targetPath(targetPath),
loadedMetadata(false)
{ }
DirectoryLayer::Node::Node(Reference<DirectoryLayer> const& directoryLayer,
Optional<Subspace> const& subspace,
IDirectory::Path const& path,
IDirectory::Path const& targetPath)
: directoryLayer(directoryLayer), subspace(subspace), path(path), targetPath(targetPath), loadedMetadata(false) {}
bool DirectoryLayer::Node::exists() const {
return subspace.present();
}
bool DirectoryLayer::Node::exists() const {
return subspace.present();
}
ACTOR Future<DirectoryLayer::Node> loadMetadata(DirectoryLayer::Node *n, Reference<Transaction> tr) {
if(!n->exists()){
n->loadedMetadata = true;
return *n;
}
Optional<FDBStandalone<ValueRef>> layer = wait(tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY)));
n->layer = layer.present() ? layer.get() : Standalone<StringRef>();
ACTOR Future<DirectoryLayer::Node> loadMetadata(DirectoryLayer::Node* n, Reference<Transaction> tr) {
if (!n->exists()) {
n->loadedMetadata = true;
return *n;
}
//Calls to loadMetadata must keep the Node alive while the future is outstanding
Future<DirectoryLayer::Node> DirectoryLayer::Node::loadMetadata(Reference<Transaction> tr) {
return FDB::loadMetadata(this, tr);
}
Optional<FDBStandalone<ValueRef>> layer = wait(tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY)));
bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const {
ASSERT(loadedMetadata);
return exists() && layer == DirectoryLayer::PARTITION_LAYER && (includeEmptySubpath || targetPath.size() > path.size());
}
n->layer = layer.present() ? layer.get() : Standalone<StringRef>();
n->loadedMetadata = true;
IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const {
return Path(targetPath.begin() + path.size(), targetPath.end());
}
Reference<DirectorySubspace> DirectoryLayer::Node::getContents() const {
ASSERT(exists());
ASSERT(loadedMetadata);
return directoryLayer->contentsOfNode(subspace.get(), path, layer);
return *n;
}
// Calls to loadMetadata must keep the Node alive while the future is outstanding
Future<DirectoryLayer::Node> DirectoryLayer::Node::loadMetadata(Reference<Transaction> tr) {
return FDB::loadMetadata(this, tr);
}
bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const {
ASSERT(loadedMetadata);
return exists() && layer == DirectoryLayer::PARTITION_LAYER &&
(includeEmptySubpath || targetPath.size() > path.size());
}
IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const {
return Path(targetPath.begin() + path.size(), targetPath.end());
}
Reference<DirectorySubspace> DirectoryLayer::Node::getContents() const {
ASSERT(exists());
ASSERT(loadedMetadata);
return directoryLayer->contentsOfNode(subspace.get(), path, layer);
}
} // namespace FDB

117
bindings/flow/Subspace.cpp Executable file → Normal file
View File

@ -21,71 +21,72 @@
#include "Subspace.h"
namespace FDB {
Subspace::Subspace(Tuple const& tuple, StringRef const& rawPrefix){
StringRef packed = tuple.pack();
Subspace::Subspace(Tuple const& tuple, StringRef const& rawPrefix) {
StringRef packed = tuple.pack();
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + packed.size());
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
this->rawPrefix.append(this->rawPrefix.arena(), packed.begin(), packed.size());
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + packed.size());
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
this->rawPrefix.append(this->rawPrefix.arena(), packed.begin(), packed.size());
}
Subspace::Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix) {
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + tuple.pack().size());
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
this->rawPrefix.append(this->rawPrefix.arena(), tuple.pack().begin(), tuple.pack().size());
}
Subspace::Subspace(StringRef const& rawPrefix) {
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
}
Subspace::~Subspace() {}
Key Subspace::key() const {
return StringRef(rawPrefix.begin(), rawPrefix.size());
}
Key Subspace::pack(const Tuple& tuple) const {
return tuple.pack().withPrefix(StringRef(rawPrefix.begin(), rawPrefix.size()));
}
Tuple Subspace::unpack(StringRef const& key) const {
if (!contains(key)) {
throw key_not_in_subspace();
}
return Tuple::unpack(key.substr(rawPrefix.size()));
}
Subspace::Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix) {
this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + tuple.pack().size());
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
this->rawPrefix.append(this->rawPrefix.arena(), tuple.pack().begin(), tuple.pack().size());
}
KeyRange Subspace::range(Tuple const& tuple) const {
VectorRef<uint8_t> begin;
VectorRef<uint8_t> end;
Subspace::Subspace(StringRef const& rawPrefix){
this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size());
}
KeyRange keyRange;
Subspace::~Subspace() { }
begin.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
begin.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
begin.push_back(keyRange.arena(), uint8_t('\x00'));
Key Subspace::key() const {
return StringRef(rawPrefix.begin(), rawPrefix.size());
}
end.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
end.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
end.push_back(keyRange.arena(), uint8_t('\xff'));
Key Subspace::pack(const Tuple& tuple) const {
return tuple.pack().withPrefix(StringRef(rawPrefix.begin(), rawPrefix.size()));
}
// FIXME: test that this uses the keyRange arena and doesn't create another one
keyRange.KeyRangeRef::operator=(
KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size())));
return keyRange;
}
Tuple Subspace::unpack(StringRef const& key) const {
if (!contains(key)) {
throw key_not_in_subspace();
}
return Tuple::unpack(key.substr(rawPrefix.size()));
}
bool Subspace::contains(KeyRef const& key) const {
return key.startsWith(StringRef(rawPrefix.begin(), rawPrefix.size()));
}
KeyRange Subspace::range(Tuple const& tuple) const {
VectorRef<uint8_t> begin;
VectorRef<uint8_t> end;
Subspace Subspace::subspace(Tuple const& tuple) const {
return Subspace(tuple, rawPrefix);
}
KeyRange keyRange;
begin.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
begin.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
begin.push_back(keyRange.arena(), uint8_t('\x00'));
end.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1);
end.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size());
end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size());
end.push_back(keyRange.arena(), uint8_t('\xff'));
// FIXME: test that this uses the keyRange arena and doesn't create another one
keyRange.KeyRangeRef::operator=(KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size())));
return keyRange;
}
bool Subspace::contains(KeyRef const& key) const {
return key.startsWith(StringRef(rawPrefix.begin(), rawPrefix.size()));
}
Subspace Subspace::subspace(Tuple const& tuple) const {
return Subspace(tuple, rawPrefix);
}
Subspace Subspace::get(Tuple const& tuple) const {
return subspace(tuple);
}
}
Subspace Subspace::get(Tuple const& tuple) const {
return subspace(tuple);
}
} // namespace FDB

98
bindings/flow/Subspace.h Executable file → Normal file
View File

@ -28,65 +28,65 @@
#include "Tuple.h"
namespace FDB {
class Subspace {
public:
Subspace(Tuple const& tuple = Tuple(), StringRef const& rawPrefix = StringRef());
Subspace(StringRef const& rawPrefix);
class Subspace {
public:
Subspace(Tuple const& tuple = Tuple(), StringRef const& rawPrefix = StringRef());
Subspace(StringRef const& rawPrefix);
virtual ~Subspace();
virtual ~Subspace();
virtual Key key() const;
virtual bool contains(KeyRef const& key) const;
virtual Key key() const;
virtual bool contains(KeyRef const& key) const;
virtual Key pack(Tuple const& tuple = Tuple()) const;
virtual Tuple unpack(KeyRef const& key) const;
virtual KeyRange range(Tuple const& tuple = Tuple()) const;
virtual Key pack(Tuple const& tuple = Tuple()) const;
virtual Tuple unpack(KeyRef const& key) const;
virtual KeyRange range(Tuple const& tuple = Tuple()) const;
template <class T>
Key pack(T const& item) const {
Tuple t;
t.append(item);
return pack(t);
}
template <class T>
Key pack(T const& item) const {
Tuple t;
t.append(item);
return pack(t);
}
Key packNested(Tuple const& item) const {
Tuple t;
t.appendNested(item);
return pack(t);
}
Key packNested(Tuple const& item) const {
Tuple t;
t.appendNested(item);
return pack(t);
}
Key pack(StringRef const& item, bool utf8=false) const {
Tuple t;
t.append(item, utf8);
return pack(t);
}
Key pack(StringRef const& item, bool utf8 = false) const {
Tuple t;
t.append(item, utf8);
return pack(t);
}
virtual Subspace subspace(Tuple const& tuple) const;
virtual Subspace get(Tuple const& tuple) const;
virtual Subspace subspace(Tuple const& tuple) const;
virtual Subspace get(Tuple const& tuple) const;
template <class T>
Subspace get(T const& item) const {
Tuple t;
t.append(item);
return get(t);
}
template <class T>
Subspace get(T const& item) const {
Tuple t;
t.append(item);
return get(t);
}
Subspace getNested(Tuple const& item) const {
Tuple t;
t.appendNested(item);
return get(t);
}
Subspace getNested(Tuple const& item) const {
Tuple t;
t.appendNested(item);
return get(t);
}
Subspace get(StringRef const& item, bool utf8=false) const {
Tuple t;
t.append(item, utf8);
return get(t);
}
Subspace get(StringRef const& item, bool utf8 = false) const {
Tuple t;
t.append(item, utf8);
return get(t);
}
private:
Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix);
Standalone<VectorRef<uint8_t>> rawPrefix;
};
}
private:
Subspace(Tuple const& tuple, Standalone<VectorRef<uint8_t>> const& rawPrefix);
Standalone<VectorRef<uint8_t>> rawPrefix;
};
} // namespace FDB
#endif

1147
bindings/flow/Tuple.cpp Executable file → Normal file
View File

File diff suppressed because it is too large Load Diff

143
bindings/flow/Tuple.h Executable file → Normal file
View File

@ -26,92 +26,93 @@
#include "bindings/flow/fdb_flow.h"
namespace FDB {
struct Uuid {
const static size_t SIZE;
struct Uuid {
const static size_t SIZE;
Uuid(StringRef const& data);
Uuid(StringRef const& data);
StringRef getData() const;
StringRef getData() const;
// Comparisons
bool operator==(Uuid const& other) const;
bool operator!=(Uuid const& other) const;
bool operator<(Uuid const& other) const;
bool operator<=(Uuid const& other) const;
bool operator>(Uuid const& other) const;
bool operator>=(Uuid const& other) const;
private:
Standalone<StringRef> data;
};
// Comparisons
bool operator==(Uuid const& other) const;
bool operator!=(Uuid const& other) const;
bool operator<(Uuid const& other) const;
bool operator<=(Uuid const& other) const;
bool operator>(Uuid const& other) const;
bool operator>=(Uuid const& other) const;
struct Tuple {
Tuple() {}
private:
Standalone<StringRef> data;
};
static Tuple unpack(StringRef const& str);
struct Tuple {
Tuple() {}
Tuple& append(Tuple const& tuple);
Tuple& append(StringRef const& str, bool utf8=false);
Tuple& append(int32_t);
Tuple& append(int64_t);
Tuple& append(bool);
Tuple& append(float);
Tuple& append(double);
Tuple& append(Uuid);
Tuple& appendNested(Tuple const&);
Tuple& appendNull();
static Tuple unpack(StringRef const& str);
StringRef pack() const { return StringRef(data.begin(), data.size()); }
Tuple& append(Tuple const& tuple);
Tuple& append(StringRef const& str, bool utf8 = false);
Tuple& append(int32_t);
Tuple& append(int64_t);
Tuple& append(bool);
Tuple& append(float);
Tuple& append(double);
Tuple& append(Uuid);
Tuple& appendNested(Tuple const&);
Tuple& appendNull();
template <typename T>
Tuple& operator<<(T const& t) {
return append(t);
}
StringRef pack() const { return StringRef(data.begin(), data.size()); }
enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED };
template <typename T>
Tuple& operator<<(T const& t) {
return append(t);
}
// this is number of elements, not length of data
size_t size() const { return offsets.size(); }
enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED };
ElementType getType(size_t index) const;
Standalone<StringRef> getString(size_t index) const;
int64_t getInt(size_t index) const;
bool getBool(size_t index) const;
float getFloat(size_t index) const;
double getDouble(size_t index) const;
Uuid getUuid(size_t index) const;
Tuple getNested(size_t index) const;
// this is number of elements, not length of data
size_t size() const { return offsets.size(); }
KeyRange range(Tuple const& tuple = Tuple()) const;
ElementType getType(size_t index) const;
Standalone<StringRef> getString(size_t index) const;
int64_t getInt(size_t index) const;
bool getBool(size_t index) const;
float getFloat(size_t index) const;
double getDouble(size_t index) const;
Uuid getUuid(size_t index) const;
Tuple getNested(size_t index) const;
Tuple subTuple(size_t beginIndex, size_t endIndex = std::numeric_limits<size_t>::max()) const;
KeyRange range(Tuple const& tuple = Tuple()) const;
// Comparisons
bool operator==(Tuple const& other) const;
bool operator!=(Tuple const& other) const;
bool operator<(Tuple const& other) const;
bool operator<=(Tuple const& other) const;
bool operator>(Tuple const& other) const;
bool operator>=(Tuple const& other) const;
Tuple subTuple(size_t beginIndex, size_t endIndex = std::numeric_limits<size_t>::max()) const;
private:
static const uint8_t NULL_CODE;
static const uint8_t BYTES_CODE;
static const uint8_t STRING_CODE;
static const uint8_t NESTED_CODE;
static const uint8_t INT_ZERO_CODE;
static const uint8_t POS_INT_END;
static const uint8_t NEG_INT_START;
static const uint8_t FLOAT_CODE;
static const uint8_t DOUBLE_CODE;
static const uint8_t FALSE_CODE;
static const uint8_t TRUE_CODE;
static const uint8_t UUID_CODE;
// Comparisons
bool operator==(Tuple const& other) const;
bool operator!=(Tuple const& other) const;
bool operator<(Tuple const& other) const;
bool operator<=(Tuple const& other) const;
bool operator>(Tuple const& other) const;
bool operator>=(Tuple const& other) const;
Tuple(const StringRef& data);
Tuple(Standalone<VectorRef<uint8_t>> data, std::vector<size_t> offsets);
Standalone<VectorRef<uint8_t>> data;
std::vector<size_t> offsets;
};
}
private:
static const uint8_t NULL_CODE;
static const uint8_t BYTES_CODE;
static const uint8_t STRING_CODE;
static const uint8_t NESTED_CODE;
static const uint8_t INT_ZERO_CODE;
static const uint8_t POS_INT_END;
static const uint8_t NEG_INT_START;
static const uint8_t FLOAT_CODE;
static const uint8_t DOUBLE_CODE;
static const uint8_t FALSE_CODE;
static const uint8_t TRUE_CODE;
static const uint8_t UUID_CODE;
Tuple(const StringRef& data);
Tuple(Standalone<VectorRef<uint8_t>> data, std::vector<size_t> offsets);
Standalone<VectorRef<uint8_t>> data;
std::vector<size_t> offsets;
};
} // namespace FDB
#endif /* _FDB_TUPLE_H_ */

778
bindings/flow/fdb_flow.actor.cpp
View File

@ -37,41 +37,42 @@ THREAD_FUNC networkThread(void* fdb) {
}
ACTOR Future<Void> _test() {
API *fdb = FDB::API::selectAPIVersion(700);
API* fdb = FDB::API::selectAPIVersion(700);
auto db = fdb->createDatabase();
state Reference<Transaction> tr = db->createTransaction();
// tr->setVersion(1);
Version ver = wait( tr->getReadVersion() );
Version ver = wait(tr->getReadVersion());
printf("%" PRId64 "\n", ver);
state std::vector< Future<Version> > versions;
state std::vector<Future<Version>> versions;
state double starttime = timer_monotonic();
state int i;
// for (i = 0; i < 100000; i++) {
// Version v = wait( tr->getReadVersion() );
// }
for ( i = 0; i < 100000; i++ ) {
versions.push_back( tr->getReadVersion() );
for (i = 0; i < 100000; i++) {
versions.push_back(tr->getReadVersion());
}
for ( i = 0; i < 100000; i++ ) {
Version v = wait( versions[i] );
for (i = 0; i < 100000; i++) {
Version v = wait(versions[i]);
}
// wait( waitForAllReady( versions ) );
printf("Elapsed: %lf\n", timer_monotonic() - starttime );
printf("Elapsed: %lf\n", timer_monotonic() - starttime);
tr->set( LiteralStringRef("foo"), LiteralStringRef("bar") );
tr->set(LiteralStringRef("foo"), LiteralStringRef("bar"));
Optional< FDBStandalone<ValueRef> > v = wait( tr->get( LiteralStringRef("foo") ) );
if ( v.present() ) {
printf("%s\n", v.get().toString().c_str() );
Optional<FDBStandalone<ValueRef>> v = wait(tr->get(LiteralStringRef("foo")));
if (v.present()) {
printf("%s\n", v.get().toString().c_str());
}
FDBStandalone<RangeResultRef> r = wait( tr->getRange( KeyRangeRef( LiteralStringRef("a"), LiteralStringRef("z") ), 100 ) );
FDBStandalone<RangeResultRef> r =
wait(tr->getRange(KeyRangeRef(LiteralStringRef("a"), LiteralStringRef("z")), 100));
for ( auto kv : r ) {
for (auto kv : r) {
printf("%s is %s\n", kv.key.toString().c_str(), kv.value.toString().c_str());
}
@ -80,7 +81,7 @@ ACTOR Future<Void> _test() {
}
void fdb_flow_test() {
API *fdb = FDB::API::selectAPIVersion(700);
API* fdb = FDB::API::selectAPIVersion(700);
fdb->setupNetwork();
startThread(networkThread, fdb);
@ -97,393 +98,430 @@ void fdb_flow_test() {
// FDB object used by bindings
namespace FDB {
class DatabaseImpl : public Database, NonCopyable {
public:
virtual ~DatabaseImpl() { fdb_database_destroy(db); }
class DatabaseImpl : public Database, NonCopyable {
public:
virtual ~DatabaseImpl() { fdb_database_destroy(db); }
Reference<Transaction> createTransaction() override;
void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) override;
Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
Future<Void> createSnapshot(const StringRef& uid, const StringRef& snap_command) override;
Reference<Transaction> createTransaction() override;
void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) override;
Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
Future<Void> createSnapshot(const StringRef& uid, const StringRef& snap_command) override;
private:
FDBDatabase* db;
explicit DatabaseImpl(FDBDatabase* db) : db(db) {}
private:
FDBDatabase* db;
explicit DatabaseImpl(FDBDatabase* db) : db(db) {}
friend class API;
};
friend class API;
};
class TransactionImpl : public Transaction, private NonCopyable, public FastAllocated<TransactionImpl> {
friend class DatabaseImpl;
class TransactionImpl : public Transaction, private NonCopyable, public FastAllocated<TransactionImpl> {
friend class DatabaseImpl;
public:
virtual ~TransactionImpl() {
if (tr) {
fdb_transaction_destroy(tr);
}
public:
virtual ~TransactionImpl() {
if (tr) {
fdb_transaction_destroy(tr);
}
void setReadVersion(Version v) override;
Future<Version> getReadVersion() override;
Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) override;
Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) override;
Future<Void> watch(const Key& key) override;
using Transaction::getRange;
Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin, const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(), bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override;
Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) override;
Future<FDBStandalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) override;
void addReadConflictRange(KeyRangeRef const& keys) override;
void addReadConflictKey(KeyRef const& key) override;
void addWriteConflictRange(KeyRangeRef const& keys) override;
void addWriteConflictKey(KeyRef const& key) override;
void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) override;
void set(const KeyRef& key, const ValueRef& value) override;
void clear(const KeyRangeRef& range) override;
void clear(const KeyRef& key) override;
Future<Void> commit() override;
Version getCommittedVersion() override;
Future<FDBStandalone<StringRef>> getVersionstamp() override;
void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Future<int64_t> getApproximateSize() override;
Future<Void> onError(Error const& e) override;
void cancel() override;
void reset() override;
TransactionImpl() : tr(nullptr) {}
TransactionImpl(TransactionImpl&& r) noexcept {
tr = r.tr;
r.tr = nullptr;
}
TransactionImpl& operator=(TransactionImpl&& r) noexcept {
tr = r.tr;
r.tr = nullptr;
return *this;
}
private:
FDBTransaction* tr;
explicit TransactionImpl(FDBDatabase* db);
};
static inline void throw_on_error( fdb_error_t e ) {
if (e)
throw Error(e);
}
void CFuture::blockUntilReady() {
throw_on_error( fdb_future_block_until_ready( f ) );
void setReadVersion(Version v) override;
Future<Version> getReadVersion() override;
Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) override;
Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) override;
Future<Void> watch(const Key& key) override;
using Transaction::getRange;
Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override;
Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) override;
Future<FDBStandalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) override;
void addReadConflictRange(KeyRangeRef const& keys) override;
void addReadConflictKey(KeyRef const& key) override;
void addWriteConflictRange(KeyRangeRef const& keys) override;
void addWriteConflictKey(KeyRef const& key) override;
void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) override;
void set(const KeyRef& key, const ValueRef& value) override;
void clear(const KeyRangeRef& range) override;
void clear(const KeyRef& key) override;
Future<Void> commit() override;
Version getCommittedVersion() override;
Future<FDBStandalone<StringRef>> getVersionstamp() override;
void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) override;
Future<int64_t> getApproximateSize() override;
Future<Void> onError(Error const& e) override;
void cancel() override;
void reset() override;
TransactionImpl() : tr(nullptr) {}
TransactionImpl(TransactionImpl&& r) noexcept {
tr = r.tr;
r.tr = nullptr;
}
TransactionImpl& operator=(TransactionImpl&& r) noexcept {
tr = r.tr;
r.tr = nullptr;
return *this;
}
void backToFutureCallback( FDBFuture* f, void* data ) {
g_network->onMainThread( Promise<Void>((SAV<Void>*)data), TaskPriority::DefaultOnMainThread ); // SOMEDAY: think about this priority
}
private:
FDBTransaction* tr;
// backToFuture<Type>( FDBFuture*, (FDBFuture* -> Type) ) -> Future<Type>
// Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread)
// and converts it into a Future<T> (with callbacks working on this thread, cancellation etc).
// You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type
ACTOR template<class T, class Function> static Future<T> backToFuture( FDBFuture* _f, Function convertValue ) {
state Reference<CFuture> f( new CFuture(_f) );
explicit TransactionImpl(FDBDatabase* db);
};
Promise<Void> ready;
Future<Void> onReady = ready.getFuture();
static inline void throw_on_error(fdb_error_t e) {
if (e)
throw Error(e);
}
throw_on_error( fdb_future_set_callback( f->f, backToFutureCallback, ready.extractRawPointer() ) );
wait( onReady );
void CFuture::blockUntilReady() {
throw_on_error(fdb_future_block_until_ready(f));
}
return convertValue( f );
}
void backToFutureCallback(FDBFuture* f, void* data) {
g_network->onMainThread(Promise<Void>((SAV<Void>*)data),
TaskPriority::DefaultOnMainThread); // SOMEDAY: think about this priority
}
void API::setNetworkOption( FDBNetworkOption option, Optional<StringRef> value ) {
if ( value.present() )
throw_on_error( fdb_network_set_option( option, value.get().begin(), value.get().size() ) );
else
throw_on_error( fdb_network_set_option( option, nullptr, 0 ) );
}
// backToFuture<Type>( FDBFuture*, (FDBFuture* -> Type) ) -> Future<Type>
// Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread)
// and converts it into a Future<T> (with callbacks working on this thread, cancellation etc).
// You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type
ACTOR template <class T, class Function>
static Future<T> backToFuture(FDBFuture* _f, Function convertValue) {
state Reference<CFuture> f(new CFuture(_f));
API* API::instance = nullptr;
API::API(int version) : version(version) {}
Promise<Void> ready;
Future<Void> onReady = ready.getFuture();
API* API::selectAPIVersion(int apiVersion) {
if(API::instance) {
if(apiVersion != API::instance->version) {
throw api_version_already_set();
}
else {
return API::instance;
}
}
throw_on_error(fdb_future_set_callback(f->f, backToFutureCallback, ready.extractRawPointer()));
wait(onReady);
if(apiVersion < 500 || apiVersion > FDB_API_VERSION) {
throw api_version_not_supported();
}
return convertValue(f);
}
throw_on_error( fdb_select_api_version_impl(apiVersion, FDB_API_VERSION) );
void API::setNetworkOption(FDBNetworkOption option, Optional<StringRef> value) {
if (value.present())
throw_on_error(fdb_network_set_option(option, value.get().begin(), value.get().size()));
else
throw_on_error(fdb_network_set_option(option, nullptr, 0));
}
API::instance = new API(apiVersion);
return API::instance;
}
API* API::instance = nullptr;
API::API(int version) : version(version) {}
bool API::isAPIVersionSelected() {
return API::instance != nullptr;
}
API* API::getInstance() {
if(API::instance == nullptr) {
throw api_version_unset();
}
else {
API* API::selectAPIVersion(int apiVersion) {
if (API::instance) {
if (apiVersion != API::instance->version) {
throw api_version_already_set();
} else {
return API::instance;
}
}
void API::setupNetwork() {
throw_on_error( fdb_setup_network() );
if (apiVersion < 500 || apiVersion > FDB_API_VERSION) {
throw api_version_not_supported();
}
void API::runNetwork() {
throw_on_error( fdb_run_network() );
throw_on_error(fdb_select_api_version_impl(apiVersion, FDB_API_VERSION));
API::instance = new API(apiVersion);
return API::instance;
}
bool API::isAPIVersionSelected() {
return API::instance != nullptr;
}
API* API::getInstance() {
if (API::instance == nullptr) {
throw api_version_unset();
} else {
return API::instance;
}
}
void API::stopNetwork() {
throw_on_error( fdb_stop_network() );
void API::setupNetwork() {
throw_on_error(fdb_setup_network());
}
void API::runNetwork() {
throw_on_error(fdb_run_network());
}
void API::stopNetwork() {
throw_on_error(fdb_stop_network());
}
bool API::evaluatePredicate(FDBErrorPredicate pred, Error const& e) {
return fdb_error_predicate(pred, e.code());
}
Reference<Database> API::createDatabase(std::string const& connFilename) {
FDBDatabase* db;
throw_on_error(fdb_create_database(connFilename.c_str(), &db));
return Reference<Database>(new DatabaseImpl(db));
}
int API::getAPIVersion() const {
return version;
}
Reference<Transaction> DatabaseImpl::createTransaction() {
return Reference<Transaction>(new TransactionImpl(db));
}
void DatabaseImpl::setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value) {
if (value.present())
throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size()));
else
throw_on_error(fdb_database_set_option(db, option, nullptr, 0));
}
Future<int64_t> DatabaseImpl::rebootWorker(const StringRef& address, bool check, int duration) {
return backToFuture<int64_t>(fdb_database_reboot_worker(db, address.begin(), address.size(), check, duration),
[](Reference<CFuture> f) {
int64_t res;
throw_on_error(fdb_future_get_int64(f->f, &res));
return res;
});
}
Future<Void> DatabaseImpl::forceRecoveryWithDataLoss(const StringRef& dcid) {
return backToFuture<Void>(fdb_database_force_recovery_with_data_loss(db, dcid.begin(), dcid.size()),
[](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
Future<Void> DatabaseImpl::createSnapshot(const StringRef& uid, const StringRef& snap_command) {
return backToFuture<Void>(
fdb_database_create_snapshot(db, uid.begin(), uid.size(), snap_command.begin(), snap_command.size()),
[](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
TransactionImpl::TransactionImpl(FDBDatabase* db) {
throw_on_error(fdb_database_create_transaction(db, &tr));
}
void TransactionImpl::setReadVersion(Version v) {
fdb_transaction_set_read_version(tr, v);
}
Future<Version> TransactionImpl::getReadVersion() {
return backToFuture<Version>(fdb_transaction_get_read_version(tr), [](Reference<CFuture> f) {
Version value;
throw_on_error(fdb_future_get_int64(f->f, &value));
return value;
});
}
Future<Optional<FDBStandalone<ValueRef>>> TransactionImpl::get(const Key& key, bool snapshot) {
return backToFuture<Optional<FDBStandalone<ValueRef>>>(
fdb_transaction_get(tr, key.begin(), key.size(), snapshot), [](Reference<CFuture> f) {
fdb_bool_t present;
uint8_t const* value;
int value_length;
throw_on_error(fdb_future_get_value(f->f, &present, &value, &value_length));
if (present) {
return Optional<FDBStandalone<ValueRef>>(FDBStandalone<ValueRef>(f, ValueRef(value, value_length)));
} else {
return Optional<FDBStandalone<ValueRef>>();
}
});
}
Future<Void> TransactionImpl::watch(const Key& key) {
return backToFuture<Void>(fdb_transaction_watch(tr, key.begin(), key.size()), [](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
Future<FDBStandalone<KeyRef>> TransactionImpl::getKey(const KeySelector& key, bool snapshot) {
return backToFuture<FDBStandalone<KeyRef>>(
fdb_transaction_get_key(tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot),
[](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error(fdb_future_get_key(f->f, &key, &key_length));
return FDBStandalone<KeyRef>(f, KeyRef(key, key_length));
});
}
Future<FDBStandalone<RangeResultRef>> TransactionImpl::getRange(const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits,
bool snapshot,
bool reverse,
FDBStreamingMode streamingMode) {
// FIXME: iteration
return backToFuture<FDBStandalone<RangeResultRef>>(
fdb_transaction_get_range(tr,
begin.key.begin(),
begin.key.size(),
begin.orEqual,
begin.offset,
end.key.begin(),
end.key.size(),
end.orEqual,
end.offset,
limits.rows,
limits.bytes,
streamingMode,
1,
snapshot,
reverse),
[](Reference<CFuture> f) {
FDBKeyValue const* kv;
int count;
fdb_bool_t more;
throw_on_error(fdb_future_get_keyvalue_array(f->f, &kv, &count, &more));
return FDBStandalone<RangeResultRef>(f,
RangeResultRef(VectorRef<KeyValueRef>((KeyValueRef*)kv, count), more));
});
}
Future<int64_t> TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) {
return backToFuture<int64_t>(fdb_transaction_get_estimated_range_size_bytes(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()),
[](Reference<CFuture> f) {
int64_t bytes;
throw_on_error(fdb_future_get_int64(f->f, &bytes));
return bytes;
});
}
Future<FDBStandalone<VectorRef<KeyRef>>> TransactionImpl::getRangeSplitPoints(const KeyRange& range,
int64_t chunkSize) {
return backToFuture<FDBStandalone<VectorRef<KeyRef>>>(
fdb_transaction_get_range_split_points(
tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size(), chunkSize),
[](Reference<CFuture> f) {
FDBKey const* ks;
int count;
throw_on_error(fdb_future_get_key_array(f->f, &ks, &count));
return FDBStandalone<VectorRef<KeyRef>>(f, VectorRef<KeyRef>((KeyRef*)ks, count));
});
}
void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) {
throw_on_error(fdb_transaction_add_conflict_range(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ));
}
void TransactionImpl::addReadConflictKey(KeyRef const& key) {
return addReadConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) {
throw_on_error(fdb_transaction_add_conflict_range(
tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE));
}
void TransactionImpl::addWriteConflictKey(KeyRef const& key) {
return addWriteConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) {
fdb_transaction_atomic_op(tr, key.begin(), key.size(), operand.begin(), operand.size(), operationType);
}
void TransactionImpl::set(const KeyRef& key, const ValueRef& value) {
fdb_transaction_set(tr, key.begin(), key.size(), value.begin(), value.size());
}
void TransactionImpl::clear(const KeyRangeRef& range) {
fdb_transaction_clear_range(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size());
}
void TransactionImpl::clear(const KeyRef& key) {
fdb_transaction_clear(tr, key.begin(), key.size());
}
Future<Void> TransactionImpl::commit() {
return backToFuture<Void>(fdb_transaction_commit(tr), [](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
Version TransactionImpl::getCommittedVersion() {
Version v;
throw_on_error(fdb_transaction_get_committed_version(tr, &v));
return v;
}
Future<FDBStandalone<StringRef>> TransactionImpl::getVersionstamp() {
return backToFuture<FDBStandalone<KeyRef>>(fdb_transaction_get_versionstamp(tr), [](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error(fdb_future_get_key(f->f, &key, &key_length));
return FDBStandalone<StringRef>(f, StringRef(key, key_length));
});
}
void TransactionImpl::setOption(FDBTransactionOption option, Optional<StringRef> value) {
if (value.present()) {
throw_on_error(fdb_transaction_set_option(tr, option, value.get().begin(), value.get().size()));
} else {
throw_on_error(fdb_transaction_set_option(tr, option, nullptr, 0));
}
}
bool API::evaluatePredicate(FDBErrorPredicate pred, Error const& e) {
return fdb_error_predicate( pred, e.code() );
}
Future<int64_t> TransactionImpl::getApproximateSize() {
return backToFuture<int64_t>(fdb_transaction_get_approximate_size(tr), [](Reference<CFuture> f) {
int64_t size = 0;
throw_on_error(fdb_future_get_int64(f->f, &size));
return size;
});
}
Reference<Database> API::createDatabase(std::string const& connFilename) {
FDBDatabase *db;
throw_on_error(fdb_create_database(connFilename.c_str(), &db));
return Reference<Database>(new DatabaseImpl(db));
}
Future<Void> TransactionImpl::onError(Error const& e) {
return backToFuture<Void>(fdb_transaction_on_error(tr, e.code()), [](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
int API::getAPIVersion() const {
return version;
}
void TransactionImpl::cancel() {
fdb_transaction_cancel(tr);
}
Reference<Transaction> DatabaseImpl::createTransaction() {
return Reference<Transaction>(new TransactionImpl(db));
}
void TransactionImpl::reset() {
fdb_transaction_reset(tr);
}
void DatabaseImpl::setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value) {
if (value.present())
throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size()));
else
throw_on_error(fdb_database_set_option(db, option, nullptr, 0));
}
Future<int64_t> DatabaseImpl::rebootWorker(const StringRef &address, bool check, int duration) {
return backToFuture<int64_t>( fdb_database_reboot_worker(db, address.begin(), address.size(), check, duration), [](Reference<CFuture> f) {
int64_t res;
throw_on_error(fdb_future_get_int64( f->f, &res ) );
return res;
} );
}
Future<Void> DatabaseImpl::forceRecoveryWithDataLoss(const StringRef &dcid) {
return backToFuture< Void > ( fdb_database_force_recovery_with_data_loss(db, dcid.begin(), dcid.size()), [](Reference<CFuture> f){
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
});
}
Future<Void> DatabaseImpl::createSnapshot(const StringRef& uid, const StringRef& snap_command) {
return backToFuture<Void>(
fdb_database_create_snapshot(db, uid.begin(), uid.size(), snap_command.begin(), snap_command.size()),
[](Reference<CFuture> f) {
throw_on_error(fdb_future_get_error(f->f));
return Void();
});
}
TransactionImpl::TransactionImpl(FDBDatabase* db) {
throw_on_error(fdb_database_create_transaction(db, &tr));
}
void TransactionImpl::setReadVersion(Version v) {
fdb_transaction_set_read_version( tr, v );
}
Future<Version> TransactionImpl::getReadVersion() {
return backToFuture<Version>( fdb_transaction_get_read_version( tr ), [](Reference<CFuture> f){
Version value;
throw_on_error( fdb_future_get_int64( f->f, &value ) );
return value;
} );
}
Future<Optional<FDBStandalone<ValueRef>>> TransactionImpl::get(const Key& key, bool snapshot) {
return backToFuture< Optional<FDBStandalone<ValueRef>> >( fdb_transaction_get( tr, key.begin(), key.size(), snapshot ), [](Reference<CFuture> f) {
fdb_bool_t present;
uint8_t const* value;
int value_length;
throw_on_error( fdb_future_get_value( f->f, &present, &value, &value_length ) );
if ( present ) {
return Optional<FDBStandalone<ValueRef>>( FDBStandalone<ValueRef>( f, ValueRef( value, value_length ) ) );
} else {
return Optional<FDBStandalone<ValueRef>>();
}
} );
}
Future<Void> TransactionImpl::watch(const Key& key) {
return backToFuture< Void >( fdb_transaction_watch( tr, key.begin(), key.size() ), [](Reference<CFuture> f) {
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
} );
}
Future<FDBStandalone<KeyRef>> TransactionImpl::getKey(const KeySelector& key, bool snapshot) {
return backToFuture< FDBStandalone<KeyRef> >( fdb_transaction_get_key( tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot ), [](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error( fdb_future_get_key( f->f, &key, &key_length ) );
return FDBStandalone<KeyRef>( f, KeyRef( key, key_length ) );
} );
}
Future<FDBStandalone<RangeResultRef>> TransactionImpl::getRange(const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, bool snapshot, bool reverse, FDBStreamingMode streamingMode) {
// FIXME: iteration
return backToFuture< FDBStandalone<RangeResultRef> >( fdb_transaction_get_range( tr, begin.key.begin(), begin.key.size(), begin.orEqual, begin.offset, end.key.begin(), end.key.size(), end.orEqual, end.offset, limits.rows, limits.bytes, streamingMode, 1, snapshot, reverse ), [](Reference<CFuture> f) {
FDBKeyValue const* kv;
int count;
fdb_bool_t more;
throw_on_error( fdb_future_get_keyvalue_array( f->f, &kv, &count, &more ) );
return FDBStandalone<RangeResultRef>( f, RangeResultRef( VectorRef<KeyValueRef>( (KeyValueRef*)kv, count ), more ) );
} );
}
Future<int64_t> TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) {
return backToFuture<int64_t>(fdb_transaction_get_estimated_range_size_bytes(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()), [](Reference<CFuture> f) {
int64_t bytes;
throw_on_error(fdb_future_get_int64(f->f, &bytes));
return bytes;
});
}
Future<FDBStandalone<VectorRef<KeyRef>>> TransactionImpl::getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) {
return backToFuture<FDBStandalone<VectorRef<KeyRef>>>(fdb_transaction_get_range_split_points(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size(), chunkSize), [](Reference<CFuture> f) {
FDBKey const* ks;
int count;
throw_on_error(fdb_future_get_key_array(f->f, &ks, &count));
return FDBStandalone<VectorRef<KeyRef>>(f, VectorRef<KeyRef>((KeyRef*)ks, count));
});
}
void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) {
throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ ) );
}
void TransactionImpl::addReadConflictKey(KeyRef const& key) {
return addReadConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) {
throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE ) );
}
void TransactionImpl::addWriteConflictKey(KeyRef const& key) {
return addWriteConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key))));
}
void TransactionImpl::atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) {
fdb_transaction_atomic_op( tr, key.begin(), key.size(), operand.begin(), operand.size(), operationType );
}
void TransactionImpl::set(const KeyRef& key, const ValueRef& value) {
fdb_transaction_set( tr, key.begin(), key.size(), value.begin(), value.size() );
}
void TransactionImpl::clear(const KeyRangeRef& range) {
fdb_transaction_clear_range( tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size() );
}
void TransactionImpl::clear(const KeyRef& key) {
fdb_transaction_clear( tr, key.begin(), key.size() );
}
Future<Void> TransactionImpl::commit() {
return backToFuture< Void >( fdb_transaction_commit( tr ), [](Reference<CFuture> f) {
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
} );
}
Version TransactionImpl::getCommittedVersion() {
Version v;
throw_on_error( fdb_transaction_get_committed_version( tr, &v ) );
return v;
}
Future<FDBStandalone<StringRef>> TransactionImpl::getVersionstamp() {
return backToFuture<FDBStandalone<KeyRef>>(fdb_transaction_get_versionstamp(tr), [](Reference<CFuture> f) {
uint8_t const* key;
int key_length;
throw_on_error( fdb_future_get_key( f->f, &key, &key_length ) );
return FDBStandalone<StringRef>( f, StringRef( key, key_length ) );
});
}
void TransactionImpl::setOption(FDBTransactionOption option, Optional<StringRef> value) {
if ( value.present() ) {
throw_on_error( fdb_transaction_set_option( tr, option, value.get().begin(), value.get().size() ) );
} else {
throw_on_error( fdb_transaction_set_option( tr, option, nullptr, 0 ) );
}
}
Future<int64_t> TransactionImpl::getApproximateSize() {
return backToFuture<int64_t>(fdb_transaction_get_approximate_size(tr), [](Reference<CFuture> f) {
int64_t size = 0;
throw_on_error(fdb_future_get_int64(f->f, &size));
return size;
});
}
Future<Void> TransactionImpl::onError(Error const& e) {
return backToFuture< Void >( fdb_transaction_on_error( tr, e.code() ), [](Reference<CFuture> f) {
throw_on_error( fdb_future_get_error( f->f ) );
return Void();
} );
}
void TransactionImpl::cancel() {
fdb_transaction_cancel( tr );
}
void TransactionImpl::reset() {
fdb_transaction_reset( tr );
}
} // namespace FDB
} // namespace FDB

214
bindings/flow/fdb_flow.h
View File

@ -30,127 +30,143 @@
#include "FDBLoanerTypes.h"
namespace FDB {
struct CFuture : NonCopyable, ReferenceCounted<CFuture>, FastAllocated<CFuture> {
CFuture() : f(nullptr) {}
explicit CFuture(FDBFuture* f) : f(f) {}
~CFuture() {
if (f) {
fdb_future_destroy(f);
}
struct CFuture : NonCopyable, ReferenceCounted<CFuture>, FastAllocated<CFuture> {
CFuture() : f(nullptr) {}
explicit CFuture(FDBFuture* f) : f(f) {}
~CFuture() {
if (f) {
fdb_future_destroy(f);
}
}
void blockUntilReady();
void blockUntilReady();
FDBFuture* f;
};
FDBFuture* f;
};
template <class T>
class FDBStandalone : public T {
public:
FDBStandalone() {}
FDBStandalone(Reference<CFuture> f, T const& t) : T(t), f(f) {}
FDBStandalone(FDBStandalone const& o) : T((T const&)o), f(o.f) {}
template <class T>
class FDBStandalone : public T {
public:
FDBStandalone() {}
FDBStandalone(Reference<CFuture> f, T const& t) : T(t), f(f) {}
FDBStandalone(FDBStandalone const& o) : T((T const&)o), f(o.f) {}
private:
Reference<CFuture> f;
};
private:
Reference<CFuture> f;
};
class ReadTransaction : public ReferenceCounted<ReadTransaction> {
public:
virtual ~ReadTransaction(){};
virtual void setReadVersion(Version v) = 0;
virtual Future<Version> getReadVersion() = 0;
class ReadTransaction : public ReferenceCounted<ReadTransaction> {
public:
virtual ~ReadTransaction(){};
virtual void setReadVersion(Version v) = 0;
virtual Future<Version> getReadVersion() = 0;
virtual Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) = 0;
virtual Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) = 0;
virtual Future<Void> watch(const Key& key) = 0;
virtual Future<Optional<FDBStandalone<ValueRef>>> get(const Key& key, bool snapshot = false) = 0;
virtual Future<FDBStandalone<KeyRef>> getKey(const KeySelector& key, bool snapshot = false) = 0;
virtual Future<Void> watch(const Key& key) = 0;
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeySelector& begin, const KeySelector& end, GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false, bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0;
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeySelector& begin, const KeySelector& end, int limit, bool snapshot = false, bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeyRange& keys, int limit, bool snapshot = false, bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limit, snapshot, reverse,
streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeyRange& keys, GetRangeLimits limits = GetRangeLimits(), bool snapshot = false, bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limits, snapshot, reverse,
streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(
const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0;
virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeySelector& begin,
const KeySelector& end,
int limit,
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeyRange& keys,
int limit,
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limit,
snapshot,
reverse,
streamingMode);
}
virtual Future<FDBStandalone<RangeResultRef>> getRange(const KeyRange& keys,
GetRangeLimits limits = GetRangeLimits(),
bool snapshot = false,
bool reverse = false,
FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) {
return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
limits,
snapshot,
reverse,
streamingMode);
}
virtual Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) = 0;
virtual Future<FDBStandalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) = 0;
virtual Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) = 0;
virtual Future<FDBStandalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) = 0;
virtual void addReadConflictRange(KeyRangeRef const& keys) = 0;
virtual void addReadConflictKey(KeyRef const& key) = 0;
virtual void addReadConflictRange(KeyRangeRef const& keys) = 0;
virtual void addReadConflictKey(KeyRef const& key) = 0;
virtual void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual void setOption(FDBTransactionOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual Future<Void> onError(Error const& e) = 0;
virtual Future<Void> onError(Error const& e) = 0;
virtual void cancel() = 0;
virtual void reset() = 0;
};
virtual void cancel() = 0;
virtual void reset() = 0;
};
class Transaction : public ReadTransaction {
public:
virtual void addWriteConflictRange(KeyRangeRef const& keys) = 0;
virtual void addWriteConflictKey(KeyRef const& key) = 0;
class Transaction : public ReadTransaction {
public:
virtual void addWriteConflictRange(KeyRangeRef const& keys) = 0;
virtual void addWriteConflictKey(KeyRef const& key) = 0;
virtual void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) = 0;
virtual void set(const KeyRef& key, const ValueRef& value) = 0;
virtual void clear(const KeyRangeRef& range) = 0;
virtual void clear(const KeyRef& key) = 0;
virtual void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) = 0;
virtual void set(const KeyRef& key, const ValueRef& value) = 0;
virtual void clear(const KeyRangeRef& range) = 0;
virtual void clear(const KeyRef& key) = 0;
virtual Future<Void> commit() = 0;
virtual Version getCommittedVersion() = 0;
virtual Future<int64_t> getApproximateSize() = 0;
virtual Future<FDBStandalone<StringRef>> getVersionstamp() = 0;
};
virtual Future<Void> commit() = 0;
virtual Version getCommittedVersion() = 0;
virtual Future<int64_t> getApproximateSize() = 0;
virtual Future<FDBStandalone<StringRef>> getVersionstamp() = 0;
};
class Database : public ReferenceCounted<Database> {
public:
virtual ~Database(){};
virtual Reference<Transaction> createTransaction() = 0;
virtual void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) = 0;
virtual Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0;
virtual Future<Void> createSnapshot(const StringRef& uid, const StringRef& snap_command) = 0;
};
class Database : public ReferenceCounted<Database> {
public:
virtual ~Database(){};
virtual Reference<Transaction> createTransaction() = 0;
virtual void setDatabaseOption(FDBDatabaseOption option, Optional<StringRef> value = Optional<StringRef>()) = 0;
virtual Future<int64_t> rebootWorker(const StringRef& address, bool check = false, int duration = 0) = 0;
virtual Future<Void> forceRecoveryWithDataLoss(const StringRef& dcid) = 0;
virtual Future<Void> createSnapshot(const StringRef& uid, const StringRef& snap_command) = 0;
};
class API {
public:
static API* selectAPIVersion(int apiVersion);
static API* getInstance();
static bool isAPIVersionSelected();
class API {
public:
static API* selectAPIVersion(int apiVersion);
static API* getInstance();
static bool isAPIVersionSelected();
void setNetworkOption(FDBNetworkOption option, Optional<StringRef> value = Optional<StringRef>());
void setNetworkOption(FDBNetworkOption option, Optional<StringRef> value = Optional<StringRef>());
void setupNetwork();
void runNetwork();
void stopNetwork();
void setupNetwork();
void runNetwork();
void stopNetwork();
Reference<Database> createDatabase(std::string const& connFilename = "");
Reference<Database> createDatabase(std::string const& connFilename = "");
bool evaluatePredicate(FDBErrorPredicate pred, Error const& e);
int getAPIVersion() const;
bool evaluatePredicate(FDBErrorPredicate pred, Error const& e);
int getAPIVersion() const;
private:
static API* instance;
private:
static API* instance;
API(int version);
int version;
};
} // namespace FDB
API(int version);
int version;
};
} // namespace FDB
#endif // FDB_FLOW_FDB_FLOW_H

201
bindings/flow/tester/DirectoryTester.actor.cpp
View File

@ -19,14 +19,14 @@
*/
#include "Tester.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
using namespace FDB;
ACTOR Future<std::vector<Tuple>> popTuples(Reference<FlowTesterData> data, int count = 1) {
state std::vector<Tuple> tuples;
while(tuples.size() < count) {
while (tuples.size() < count) {
Standalone<StringRef> sizeStr = wait(data->stack.pop()[0].value);
int size = Tuple::unpack(sizeStr).getInt(0);
@ -34,7 +34,7 @@ ACTOR Future<std::vector<Tuple>> popTuples(Reference<FlowTesterData> data, int c
state Tuple tuple;
state int index;
for(index = 0; index < tupleItems.size(); ++index) {
for (index = 0; index < tupleItems.size(); ++index) {
Standalone<StringRef> itemStr = wait(tupleItems[index].value);
tuple.append(Tuple::unpack(itemStr));
}
@ -54,9 +54,9 @@ ACTOR Future<std::vector<IDirectory::Path>> popPaths(Reference<FlowTesterData> d
std::vector<Tuple> tuples = wait(popTuples(data, count));
std::vector<IDirectory::Path> paths;
for(auto &tuple : tuples) {
for (auto& tuple : tuples) {
IDirectory::Path path;
for(int i = 0; i < tuple.size(); ++i) {
for (int i = 0; i < tuple.size(); ++i) {
path.push_back(tuple.getString(i));
}
@ -74,9 +74,9 @@ ACTOR Future<IDirectory::Path> popPath(Reference<FlowTesterData> data) {
std::string pathToString(IDirectory::Path const& path) {
std::string str;
str += "[";
for(int i = 0; i < path.size(); ++i) {
for (int i = 0; i < path.size(); ++i) {
str += path[i].toString();
if(i < path.size() - 1) {
if (i < path.size() - 1) {
str += ", ";
}
}
@ -86,21 +86,21 @@ std::string pathToString(IDirectory::Path const& path) {
IDirectory::Path combinePaths(IDirectory::Path const& path1, IDirectory::Path const& path2) {
IDirectory::Path outPath(path1.begin(), path1.end());
for(auto p : path2) {
for (auto p : path2) {
outPath.push_back(p);
}
return outPath;
}
void logOp(std::string message, bool force=false) {
if(LOG_OPS || force) {
void logOp(std::string message, bool force = false) {
if (LOG_OPS || force) {
printf("%s\n", message.c_str());
fflush(stdout);
}
}
//DIRECTORY_CREATE_SUBSPACE
// DIRECTORY_CREATE_SUBSPACE
struct DirectoryCreateSubspaceFunc : InstructionFunc {
static const char* name;
@ -108,7 +108,8 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc {
state Tuple path = wait(popTuple(data));
Tuple rawPrefix = wait(data->stack.waitAndPop());
logOp(format("Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str()));
logOp(format(
"Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str()));
data->directoryData.push(new Subspace(path, rawPrefix.getString(0)));
return Void();
}
@ -116,7 +117,7 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc {
const char* DirectoryCreateSubspaceFunc::name = "DIRECTORY_CREATE_SUBSPACE";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateSubspaceFunc);
//DIRECTORY_CREATE_LAYER
// DIRECTORY_CREATE_LAYER
struct DirectoryCreateLayerFunc : InstructionFunc {
static const char* name;
@ -127,15 +128,21 @@ struct DirectoryCreateLayerFunc : InstructionFunc {
int index2 = args[1].getInt(0);
bool allowManualPrefixes = args[2].getInt(0) != 0;
if(!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) {
if (!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) {
logOp("Create directory layer: None");
data->directoryData.push();
}
else {
} else {
Subspace* nodeSubspace = data->directoryData.directoryList[index1].subspace.get();
Subspace* contentSubspace = data->directoryData.directoryList[index2].subspace.get();
logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, allow_manual_prefixes = %d", index1, nodeSubspace->key().printable().c_str(), index2, nodeSubspace->key().printable().c_str(), allowManualPrefixes));
data->directoryData.push(Reference<IDirectory>(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes)));
logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, "
"allow_manual_prefixes = %d",
index1,
nodeSubspace->key().printable().c_str(),
index2,
nodeSubspace->key().printable().c_str(),
allowManualPrefixes));
data->directoryData.push(
Reference<IDirectory>(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes)));
}
return Void();
@ -144,7 +151,7 @@ struct DirectoryCreateLayerFunc : InstructionFunc {
const char* DirectoryCreateLayerFunc::name = "DIRECTORY_CREATE_LAYER";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateLayerFunc);
//DIRECTORY_CHANGE
// DIRECTORY_CHANGE
struct DirectoryChangeFunc : InstructionFunc {
static const char* name;
@ -153,13 +160,17 @@ struct DirectoryChangeFunc : InstructionFunc {
data->directoryData.directoryListIndex = index.getInt(0);
ASSERT(data->directoryData.directoryListIndex < data->directoryData.directoryList.size());
if(!data->directoryData.directoryList[data->directoryData.directoryListIndex].valid()) {
if (!data->directoryData.directoryList[data->directoryData.directoryListIndex].valid()) {
data->directoryData.directoryListIndex = data->directoryData.directoryErrorIndex;
}
if(LOG_DIRS) {
if (LOG_DIRS) {
DirectoryOrSubspace d = data->directoryData.directoryList[data->directoryData.directoryListIndex];
printf("Changed directory to %d (%s @\'%s\')\n", data->directoryData.directoryListIndex, d.typeString().c_str(), d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str() : d.subspace.get()->key().printable().c_str());
printf("Changed directory to %d (%s @\'%s\')\n",
data->directoryData.directoryListIndex,
d.typeString().c_str(),
d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str()
: d.subspace.get()->key().printable().c_str());
fflush(stdout);
}
@ -169,7 +180,7 @@ struct DirectoryChangeFunc : InstructionFunc {
const char* DirectoryChangeFunc::name = "DIRECTORY_CHANGE";
REGISTER_INSTRUCTION_FUNC(DirectoryChangeFunc);
//DIRECTORY_SET_ERROR_INDEX
// DIRECTORY_SET_ERROR_INDEX
struct DirectorySetErrorIndexFunc : InstructionFunc {
static const char* name;
@ -183,7 +194,7 @@ struct DirectorySetErrorIndexFunc : InstructionFunc {
const char* DirectorySetErrorIndexFunc::name = "DIRECTORY_SET_ERROR_INDEX";
REGISTER_INSTRUCTION_FUNC(DirectorySetErrorIndexFunc);
//DIRECTORY_CREATE_OR_OPEN
// DIRECTORY_CREATE_OR_OPEN
struct DirectoryCreateOrOpenFunc : InstructionFunc {
static const char* name;
@ -193,11 +204,12 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc {
Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("create_or_open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str()));
logOp(format("create_or_open %s: layer=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, layer] () {
return directory->createOrOpen(instruction->tr, path, layer);
}));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
instruction, [this, directory, layer]() { return directory->createOrOpen(instruction->tr, path, layer); }));
data->directoryData.push(dirSubspace);
@ -207,7 +219,7 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc {
const char* DirectoryCreateOrOpenFunc::name = "DIRECTORY_CREATE_OR_OPEN";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateOrOpenFunc);
//DIRECTORY_CREATE
// DIRECTORY_CREATE
struct DirectoryCreateFunc : InstructionFunc {
static const char* name;
@ -215,14 +227,19 @@ struct DirectoryCreateFunc : InstructionFunc {
state IDirectory::Path path = wait(popPath(data));
std::vector<Tuple> args = wait(data->stack.waitAndPop(2));
Standalone<StringRef> layer = args[0].getType(0) == Tuple::NULL_TYPE ? StringRef() : args[0].getString(0);
Optional<Standalone<StringRef>> prefix = args[1].getType(0) == Tuple::NULL_TYPE ? Optional<Standalone<StringRef>>() : args[1].getString(0);
Optional<Standalone<StringRef>> prefix =
args[1].getType(0) == Tuple::NULL_TYPE ? Optional<Standalone<StringRef>>() : args[1].getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("create %s: layer=%s, prefix=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str(), prefix.present() ? prefix.get().printable().c_str() : "<not present>"));
logOp(format("create %s: layer=%s, prefix=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str(),
prefix.present() ? prefix.get().printable().c_str() : "<not present>"));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, layer, prefix] () {
return directory->create(instruction->tr, path, layer, prefix);
}));
Reference<DirectorySubspace> dirSubspace =
wait(executeMutation(instruction, [this, directory, layer, prefix]() {
return directory->create(instruction->tr, path, layer, prefix);
}));
data->directoryData.push(dirSubspace);
@ -232,7 +249,7 @@ struct DirectoryCreateFunc : InstructionFunc {
const char* DirectoryCreateFunc::name = "DIRECTORY_CREATE";
REGISTER_INSTRUCTION_FUNC(DirectoryCreateFunc);
//DIRECTORY_OPEN
// DIRECTORY_OPEN
struct DirectoryOpenFunc : InstructionFunc {
static const char* name;
@ -242,7 +259,9 @@ struct DirectoryOpenFunc : InstructionFunc {
Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str()));
logOp(format("open %s: layer=%s",
pathToString(combinePaths(directory->getPath(), path)).c_str(),
layer.printable().c_str()));
Reference<DirectorySubspace> dirSubspace = wait(directory->open(instruction->tr, path, layer));
data->directoryData.push(dirSubspace);
@ -252,7 +271,7 @@ struct DirectoryOpenFunc : InstructionFunc {
const char* DirectoryOpenFunc::name = "DIRECTORY_OPEN";
REGISTER_INSTRUCTION_FUNC(DirectoryOpenFunc);
//DIRECTORY_MOVE
// DIRECTORY_MOVE
struct DirectoryMoveFunc : InstructionFunc {
static const char* name;
@ -260,11 +279,12 @@ struct DirectoryMoveFunc : InstructionFunc {
std::vector<IDirectory::Path> paths = wait(popPaths(data, 2));
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("move %s to %s", pathToString(combinePaths(directory->getPath(), paths[0])).c_str(), pathToString(combinePaths(directory->getPath(), paths[1])).c_str()));
logOp(format("move %s to %s",
pathToString(combinePaths(directory->getPath(), paths[0])).c_str(),
pathToString(combinePaths(directory->getPath(), paths[1])).c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, paths] () {
return directory->move(instruction->tr, paths[0], paths[1]);
}));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
instruction, [this, directory, paths]() { return directory->move(instruction->tr, paths[0], paths[1]); }));
data->directoryData.push(dirSubspace);
@ -274,7 +294,7 @@ struct DirectoryMoveFunc : InstructionFunc {
const char* DirectoryMoveFunc::name = "DIRECTORY_MOVE";
REGISTER_INSTRUCTION_FUNC(DirectoryMoveFunc);
//DIRECTORY_MOVE_TO
// DIRECTORY_MOVE_TO
struct DirectoryMoveToFunc : InstructionFunc {
static const char* name;
@ -284,9 +304,8 @@ struct DirectoryMoveToFunc : InstructionFunc {
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("move %s to %s", pathToString(directory->getPath()).c_str(), pathToString(path).c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, path] () {
return directory->moveTo(instruction->tr, path);
}));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(
instruction, [this, directory, path]() { return directory->moveTo(instruction->tr, path); }));
data->directoryData.push(dirSubspace);
@ -296,27 +315,22 @@ struct DirectoryMoveToFunc : InstructionFunc {
const char* DirectoryMoveToFunc::name = "DIRECTORY_MOVE_TO";
REGISTER_INSTRUCTION_FUNC(DirectoryMoveToFunc);
//DIRECTORY_REMOVE
// DIRECTORY_REMOVE
struct DirectoryRemoveFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory();
if(count.getInt(0) == 0) {
if (count.getInt(0) == 0) {
logOp(format("remove %s", pathToString(directory->getPath()).c_str()));
wait(executeMutation(instruction, [this] () {
return directory->remove(instruction->tr);
}));
}
else {
wait(executeMutation(instruction, [this]() { return directory->remove(instruction->tr); }));
} else {
IDirectory::Path path = wait(popPath(data));
logOp(format("remove %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
wait(executeMutation(instruction, [this, path] () {
return directory->remove(instruction->tr, path);
}));
wait(executeMutation(instruction, [this, path]() { return directory->remove(instruction->tr, path); }));
}
return Void();
@ -325,27 +339,24 @@ struct DirectoryRemoveFunc : InstructionFunc {
const char* DirectoryRemoveFunc::name = "DIRECTORY_REMOVE";
REGISTER_INSTRUCTION_FUNC(DirectoryRemoveFunc);
//DIRECTORY_REMOVE_IF_EXISTS
// DIRECTORY_REMOVE_IF_EXISTS
struct DirectoryRemoveIfExistsFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory();
if(count.getInt(0) == 0) {
if (count.getInt(0) == 0) {
logOp(format("remove_if_exists %s", pathToString(directory->getPath()).c_str()));
wait(success(executeMutation(instruction, [this] () {
return directory->removeIfExists(instruction->tr);
})));
}
else {
wait(
success(executeMutation(instruction, [this]() { return directory->removeIfExists(instruction->tr); })));
} else {
IDirectory::Path path = wait(popPath(data));
logOp(format("remove_if_exists %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
wait(success(executeMutation(instruction, [this, path] () {
return directory->removeIfExists(instruction->tr, path);
})));
wait(success(executeMutation(instruction,
[this, path]() { return directory->removeIfExists(instruction->tr, path); })));
}
return Void();
@ -354,7 +365,7 @@ struct DirectoryRemoveIfExistsFunc : InstructionFunc {
const char* DirectoryRemoveIfExistsFunc::name = "DIRECTORY_REMOVE_IF_EXISTS";
REGISTER_INSTRUCTION_FUNC(DirectoryRemoveIfExistsFunc);
//DIRECTORY_LIST
// DIRECTORY_LIST
struct DirectoryListFunc : InstructionFunc {
static const char* name;
@ -362,12 +373,11 @@ struct DirectoryListFunc : InstructionFunc {
Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory();
state Standalone<VectorRef<StringRef>> subdirs;
if(count.getInt(0) == 0) {
if (count.getInt(0) == 0) {
logOp(format("list %s", pathToString(directory->getPath()).c_str()));
Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr));
subdirs = _subdirs;
}
else {
} else {
IDirectory::Path path = wait(popPath(data));
logOp(format("list %s", pathToString(combinePaths(directory->getPath(), path)).c_str()));
Standalone<VectorRef<StringRef>> _subdirs = wait(directory->list(instruction->tr, path));
@ -375,7 +385,7 @@ struct DirectoryListFunc : InstructionFunc {
}
Tuple subdirTuple;
for(auto &sd : subdirs) {
for (auto& sd : subdirs) {
subdirTuple.append(sd, true);
}
@ -386,7 +396,7 @@ struct DirectoryListFunc : InstructionFunc {
const char* DirectoryListFunc::name = "DIRECTORY_LIST";
REGISTER_INSTRUCTION_FUNC(DirectoryListFunc);
//DIRECTORY_EXISTS
// DIRECTORY_EXISTS
struct DirectoryExistsFunc : InstructionFunc {
static const char* name;
@ -394,12 +404,11 @@ struct DirectoryExistsFunc : InstructionFunc {
Tuple count = wait(data->stack.waitAndPop());
state Reference<IDirectory> directory = data->directoryData.directory();
state bool result;
if(count.getInt(0) == 0) {
if (count.getInt(0) == 0) {
bool _result = wait(directory->exists(instruction->tr));
result = _result;
logOp(format("exists %s: %d", pathToString(directory->getPath()).c_str(), result));
}
else {
} else {
state IDirectory::Path path = wait(popPath(data));
bool _result = wait(directory->exists(instruction->tr, path));
result = _result;
@ -413,7 +422,7 @@ struct DirectoryExistsFunc : InstructionFunc {
const char* DirectoryExistsFunc::name = "DIRECTORY_EXISTS";
REGISTER_INSTRUCTION_FUNC(DirectoryExistsFunc);
//DIRECTORY_PACK_KEY
// DIRECTORY_PACK_KEY
struct DirectoryPackKeyFunc : InstructionFunc {
static const char* name;
@ -427,17 +436,19 @@ struct DirectoryPackKeyFunc : InstructionFunc {
const char* DirectoryPackKeyFunc::name = "DIRECTORY_PACK_KEY";
REGISTER_INSTRUCTION_FUNC(DirectoryPackKeyFunc);
//DIRECTORY_UNPACK_KEY
// DIRECTORY_UNPACK_KEY
struct DirectoryUnpackKeyFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple key = wait(data->stack.waitAndPop());
Subspace *subspace = data->directoryData.subspace();
logOp(format("Unpack %s in subspace with prefix %s", key.getString(0).printable().c_str(), subspace->key().printable().c_str()));
Subspace* subspace = data->directoryData.subspace();
logOp(format("Unpack %s in subspace with prefix %s",
key.getString(0).printable().c_str(),
subspace->key().printable().c_str()));
Tuple tuple = subspace->unpack(key.getString(0));
for(int i = 0; i < tuple.size(); ++i) {
data->stack.push(tuple.subTuple(i, i+1).pack());
for (int i = 0; i < tuple.size(); ++i) {
data->stack.push(tuple.subTuple(i, i + 1).pack());
}
return Void();
@ -446,7 +457,7 @@ struct DirectoryUnpackKeyFunc : InstructionFunc {
const char* DirectoryUnpackKeyFunc::name = "DIRECTORY_UNPACK_KEY";
REGISTER_INSTRUCTION_FUNC(DirectoryUnpackKeyFunc);
//DIRECTORY_RANGE
// DIRECTORY_RANGE
struct DirectoryRangeFunc : InstructionFunc {
static const char* name;
@ -462,7 +473,7 @@ struct DirectoryRangeFunc : InstructionFunc {
const char* DirectoryRangeFunc::name = "DIRECTORY_RANGE";
REGISTER_INSTRUCTION_FUNC(DirectoryRangeFunc);
//DIRECTORY_CONTAINS
// DIRECTORY_CONTAINS
struct DirectoryContainsFunc : InstructionFunc {
static const char* name;
@ -477,15 +488,15 @@ struct DirectoryContainsFunc : InstructionFunc {
const char* DirectoryContainsFunc::name = "DIRECTORY_CONTAINS";
REGISTER_INSTRUCTION_FUNC(DirectoryContainsFunc);
//DIRECTORY_OPEN_SUBSPACE
// DIRECTORY_OPEN_SUBSPACE
struct DirectoryOpenSubspaceFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple tuple = wait(popTuple(data));
Subspace *subspace = data->directoryData.subspace();
Subspace* subspace = data->directoryData.subspace();
logOp(format("open_subspace %s (at %s)", tupleToString(tuple).c_str(), subspace->key().printable().c_str()));
Subspace *child = new Subspace(subspace->subspace(tuple));
Subspace* child = new Subspace(subspace->subspace(tuple));
data->directoryData.push(child);
return Void();
@ -494,7 +505,7 @@ struct DirectoryOpenSubspaceFunc : InstructionFunc {
const char* DirectoryOpenSubspaceFunc::name = "DIRECTORY_OPEN_SUBSPACE";
REGISTER_INSTRUCTION_FUNC(DirectoryOpenSubspaceFunc);
//DIRECTORY_LOG_SUBSPACE
// DIRECTORY_LOG_SUBSPACE
struct DirectoryLogSubspaceFunc : InstructionFunc {
static const char* name;
@ -510,7 +521,7 @@ struct DirectoryLogSubspaceFunc : InstructionFunc {
const char* DirectoryLogSubspaceFunc::name = "DIRECTORY_LOG_SUBSPACE";
REGISTER_INSTRUCTION_FUNC(DirectoryLogSubspaceFunc);
//DIRECTORY_LOG_DIRECTORY
// DIRECTORY_LOG_DIRECTORY
struct DirectoryLogDirectoryFunc : InstructionFunc {
static const char* name;
@ -520,22 +531,23 @@ struct DirectoryLogDirectoryFunc : InstructionFunc {
state bool exists = wait(directory->exists(instruction->tr));
state Tuple childrenTuple;
if(exists) {
if (exists) {
Standalone<VectorRef<StringRef>> children = wait(directory->list(instruction->tr));
for(auto &c : children) {
for (auto& c : children) {
childrenTuple.append(c, true);
}
}
}
Subspace logSubspace(Tuple().append(data->directoryData.directoryListIndex), prefix.getString(0));
Tuple pathTuple;
for(auto &p : directory->getPath()) {
for (auto& p : directory->getPath()) {
pathTuple.append(p, true);
}
instruction->tr->set(logSubspace.pack(LiteralStringRef("path"), true), pathTuple.pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true), Tuple().append(directory->getLayer()).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true),
Tuple().append(directory->getLayer()).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append(exists ? 1 : 0).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("children"), true), childrenTuple.pack());
@ -545,13 +557,13 @@ struct DirectoryLogDirectoryFunc : InstructionFunc {
const char* DirectoryLogDirectoryFunc::name = "DIRECTORY_LOG_DIRECTORY";
REGISTER_INSTRUCTION_FUNC(DirectoryLogDirectoryFunc);
//DIRECTORY_STRIP_PREFIX
// DIRECTORY_STRIP_PREFIX
struct DirectoryStripPrefixFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple str = wait(data->stack.waitAndPop());
Subspace *subspace = data->directoryData.subspace();
Subspace* subspace = data->directoryData.subspace();
ASSERT(str.getString(0).startsWith(subspace->key()));
data->stack.pushTuple(str.getString(0).substr(subspace->key().size()));
return Void();
@ -559,4 +571,3 @@ struct DirectoryStripPrefixFunc : InstructionFunc {
};
const char* DirectoryStripPrefixFunc::name = "DIRECTORY_STRIP_PREFIX";
REGISTER_INSTRUCTION_FUNC(DirectoryStripPrefixFunc);

482
bindings/flow/tester/Tester.actor.cpp
View File

File diff suppressed because it is too large Load Diff

84
bindings/flow/tester/Tester.actor.h
View File

@ -18,12 +18,13 @@
* limitations under the License.
*/
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version.
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDB_FLOW_TESTER_TESTER_ACTOR_G_H)
#define FDB_FLOW_TESTER_TESTER_ACTOR_G_H
#include "Tester.actor.g.h"
#define FDB_FLOW_TESTER_TESTER_ACTOR_G_H
#include "Tester.actor.g.h"
#elif !defined(FDB_FLOW_TESTER_TESTER_ACTOR_H)
#define FDB_FLOW_TESTER_TESTER_ACTOR_H
#define FDB_FLOW_TESTER_TESTER_ACTOR_H
#pragma once
@ -34,7 +35,7 @@
#include "bindings/flow/IDirectory.h"
#include "bindings/flow/Subspace.h"
#include "bindings/flow/DirectoryLayer.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
constexpr bool LOG_ALL = false;
constexpr bool LOG_INSTRUCTIONS = LOG_ALL || false;
@ -56,19 +57,13 @@ struct FlowTesterStack {
uint32_t index;
std::vector<StackItem> data;
void push(Future<Standalone<StringRef>> value) {
data.push_back(StackItem(index, value));
}
void push(Future<Standalone<StringRef>> value) { data.push_back(StackItem(index, value)); }
void push(Standalone<StringRef> value) {
push(Future<Standalone<StringRef>>(value));
}
void push(Standalone<StringRef> value) { push(Future<Standalone<StringRef>>(value)); }
void push(const StackItem& item) {
data.push_back(item);
}
void push(const StackItem& item) { data.push_back(item); }
void pushTuple(StringRef value, bool utf8=false) {
void pushTuple(StringRef value, bool utf8 = false) {
FDB::Tuple t;
t.append(value, utf8);
data.push_back(StackItem(index, t.pack()));
@ -101,9 +96,7 @@ struct FlowTesterStack {
data.push_back(data.back());
}
void clear() {
data.clear();
}
void clear() { data.clear(); }
};
struct InstructionData : public ReferenceCounted<InstructionData> {
@ -113,21 +106,21 @@ struct InstructionData : public ReferenceCounted<InstructionData> {
Reference<FDB::Transaction> tr;
InstructionData(bool _isDatabase, bool _isSnapshot, StringRef _instruction, Reference<FDB::Transaction> _tr)
: isDatabase(_isDatabase)
, isSnapshot(_isSnapshot)
, instruction(_instruction)
, tr(_tr) {}
: isDatabase(_isDatabase), isSnapshot(_isSnapshot), instruction(_instruction), tr(_tr) {}
};
struct FlowTesterData;
struct InstructionFunc : IDispatched<InstructionFunc, std::string, std::function<Future<Void>(Reference<FlowTesterData> data, Reference<InstructionData> instruction)>> {
struct InstructionFunc
: IDispatched<InstructionFunc,
std::string,
std::function<Future<Void>(Reference<FlowTesterData> data, Reference<InstructionData> instruction)>> {
static Future<Void> call(std::string op, Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
ASSERT(data);
ASSERT(instruction);
auto it = dispatches().find(op);
if(it == dispatches().end()) {
if (it == dispatches().end()) {
fprintf(stderr, "Unrecognized instruction: %s\n", op.c_str());
ASSERT(false);
}
@ -143,24 +136,20 @@ struct DirectoryOrSubspace {
DirectoryOrSubspace() {}
DirectoryOrSubspace(Reference<FDB::IDirectory> directory) : directory(directory) {}
DirectoryOrSubspace(FDB::Subspace *subspace) : subspace(subspace) {}
DirectoryOrSubspace(Reference<FDB::DirectorySubspace> dirSubspace) : directory(dirSubspace), subspace(dirSubspace.getPtr()) {}
DirectoryOrSubspace(FDB::Subspace* subspace) : subspace(subspace) {}
DirectoryOrSubspace(Reference<FDB::DirectorySubspace> dirSubspace)
: directory(dirSubspace), subspace(dirSubspace.getPtr()) {}
bool valid() {
return directory.present() || subspace.present();
}
bool valid() { return directory.present() || subspace.present(); }
std::string typeString() {
if(directory.present() && subspace.present()) {
if (directory.present() && subspace.present()) {
return "DirectorySubspace";
}
else if(directory.present()) {
} else if (directory.present()) {
return "IDirectory";
}
else if(subspace.present()) {
} else if (subspace.present()) {
return "Subspace";
}
else {
} else {
return "InvalidDirectory";
}
}
@ -190,8 +179,8 @@ struct DirectoryTesterData {
template <class T>
void push(T item) {
directoryList.push_back(DirectoryOrSubspace(item));
if(LOG_DIRS) {
printf("Pushed %s at %lu\n", directoryList.back().typeString().c_str(), directoryList.size()-1);
if (LOG_DIRS) {
printf("Pushed %s at %lu\n", directoryList.back().typeString().c_str(), directoryList.size() - 1);
fflush(stdout);
}
}
@ -200,7 +189,7 @@ struct DirectoryTesterData {
};
struct FlowTesterData : public ReferenceCounted<FlowTesterData> {
FDB::API *api;
FDB::API* api;
Reference<FDB::Database> db;
Standalone<FDB::RangeResultRef> instructions;
Standalone<StringRef> trName;
@ -211,12 +200,11 @@ struct FlowTesterData : public ReferenceCounted<FlowTesterData> {
std::vector<Future<Void>> subThreads;
Future<Void> processInstruction(Reference<InstructionData> instruction) {
return InstructionFunc::call(instruction->instruction.toString(), Reference<FlowTesterData>::addRef(this), instruction);
return InstructionFunc::call(
instruction->instruction.toString(), Reference<FlowTesterData>::addRef(this), instruction);
}
FlowTesterData(FDB::API *api) {
this->api = api;
}
FlowTesterData(FDB::API* api) { this->api = api; }
};
std::string tupleToString(FDB::Tuple const& tuple);
@ -226,16 +214,14 @@ Future<decltype(std::declval<F>()().getValue())> executeMutation(Reference<Instr
loop {
try {
state decltype(std::declval<F>()().getValue()) result = wait(func());
if(instruction->isDatabase) {
if (instruction->isDatabase) {
wait(instruction->tr->commit());
}
return result;
}
catch(Error &e) {
if(instruction->isDatabase) {
} catch (Error& e) {
if (instruction->isDatabase) {
wait(instruction->tr->onError(e));
}
else {
} else {
throw;
}
}

6
bindings/java/CMakeLists.txt
View File

@ -59,6 +59,8 @@ set(JAVA_BINDING_SRCS
src/main/com/apple/foundationdb/subspace/Subspace.java
src/main/com/apple/foundationdb/Transaction.java
src/main/com/apple/foundationdb/TransactionContext.java
src/main/com/apple/foundationdb/EventKeeper.java
src/main/com/apple/foundationdb/MapEventKeeper.java
src/main/com/apple/foundationdb/testing/AbstractWorkload.java
src/main/com/apple/foundationdb/testing/WorkloadContext.java
src/main/com/apple/foundationdb/testing/Promise.java
@ -257,6 +259,10 @@ if(NOT OPEN_FOR_IDE)
set(TEST_CP ${tests_jar} ${target_jar})
if(RUN_JUNIT_TESTS OR RUN_JAVA_INTEGRATION_TESTS)
if (USE_SANITIZER)
message(WARNING "Cannot run java tests with sanitizer builds")
return()
endif()
# We use Junit libraries for both JUnit and integration testing structures, so download in either case
# https://search.maven.org/remotecontent?filepath=org/junit/jupiter/junit-jupiter-engine/5.7.1/junit-jupiter-engine-5.7.1.jar
file(DOWNLOAD "https://search.maven.org/remotecontent?filepath=org/junit/jupiter/junit-jupiter-engine/5.7.1/junit-jupiter-engine-5.7.1.jar"

36
bindings/java/JavaWorkload.cpp
View File

@ -280,7 +280,8 @@ struct JVM {
w.name = name;
w.signature = sig;
w.fnPtr = std::get<2>(t);
log->trace(info, "PreparedNativeMethod",
log->trace(info,
"PreparedNativeMethod",
{ { "Name", w.name },
{ "Signature", w.signature },
{ "Ptr", std::to_string(reinterpret_cast<uintptr_t>(w.fnPtr)) } });
@ -362,7 +363,8 @@ struct JVM {
{ "getOption", "(JLjava/lang/String;Z)Z", reinterpret_cast<void*>(&getOptionBool) },
{ "getOption", "(JLjava/lang/String;J)J", reinterpret_cast<void*>(&getOptionLong) },
{ "getOption", "(JLjava/lang/String;D)D", reinterpret_cast<void*>(&getOptionDouble) },
{ "getOption", "(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;",
{ "getOption",
"(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;",
reinterpret_cast<void*>(&getOptionString) },
{ "getClientID", "(J)I", reinterpret_cast<void*>(&getClientID) },
{ "getClientCount", "(J)I", reinterpret_cast<void*>(&getClientCount) },
@ -391,7 +393,8 @@ struct JVM {
auto impl = env->GetLongField(res, field);
checkException();
if (impl != jContext) {
log->trace(error, "ContextNotCorrect",
log->trace(error,
"ContextNotCorrect",
{ { "Expected", std::to_string(jContext) }, { "Impl", std::to_string(impl) } });
std::terminate();
}
@ -471,14 +474,16 @@ struct JVM {
}
jobject createDatabase(jobject workload, FDBDatabase* db) {
auto executor =
env->CallObjectMethod(workload, getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"),
"getExecutor", "()Ljava/util/concurrent/Executor;"));
auto executor = env->CallObjectMethod(workload,
getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"),
"getExecutor",
"()Ljava/util/concurrent/Executor;"));
auto databaseClass = getClass("com/apple/foundationdb/FDBDatabase");
jlong databasePtr = reinterpret_cast<jlong>(db);
jobject javaDatabase =
env->NewObject(databaseClass, getMethod(databaseClass, "<init>", "(JLjava/util/concurrent/Executor;)V"),
databasePtr, executor);
jobject javaDatabase = env->NewObject(databaseClass,
getMethod(databaseClass, "<init>", "(JLjava/util/concurrent/Executor;)V"),
databasePtr,
executor);
env->DeleteLocalRef(executor);
return javaDatabase;
}
@ -491,9 +496,10 @@ struct JVM {
jPromise = createPromise(std::move(promise));
env->CallVoidMethod(
workload,
getMethod(clazz, method,
"(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"),
jdb, jPromise);
getMethod(
clazz, method, "(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"),
jdb,
jPromise);
env->DeleteLocalRef(jdb);
env->DeleteLocalRef(jPromise);
jPromise = nullptr;
@ -515,7 +521,7 @@ struct JavaWorkload : FDBWorkload {
bool failed = false;
jobject workload = nullptr;
JavaWorkload(const std::shared_ptr<JVM>& jvm, FDBLogger& log, const std::string& name)
: jvm(jvm), log(log), name(name) {
: jvm(jvm), log(log), name(name) {
boost::replace_all(this->name, ".", "/");
}
~JavaWorkload() {
@ -588,9 +594,7 @@ struct JavaWorkload : FDBWorkload {
log.trace(error, "CheckFailedWithJNIError", { { "Error", e.toString() }, { "Location", e.location() } });
}
}
void getMetrics(std::vector<FDBPerfMetric>& out) const override {
jvm->getMetrics(workload, name, out);
}
void getMetrics(std::vector<FDBPerfMetric>& out) const override { jvm->getMetrics(workload, name, out); }
};
struct JavaWorkloadFactory : FDBWorkloadFactory {

1037
bindings/java/fdbJNI.cpp
View File

File diff suppressed because it is too large Load Diff

44
bindings/java/src/integration/com/apple/foundationdb/RangeQueryIntegrationTest.java
View File

@ -21,6 +21,8 @@ package com.apple.foundationdb;
import java.util.Iterator;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Random;
import java.util.TreeMap;
import com.apple.foundationdb.async.AsyncIterable;
@ -55,6 +57,48 @@ class RangeQueryIntegrationTest {
}
}
private void loadData(Database db, Map<byte[], byte[]> dataToLoad) {
db.run(tr -> {
for (Map.Entry<byte[], byte[]> entry : dataToLoad.entrySet()) {
tr.set(entry.getKey(), entry.getValue());
}
return null;
});
}
@Test
public void canGetRowWithKeySelector() throws Exception {
Random rand = new Random();
byte[] key = new byte[128];
byte[] value = new byte[128];
rand.nextBytes(key);
key[0] = (byte)0xEE;
rand.nextBytes(value);
NavigableMap<byte[], byte[]> data = new TreeMap<>(ByteArrayUtil.comparator());
data.put(key, value);
try (Database db = fdb.open()) {
loadData(db, data);
db.run(tr -> {
byte[] actualValue = tr.get(key).join();
Assertions.assertNotNull(actualValue, "Missing key!");
Assertions.assertArrayEquals(value, actualValue, "incorrect value!");
KeySelector start = KeySelector.firstGreaterOrEqual(new byte[] { key[0] });
KeySelector end = KeySelector.firstGreaterOrEqual(ByteArrayUtil.strinc(start.getKey()));
AsyncIterable<KeyValue> kvIterable = tr.getRange(start, end);
AsyncIterator<KeyValue> kvs = kvIterable.iterator();
Assertions.assertTrue(kvs.hasNext(), "Did not return a record!");
KeyValue n = kvs.next();
Assertions.assertArrayEquals(key, n.getKey(), "Did not return a key correctly!");
Assertions.assertArrayEquals(value, n.getValue(), "Did not return the corect value!");
return null;
});
}
}
@Test
void rangeQueryReturnsResults() throws Exception {
/*

123
bindings/java/src/junit/com/apple/foundationdb/EventKeeperTest.java Normal file
View File

@ -0,0 +1,123 @@
/*
* EventKeeperTest.java
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.apple.foundationdb;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.apple.foundationdb.EventKeeper.Events;
import com.apple.foundationdb.async.AsyncIterator;
import com.apple.foundationdb.tuple.ByteArrayUtil;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/**
* Basic test code for testing basic Transaction Timer logic.
*
* These tests don't check for a whole lot, they just verify that
* instrumentation works as expected for specific patterns.
*/
class EventKeeperTest {
@Test
void testSetVersion() throws Exception {
EventKeeper timer = new MapEventKeeper();
try (FDBTransaction txn = new FDBTransaction(1, null, null, timer)) {
Assertions.assertThrows(UnsatisfiedLinkError.class,
() -> { txn.setReadVersion(1L); }, "Test should call a bad native method");
long jniCalls = timer.getCount(Events.JNI_CALL);
Assertions.assertEquals(1L, jniCalls, "Unexpected number of JNI calls:");
}catch(UnsatisfiedLinkError ignored){
//this is necessary to prevent an exception being thrown at close time
}
}
@Test
void testGetReadVersion() throws Exception {
EventKeeper timer = new MapEventKeeper();
try (FDBTransaction txn = new FDBTransaction(1, null, null, timer)) {
Assertions.assertThrows(UnsatisfiedLinkError.class,
() -> { txn.getReadVersion(); }, "Test should call a bad native method");
long jniCalls = timer.getCount(Events.JNI_CALL);
Assertions.assertEquals(1L, jniCalls, "Unexpected number of JNI calls:");
}catch(UnsatisfiedLinkError ignored){
//required to prevent an extra exception being thrown at close time
}
}
@Test
void testGetRangeRecordsFetches() throws Exception {
EventKeeper timer = new MapEventKeeper();
List<KeyValue> testKvs = Arrays.asList(new KeyValue("hello".getBytes(), "goodbye".getBytes()));
FDBTransaction txn = new FakeFDBTransaction(testKvs, 1L, null, null);
RangeQuery query = new RangeQuery(txn, true, KeySelector.firstGreaterOrEqual(new byte[] { 0x00 }),
KeySelector.firstGreaterOrEqual(new byte[] { (byte)0xFF }), -1, false,
StreamingMode.ITERATOR, timer);
AsyncIterator<KeyValue> iter = query.iterator();
List<KeyValue> iteratedItems = new ArrayList<>();
while (iter.hasNext()) {
iteratedItems.add(iter.next());
}
// basic verification that we got back what we expected to get back.
Assertions.assertEquals(testKvs.size(), iteratedItems.size(), "Incorrect iterated list, size incorrect.");
int expectedByteSize = 0;
for (KeyValue expected : testKvs) {
byte[] eKey = expected.getKey();
byte[] eVal = expected.getValue();
expectedByteSize += eKey.length + 4;
expectedByteSize += eVal.length + 4;
boolean found = false;
for (KeyValue actual : iteratedItems) {
byte[] aKey = actual.getKey();
byte[] aVal = actual.getValue();
if (ByteArrayUtil.compareTo(eKey, 0, eKey.length, aKey, 0, aKey.length) == 0) {
int cmp = ByteArrayUtil.compareTo(eVal, 0, eVal.length, aVal, 0, aVal.length);
Assertions.assertEquals(0, cmp, "Incorrect value returned");
found = true;
break;
}
}
Assertions.assertTrue(found, "missing key!");
}
// now check the timer and see if it recorded any events
Assertions.assertEquals(1, timer.getCount(Events.RANGE_QUERY_FETCHES), "Unexpected number of chunk fetches");
Assertions.assertEquals(testKvs.size(), timer.getCount(Events.RANGE_QUERY_RECORDS_FETCHED),
"Unexpected number of tuples fetched");
Assertions.assertEquals(expectedByteSize, timer.getCount(Events.BYTES_FETCHED),
"Incorrect number of bytes fetched");
}
}

11
bindings/java/src/junit/com/apple/foundationdb/FakeFDBTransaction.java
View File

@ -67,6 +67,15 @@ public class FakeFDBTransaction extends FDBTransaction {
}
}
public FakeFDBTransaction(List<KeyValue> backingData, long cPtr, Database db,
Executor executor) {
this(cPtr, db, executor);
for (KeyValue entry : backingData) {
this.backingData.put(entry.getKey(), entry.getValue());
}
}
@Override
public CompletableFuture<byte[]> get(byte[] key) {
return CompletableFuture.completedFuture(this.backingData.get(key));
@ -92,7 +101,7 @@ public class FakeFDBTransaction extends FDBTransaction {
// holder variable so that we can pass the range to the results function safely
final NavigableMap<byte[], byte[]> retMap = range;
FutureResults fr = new FutureResults(-1L, false, executor) {
FutureResults fr = new FutureResults(-1L, false, executor, null) {
@Override
protected void registerMarshalCallback(Executor executor) {
// no-op

11
bindings/java/src/main/com/apple/foundationdb/Database.java
View File

@ -62,6 +62,17 @@ public interface Database extends AutoCloseable, TransactionContext {
*/
Transaction createTransaction(Executor e);
/**
* Creates a {@link Transaction} that operates on this {@code Database} with the given {@link Executor}
* for asynchronous callbacks.
*
* @param e the {@link Executor} to use when executing asynchronous callbacks for the database
* @param eventKeeper the {@link EventKeeper} to use when tracking instrumented calls for the transaction.
*
* @return a newly created {@code Transaction} that reads from and writes to this {@code Database}.
*/
Transaction createTransaction(Executor e, EventKeeper eventKeeper);
/**
* Returns a set of options that can be set on a {@code Database}
*

172
bindings/java/src/main/com/apple/foundationdb/EventKeeper.java Normal file
View File

@ -0,0 +1,172 @@
/*
* EventKeeper.java
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.apple.foundationdb;
import java.util.concurrent.TimeUnit;
/**
* A device for externally instrumenting the FDB java driver, for monitoring
* purposes.
*
* Note that implementations as expected to be thread-safe, and may be manipulated
* from multiple threads even in nicely single-threaded-looking applications.
*/
public interface EventKeeper {
/**
* Count the number of events which occurred.
*
* @param event the event which occurred
* @param amt the number of times that even occurred
*/
void count(Event event, long amt);
/**
* Convenience method to add 1 to the number of events which occurred.
*
* @param event the event which occurred.
*/
default void increment(Event event) { count(event, 1L); }
/**
* Count the time taken to perform an event, in nanoseconds.
*
* Note that {@code event.isTimeEvent()} should return true here.
*
* @param event the event which was timed (the event should be a time event).
* @param nanos the amount of time taken (in nanoseconds)
*/
void timeNanos(Event event, long nanos);
/**
* Count the time taken to perform an action, in the specified units.
*
* Note that {@code event.isTimeEvent()} should return true.
*
* @param event the event which was timed.
* @param duration the time taken
* @param theUnit the unit of time in which the time measurement was taken
*/
default void time(Event event, long duration, TimeUnit theUnit) { timeNanos(event, theUnit.toNanos(duration)); }
/**
* Get the number of events which occurred since this timer was created.
*
* If the event was never recorded, then this returns 0.
*
* @param event the event to get the count for
* @return the number of times the event was triggered. If the event has never been triggered,
* then this returns 0
*/
long getCount(Event event);
/**
* Get the amount of time taken by this event, in nanoseconds.
*
* @param event the event to get the time for
* @return the total time measured for this event, in nanoseconds. If the event was never recorded,
* return 0 instead.
*/
long getTimeNanos(Event event);
/**
* Get the amount of time taken by this event, in the specified units.
*
* Important note: If the time that was measured in nanoseconds does not evenly divide the unit that
* is specified (which is likely, considering time), then some precision may be lost in the conversion. Use
* this carefully.
*
* @param event the event to get the time for
* @param theUnit the unit to get time in
* @return the total time measured for this event, in the specified unit. If the event was never recorded,
* return 0.
*/
default long getTime(Event event, TimeUnit theUnit) {
return theUnit.convert(getTimeNanos(event), TimeUnit.NANOSECONDS);
}
/**
* Marker interface for tracking the specific type of event that occurs, and metadata about said event.
*
* Implementations should be sure to provide a quality {@code equals} and {@code hashCode}.
*/
interface Event {
/**
* @return the name of this event, as a unique string. This name should generally be unique, because
* it's likely that {@code EventKeeper} implementations will rely on this for uniqueness.
*/
String name();
/**
* @return true if this event represents a timed event, rather than a counter event.
*/
default boolean isTimeEvent() { return false; };
}
/**
* An enumeration of static events which occur within the FDB Java driver.
*/
enum Events implements Event {
/**
* The number of JNI calls that were exercised.
*/
JNI_CALL,
/**
* The total number of bytes pulled from the native layer, including length delimiters., from
* {@link Transaction#get(byte[])}, {@link Transaction#getKey(KeySelector)},
* {@link Transaction#getRange(KeySelector, KeySelector)} (and related method
* overrides), or any other read-type operation which occurs on a Transaction
*/
BYTES_FETCHED,
/**
* The number of times a DirectBuffer was used to transfer a range query chunk
* across the JNI boundary
*/
RANGE_QUERY_DIRECT_BUFFER_HIT,
/**
* The number of times a range query chunk was unable to use a DirectBuffer to
* transfer data across the JNI boundary
*/
RANGE_QUERY_DIRECT_BUFFER_MISS,
/**
* The number of direct fetches made during a range query
*/
RANGE_QUERY_FETCHES,
/**
* The number of tuples fetched during a range query
*/
RANGE_QUERY_RECORDS_FETCHED,
/**
* The number of times a range query chunk fetch failed
*/
RANGE_QUERY_CHUNK_FAILED,
/**
* The time taken to perform an internal `getRange` fetch, in nanoseconds
*/
RANGE_QUERY_FETCH_TIME_NANOS {
@Override
public boolean isTimeEvent() {
return true;
}
};
}
}

48
bindings/java/src/main/com/apple/foundationdb/FDB.java
View File

@ -364,6 +364,28 @@ public class FDB {
return open(clusterFilePath, DEFAULT_EXECUTOR);
}
/**
* Initializes networking if required and connects to the cluster specified by {@code clusterFilePath}.<br>
* <br>
* A single client can use this function multiple times to connect to different
* clusters simultaneously, with each invocation requiring its own cluster file.
* To connect to multiple clusters running at different, incompatible versions,
* the <a href="/foundationdb/api-general.html#multi-version-client-api" target="_blank">multi-version client API</a>
* must be used.
*
* @param clusterFilePath the
* <a href="/foundationdb/administration.html#foundationdb-cluster-file" target="_blank">cluster file</a>
* defining the FoundationDB cluster. This can be {@code null} if the
* <a href="/foundationdb/administration.html#default-cluster-file" target="_blank">default fdb.cluster file</a>
* is to be used.
* @param eventKeeper the EventKeeper to use for instrumentation calls, or {@code null} if no instrumentation is desired.
*
* @return a {@code CompletableFuture} that will be set to a FoundationDB {@link Database}
*/
public Database open(String clusterFilePath, EventKeeper eventKeeper) throws FDBException {
return open(clusterFilePath, DEFAULT_EXECUTOR, eventKeeper);
}
/**
* Initializes networking if required and connects to the cluster specified by {@code clusterFilePath}.<br>
* <br>
@ -383,13 +405,37 @@ public class FDB {
* @return a {@code CompletableFuture} that will be set to a FoundationDB {@link Database}
*/
public Database open(String clusterFilePath, Executor e) throws FDBException {
return open(clusterFilePath, e, null);
}
/**
* Initializes networking if required and connects to the cluster specified by {@code clusterFilePath}.<br>
* <br>
* A single client can use this function multiple times to connect to different
* clusters simultaneously, with each invocation requiring its own cluster file.
* To connect to multiple clusters running at different, incompatible versions,
* the <a href="/foundationdb/api-general.html#multi-version-client-api" target="_blank">multi-version client API</a>
* must be used.
*
* @param clusterFilePath the
* <a href="/foundationdb/administration.html#foundationdb-cluster-file" target="_blank">cluster file</a>
* defining the FoundationDB cluster. This can be {@code null} if the
* <a href="/foundationdb/administration.html#default-cluster-file" target="_blank">default fdb.cluster file</a>
* is to be used.
* @param e the {@link Executor} to use to execute asynchronous callbacks
* @param eventKeeper the {@link EventKeeper} to use to record instrumentation metrics, or {@code null} if no
* instrumentation is desired.
*
* @return a {@code CompletableFuture} that will be set to a FoundationDB {@link Database}
*/
public Database open(String clusterFilePath, Executor e, EventKeeper eventKeeper) throws FDBException {
synchronized(this) {
if(!isConnected()) {
startNetwork();
}
}
return new FDBDatabase(Database_create(clusterFilePath), e);
return new FDBDatabase(Database_create(clusterFilePath), e, eventKeeper);
}
/**

17
bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java
View File

@ -31,11 +31,17 @@ import com.apple.foundationdb.async.AsyncUtil;
class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsumer {
private DatabaseOptions options;
private final Executor executor;
private final EventKeeper eventKeeper;
protected FDBDatabase(long cPtr, Executor executor) {
this(cPtr, executor, null);
}
protected FDBDatabase(long cPtr, Executor executor, EventKeeper eventKeeper) {
super(cPtr);
this.executor = executor;
this.options = new DatabaseOptions(this);
this.eventKeeper = eventKeeper;
}
@Override
@ -112,14 +118,19 @@ class FDBDatabase extends NativeObjectWrapper implements Database, OptionConsume
@Override
public Transaction createTransaction(Executor e) {
return createTransaction(e, eventKeeper);
}
@Override
public Transaction createTransaction(Executor e, EventKeeper eventKeeper) {
pointerReadLock.lock();
Transaction tr = null;
try {
tr = new FDBTransaction(Database_createTransaction(getPtr()), this, e);
tr = new FDBTransaction(Database_createTransaction(getPtr()), this, e, eventKeeper);
tr.options().setUsedDuringCommitProtectionDisable();
return tr;
} catch(RuntimeException err) {
if(tr != null) {
} catch (RuntimeException err) {
if (tr != null) {
tr.close();
}

131
bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java
View File

@ -26,6 +26,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.function.Function;
import com.apple.foundationdb.EventKeeper.Events;
import com.apple.foundationdb.async.AsyncIterable;
import com.apple.foundationdb.async.AsyncUtil;
import com.apple.foundationdb.tuple.ByteArrayUtil;
@ -34,6 +35,7 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
private final Database database;
private final Executor executor;
private final TransactionOptions options;
private final EventKeeper eventKeeper;
private boolean transactionOwner;
public final ReadTransaction snapshot;
@ -93,9 +95,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
// getRange -> KeySelectors
///////////////////
@Override
public AsyncIterable<KeyValue> getRange(KeySelector begin, KeySelector end,
int limit, boolean reverse, StreamingMode mode) {
return new RangeQuery(FDBTransaction.this, true, begin, end, limit, reverse, mode);
public AsyncIterable<KeyValue> getRange(KeySelector begin, KeySelector end, int limit, boolean reverse,
StreamingMode mode) {
return new RangeQuery(FDBTransaction.this, true, begin, end, limit, reverse, mode, eventKeeper);
}
@Override
public AsyncIterable<KeyValue> getRange(KeySelector begin, KeySelector end,
@ -195,9 +197,15 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
}
protected FDBTransaction(long cPtr, Database database, Executor executor) {
//added for backwards compatibility with subclasses contained in different projects
this(cPtr,database,executor,null);
}
protected FDBTransaction(long cPtr, Database database, Executor executor, EventKeeper eventKeeper) {
super(cPtr);
this.database = database;
this.executor = executor;
this.eventKeeper = eventKeeper;
snapshot = new ReadSnapshot();
options = new TransactionOptions(this);
transactionOwner = true;
@ -220,6 +228,17 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public void setReadVersion(long version) {
/*
* Note that this is done outside of the lock, because we don't want to rely on
* the caller code being particularly efficient, and if we get a bad
* implementation of a eventKeeper, we could end up holding the pointerReadLock for an
* arbitrary amount of time; this would be Bad(TM), so we execute this outside
* the lock, so that in the worst case only the caller thread itself can be hurt
* by bad callbacks.
*/
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
Transaction_setVersion(getPtr(), version);
@ -233,6 +252,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
*/
@Override
public CompletableFuture<Long> getReadVersion() {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
return new FutureInt64(Transaction_getReadVersion(getPtr()), executor);
@ -250,9 +272,12 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
}
private CompletableFuture<byte[]> get_internal(byte[] key, boolean isSnapshot) {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
return new FutureResult(Transaction_get(getPtr(), key, isSnapshot), executor);
return new FutureResult(Transaction_get(getPtr(), key, isSnapshot), executor,eventKeeper);
} finally {
pointerReadLock.unlock();
}
@ -267,10 +292,14 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
}
private CompletableFuture<byte[]> getKey_internal(KeySelector selector, boolean isSnapshot) {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
return new FutureKey(Transaction_getKey(getPtr(),
selector.getKey(), selector.orEqual(), selector.getOffset(), isSnapshot), executor);
return new FutureKey(
Transaction_getKey(getPtr(), selector.getKey(), selector.orEqual(), selector.getOffset(), isSnapshot),
executor, eventKeeper);
} finally {
pointerReadLock.unlock();
}
@ -278,6 +307,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public CompletableFuture<Long> getEstimatedRangeSizeBytes(byte[] begin, byte[] end) {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
return new FutureInt64(Transaction_getEstimatedRangeSizeBytes(getPtr(), begin, end), executor);
@ -312,7 +344,7 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public AsyncIterable<KeyValue> getRange(KeySelector begin, KeySelector end,
int limit, boolean reverse, StreamingMode mode) {
return new RangeQuery(this, false, begin, end, limit, reverse, mode);
return new RangeQuery(this, false, begin, end, limit, reverse, mode, eventKeeper);
}
@Override
public AsyncIterable<KeyValue> getRange(KeySelector begin, KeySelector end,
@ -384,9 +416,12 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
// Users of this function must close the returned FutureResults when finished
protected FutureResults getRange_internal(
KeySelector begin, KeySelector end,
int rowLimit, int targetBytes, int streamingMode,
int iteration, boolean isSnapshot, boolean reverse) {
KeySelector begin, KeySelector end,
int rowLimit, int targetBytes, int streamingMode,
int iteration, boolean isSnapshot, boolean reverse) {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
/*System.out.println(String.format(
@ -397,7 +432,7 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
Transaction_getRange(getPtr(), begin.getKey(), begin.orEqual(), begin.getOffset(),
end.getKey(), end.orEqual(), end.getOffset(), rowLimit, targetBytes,
streamingMode, iteration, isSnapshot, reverse),
FDB.instance().isDirectBufferQueriesEnabled(), executor);
FDB.instance().isDirectBufferQueriesEnabled(), executor, eventKeeper);
} finally {
pointerReadLock.unlock();
}
@ -435,8 +470,10 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
addConflictRange(key, ByteArrayUtil.join(key, new byte[] { (byte)0 }), ConflictRangeType.WRITE);
}
private void addConflictRange(byte[] keyBegin, byte[] keyEnd,
ConflictRangeType type) {
private void addConflictRange(byte[] keyBegin, byte[] keyEnd, ConflictRangeType type) {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
Transaction_addConflictRange(getPtr(), keyBegin, keyEnd, type.code());
@ -469,8 +506,11 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public void set(byte[] key, byte[] value) {
if(key == null || value == null)
if (key == null || value == null)
throw new IllegalArgumentException("Keys/Values must be non-null");
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
Transaction_set(getPtr(), key, value);
@ -481,8 +521,11 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public void clear(byte[] key) {
if(key == null)
if (key == null)
throw new IllegalArgumentException("Key cannot be null");
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
Transaction_clear(getPtr(), key);
@ -493,8 +536,11 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public void clear(byte[] beginKey, byte[] endKey) {
if(beginKey == null || endKey == null)
if (beginKey == null || endKey == null)
throw new IllegalArgumentException("Keys cannot be null");
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
Transaction_clear(getPtr(), beginKey, endKey);
@ -516,6 +562,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public void mutate(MutationType optype, byte[] key, byte[] value) {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
Transaction_mutate(getPtr(), optype.code(), key, value);
@ -526,6 +575,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public void setOption(int code, byte[] param) {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
Transaction_setOption(getPtr(), code, param);
@ -536,6 +588,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public CompletableFuture<Void> commit() {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
return new FutureVoid(Transaction_commit(getPtr()), executor);
@ -546,6 +601,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public Long getCommittedVersion() {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
return Transaction_getCommittedVersion(getPtr());
@ -556,9 +614,12 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public CompletableFuture<byte[]> getVersionstamp() {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
return new FutureKey(Transaction_getVersionstamp(getPtr()), executor);
return new FutureKey(Transaction_getVersionstamp(getPtr()), executor, eventKeeper);
} finally {
pointerReadLock.unlock();
}
@ -566,6 +627,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public CompletableFuture<Long> getApproximateSize() {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
return new FutureInt64(Transaction_getApproximateSize(getPtr()), executor);
@ -576,6 +640,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public CompletableFuture<Void> watch(byte[] key) throws FDBException {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
return new FutureVoid(Transaction_watch(getPtr(), key), executor);
@ -586,24 +653,27 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public CompletableFuture<Transaction> onError(Throwable e) {
if((e instanceof CompletionException || e instanceof ExecutionException) && e.getCause() != null) {
if ((e instanceof CompletionException || e instanceof ExecutionException) && e.getCause() != null) {
e = e.getCause();
}
if(!(e instanceof FDBException)) {
if (!(e instanceof FDBException)) {
CompletableFuture<Transaction> future = new CompletableFuture<>();
future.completeExceptionally(e);
return future;
}
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
CompletableFuture<Void> f = new FutureVoid(Transaction_onError(getPtr(), ((FDBException)e).getCode()), executor);
CompletableFuture<Void> f = new FutureVoid(Transaction_onError(getPtr(), ((FDBException) e).getCode()),
executor);
final Transaction tr = transfer();
return f.thenApply(v -> tr)
.whenComplete((v, t) -> {
if(t != null) {
tr.close();
}
});
return f.thenApply(v -> tr).whenComplete((v, t) -> {
if (t != null) {
tr.close();
}
});
} finally {
pointerReadLock.unlock();
if(!transactionOwner) {
@ -614,6 +684,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
public void cancel() {
if(eventKeeper!=null){
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
Transaction_cancel(getPtr());
@ -623,6 +696,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
}
public CompletableFuture<String[]> getAddressesForKey(byte[] key) {
if(eventKeeper!=null){
eventKeeper.increment(Events.JNI_CALL);
}
pointerReadLock.lock();
try {
return new FutureStrings(Transaction_getKeyLocations(getPtr(), key), executor);
@ -672,6 +748,9 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC
@Override
protected void closeInternal(long cPtr) {
if(eventKeeper!=null){
eventKeeper.increment(Events.JNI_CALL);
}
if(transactionOwner) {
Transaction_dispose(cPtr);
}

14
bindings/java/src/main/com/apple/foundationdb/FutureKey.java
View File

@ -22,9 +22,13 @@ package com.apple.foundationdb;
import java.util.concurrent.Executor;
import com.apple.foundationdb.EventKeeper.Events;
class FutureKey extends NativeFuture<byte[]> {
FutureKey(long cPtr, Executor executor) {
private final EventKeeper eventKeeper;
FutureKey(long cPtr, Executor executor, EventKeeper eventKeeper) {
super(cPtr);
this.eventKeeper = eventKeeper;
registerMarshalCallback(executor);
}
@ -32,6 +36,14 @@ class FutureKey extends NativeFuture<byte[]> {
protected byte[] getIfDone_internal(long cPtr) throws FDBException {
return FutureKey_get(cPtr);
}
@Override
protected void postMarshal(byte[] value) {
if(value!=null && eventKeeper!=null){
eventKeeper.count(Events.BYTES_FETCHED, value.length);
}
super.postMarshal(value);
}
private native byte[] FutureKey_get(long cPtr) throws FDBException;
}

15
bindings/java/src/main/com/apple/foundationdb/FutureResult.java
View File

@ -22,9 +22,14 @@ package com.apple.foundationdb;
import java.util.concurrent.Executor;
import com.apple.foundationdb.EventKeeper.Events;
class FutureResult extends NativeFuture<byte[]> {
FutureResult(long cPtr, Executor executor) {
private final EventKeeper eventKeeper;
FutureResult(long cPtr, Executor executor, EventKeeper eventKeeper) {
super(cPtr);
this.eventKeeper = eventKeeper;
registerMarshalCallback(executor);
}
@ -33,5 +38,13 @@ class FutureResult extends NativeFuture<byte[]> {
return FutureResult_get(cPtr);
}
@Override
protected void postMarshal(byte[] value){
if(value!=null && eventKeeper!=null){
eventKeeper.count(Events.BYTES_FETCHED, value.length);
}
super.postMarshal(value);
}
private native byte[] FutureResult_get(long cPtr) throws FDBException;
}

25
bindings/java/src/main/com/apple/foundationdb/FutureResults.java
View File

@ -23,20 +23,27 @@ package com.apple.foundationdb;
import java.nio.ByteBuffer;
import java.util.concurrent.Executor;
import com.apple.foundationdb.EventKeeper.Events;
class FutureResults extends NativeFuture<RangeResultInfo> {
FutureResults(long cPtr, boolean enableDirectBufferQueries, Executor executor) {
private final EventKeeper eventKeeper;
FutureResults(long cPtr, boolean enableDirectBufferQueries, Executor executor, EventKeeper eventKeeper) {
super(cPtr);
registerMarshalCallback(executor);
this.enableDirectBufferQueries = enableDirectBufferQueries;
this.eventKeeper = eventKeeper;
}
@Override
protected void postMarshal() {
protected void postMarshal(RangeResultInfo rri) {
// We can't close because this class actually marshals on-demand
}
@Override
protected RangeResultInfo getIfDone_internal(long cPtr) throws FDBException {
if (eventKeeper != null) {
eventKeeper.increment(Events.JNI_CALL);
}
FDBException err = Future_getError(cPtr);
if(err != null && !err.isSuccess()) {
@ -47,9 +54,15 @@ class FutureResults extends NativeFuture<RangeResultInfo> {
}
public RangeResult getResults() {
ByteBuffer buffer = enableDirectBufferQueries
? DirectBufferPool.getInstance().poll()
: null;
ByteBuffer buffer = enableDirectBufferQueries ? DirectBufferPool.getInstance().poll() : null;
if (buffer != null && eventKeeper != null) {
eventKeeper.increment(Events.RANGE_QUERY_DIRECT_BUFFER_HIT);
eventKeeper.increment(Events.JNI_CALL);
} else if (eventKeeper != null) {
eventKeeper.increment(Events.RANGE_QUERY_DIRECT_BUFFER_MISS);
eventKeeper.increment(Events.JNI_CALL);
}
try {
pointerReadLock.lock();
if (buffer != null) {
@ -68,6 +81,6 @@ class FutureResults extends NativeFuture<RangeResultInfo> {
private boolean enableDirectBufferQueries = false;
private native RangeResult FutureResults_get(long cPtr) throws FDBException;
private native void FutureResults_getDirect(long cPtr, ByteBuffer buffer, int capacity)
private native boolean FutureResults_getDirect(long cPtr, ByteBuffer buffer, int capacity)
throws FDBException;
}

72
bindings/java/src/main/com/apple/foundationdb/MapEventKeeper.java Normal file
View File

@ -0,0 +1,72 @@
/*
* MapEventKeeper.java
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.apple.foundationdb;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong;
/**
* A simple map-based EventKeeper.
*
* This class is thread-safe(per the {@link EventKeeper} spec). It holds all counters in memory;
*/
public class MapEventKeeper implements EventKeeper {
private final ConcurrentMap<Event, Count> map = new ConcurrentHashMap<>();
@Override
public void count(Event event, long amt) {
Count counter = map.computeIfAbsent(event, (l) -> new Count());
counter.cnt.addAndGet(amt);
}
@Override
public void timeNanos(Event event, long nanos) {
Count counter = map.computeIfAbsent(event, (l)->new Count());
counter.cnt.incrementAndGet();
counter.duration.addAndGet(nanos);
}
@Override
public long getCount(Event event) {
Count lng = map.get(event);
if (lng == null) {
return 0L;
}
return lng.cnt.get();
}
@Override
public long getTimeNanos(Event event) {
Count lng = map.get(event);
if (lng == null) {
return 0L;
}
return lng.duration.get();
}
private static class Count {
private final AtomicLong cnt = new AtomicLong(0L);
private final AtomicLong duration = new AtomicLong(0L);
Count(){ }
}
}

6
bindings/java/src/main/com/apple/foundationdb/NativeFuture.java
View File

@ -54,8 +54,8 @@ abstract class NativeFuture<T> extends CompletableFuture<T> implements AutoClose
}
private void marshalWhenDone() {
T val = null;
try {
T val = null;
boolean shouldComplete = false;
try {
pointerReadLock.lock();
@ -77,11 +77,11 @@ abstract class NativeFuture<T> extends CompletableFuture<T> implements AutoClose
} catch(Throwable t) {
completeExceptionally(t);
} finally {
postMarshal();
postMarshal(val);
}
}
protected void postMarshal() {
protected void postMarshal(T value) {
close();
}

50
bindings/java/src/main/com/apple/foundationdb/RangeQuery.java
View File

@ -26,6 +26,7 @@ import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.function.BiConsumer;
import com.apple.foundationdb.EventKeeper.Events;
import com.apple.foundationdb.async.AsyncIterable;
import com.apple.foundationdb.async.AsyncIterator;
import com.apple.foundationdb.async.AsyncUtil;
@ -52,10 +53,10 @@ class RangeQuery implements AsyncIterable<KeyValue> {
private final int rowLimit;
private final boolean reverse;
private final StreamingMode streamingMode;
private final EventKeeper eventKeeper;
RangeQuery(FDBTransaction transaction, boolean isSnapshot,
KeySelector begin, KeySelector end, int rowLimit,
boolean reverse, StreamingMode streamingMode) {
RangeQuery(FDBTransaction transaction, boolean isSnapshot, KeySelector begin, KeySelector end, int rowLimit,
boolean reverse, StreamingMode streamingMode, EventKeeper eventKeeper) {
this.tr = transaction;
this.begin = begin;
this.end = end;
@ -63,6 +64,7 @@ class RangeQuery implements AsyncIterable<KeyValue> {
this.rowLimit = rowLimit;
this.reverse = reverse;
this.streamingMode = streamingMode;
this.eventKeeper = eventKeeper;
}
/**
@ -88,9 +90,10 @@ class RangeQuery implements AsyncIterable<KeyValue> {
.whenComplete((result, e) -> range.close());
}
// If the streaming mode is not EXACT, simply collect the results of an iteration into a list
return AsyncUtil.collect(
new RangeQuery(tr, snapshot, begin, end, rowLimit, reverse, mode), tr.getExecutor());
// If the streaming mode is not EXACT, simply collect the results of an
// iteration into a list
return AsyncUtil.collect(new RangeQuery(tr, snapshot, begin, end, rowLimit, reverse, mode, eventKeeper),
tr.getExecutor());
}
/**
@ -152,9 +155,12 @@ class RangeQuery implements AsyncIterable<KeyValue> {
@Override
public void accept(RangeResultInfo data, Throwable error) {
try {
if(error != null) {
if (error != null) {
if (eventKeeper != null) {
eventKeeper.increment(Events.RANGE_QUERY_CHUNK_FAILED);
}
promise.completeExceptionally(error);
if(error instanceof Error) {
if (error instanceof Error) {
throw (Error) error;
}
@ -213,12 +219,20 @@ class RangeQuery implements AsyncIterable<KeyValue> {
fetchOutstanding = true;
nextChunk = null;
fetchingChunk = tr.getRange_internal(begin, end,
rowsLimited ? rowsRemaining : 0, 0, streamingMode.code(),
nextFuture = new CompletableFuture<>();
final long sTime = System.nanoTime();
fetchingChunk = tr.getRange_internal(begin, end, rowsLimited ? rowsRemaining : 0, 0, streamingMode.code(),
++iteration, snapshot, reverse);
nextFuture = new CompletableFuture<>();
fetchingChunk.whenComplete(new FetchComplete(fetchingChunk, nextFuture));
BiConsumer<RangeResultInfo,Throwable> cons = new FetchComplete(fetchingChunk,nextFuture);
if(eventKeeper!=null){
eventKeeper.increment(Events.RANGE_QUERY_FETCHES);
cons = cons.andThen((r,t)->{
eventKeeper.timeNanos(Events.RANGE_QUERY_FETCH_TIME_NANOS, System.nanoTime()-sTime);
});
}
fetchingChunk.whenComplete(cons);
}
@Override
@ -233,7 +247,7 @@ class RangeQuery implements AsyncIterable<KeyValue> {
// We have a chunk and are still working though it
if(index < chunk.values.size()) {
return AsyncUtil.READY_TRUE;
return AsyncUtil.READY_TRUE;
}
// If we are at the end of the current chunk there is either:
@ -267,6 +281,16 @@ class RangeQuery implements AsyncIterable<KeyValue> {
prevKey = result.getKey();
index++;
if (eventKeeper != null) {
// We record the BYTES_FETCHED here, rather than at a lower level,
// because some parts of the construction of a RangeResult occur underneath
// the JNI boundary, and we don't want to pass the eventKeeper down there
// (note: account for the length fields as well when recording the bytes
// fetched)
eventKeeper.count(Events.BYTES_FETCHED, result.getKey().length + result.getValue().length + 8);
eventKeeper.increment(Events.RANGE_QUERY_RECORDS_FETCHED);
}
// If this is the first call to next() on a chunk there cannot
// be another waiting, since we could not have issued a request
assert(!(initialNext && nextChunk != null));

11
bindings/java/src/main/com/apple/foundationdb/tuple/FastByteComparisons.java
View File

@ -72,6 +72,13 @@ abstract class FastByteComparisons {
T buffer2, int offset2, int length2);
}
/**
* @return a byte[] comparator for use in sorting, collections, and so on internally
* to the Java code.
*/
public static Comparator<byte[]> comparator(){
return LexicographicalComparerHolder.getBestComparer();
}
/**
* Pure Java Comparer
*
@ -291,8 +298,4 @@ abstract class FastByteComparisons {
}
}
}
public static Comparator<byte[]> comparator() {
return LexicographicalComparerHolder.BEST_COMPARER;
}
}

1
bindings/java/src/tests.cmake
View File

@ -32,6 +32,7 @@ set(JAVA_JUNIT_TESTS
src/junit/com/apple/foundationdb/tuple/TuplePackingTest.java
src/junit/com/apple/foundationdb/tuple/TupleSerializationTest.java
src/junit/com/apple/foundationdb/RangeQueryTest.java
src/junit/com/apple/foundationdb/EventKeeperTest.java
)
# Resources that are used in unit testing, but are not explicitly test files (JUnit rules, utility

158
build/docker/buildspec.yml
View File

@ -1,158 +0,0 @@
version: 0.2
env:
secrets-manager:
DOCKERHUB_AUTH: dockerhub_foundationdb:foundationdb
phases:
install:
commands:
- echo "install phase"
- 'ACCOUNT_ID=$(echo $CODEBUILD_BUILD_ARN | cut -d : -f 5)'
- REGISTRY=${ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com
- aws ecr get-login-password | docker login --username AWS --password-stdin ${REGISTRY}
- docker pull ${REGISTRY}/centos:6
- docker tag ${REGISTRY}/centos:6 centos:6
- docker pull ${REGISTRY}/centos:7
- docker tag ${REGISTRY}/centos:7 centos:7
pre_build:
commands:
- echo "pre_build phase"
- COMMIT_HASH=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7)
- DATE_STR=$(date +"%Y%m%d%H%M%S")
build:
commands:
- echo "build phase"
- ################################################################################
- # CENTOS 7 foundationdb/build
- ################################################################################
- cd ${CODEBUILD_SRC_DIR}/build/docker/centos7/build
- docker pull ${REGISTRY}/foundationdb/build:centos7-latest || true
- docker build --cache-from ${REGISTRY}/foundationdb/build:centos7-latest
--tag ${REGISTRY}/foundationdb/build:centos7-${DATE_STR}-${COMMIT_HASH}
--tag ${REGISTRY}/foundationdb/build:centos7-latest
--tag ${REGISTRY}/foundationdb/build:latest
--tag foundationdb/build:centos7-${DATE_STR}-${COMMIT_HASH}
--tag foundationdb/build:centos7-latest
--tag foundationdb/build:latest
.
- ################################################################################
- # CENTOS 7 foundationdb/devel
- ################################################################################
- cd ${CODEBUILD_SRC_DIR}/build/docker/centos7/devel
- docker pull ${REGISTRY}/foundationdb/devel:centos7-latest || true
- docker build --cache-from ${REGISTRY}/foundationdb/devel:centos7-latest
--build-arg REPOSITORY=${REGISTRY}/foundationdb/build
--tag ${REGISTRY}/foundationdb/devel:centos7-${DATE_STR}-${COMMIT_HASH}
--tag ${REGISTRY}/foundationdb/devel:centos7-latest
--tag ${REGISTRY}/foundationdb/devel:latest
--tag foundationdb/devel:centos7-${DATE_STR}-${COMMIT_HASH}
--tag foundationdb/devel:centos7-latest
--tag foundationdb/devel:latest
.
- ################################################################################
- # CENTOS 7 foundationdb/distcc
- ################################################################################
- cd ${CODEBUILD_SRC_DIR}/build/docker/centos7/distcc
- docker pull ${REGISTRY}/foundationdb/distcc:centos7-latest || true
- docker build --cache-from ${REGISTRY}/foundationdb/distcc:centos7-latest
--build-arg REPOSITORY=${REGISTRY}/foundationdb/build
--tag ${REGISTRY}/foundationdb/distcc:centos7-${DATE_STR}-${COMMIT_HASH}
--tag ${REGISTRY}/foundationdb/distcc:centos7-latest
--tag ${REGISTRY}/foundationdb/distcc:latest
--tag foundationdb/distcc:centos7-${DATE_STR}-${COMMIT_HASH}
--tag foundationdb/distcc:centos7-latest
--tag foundationdb/distcc:latest
.
- ################################################################################
- # CENTOS 6 foundationdb/build
- ################################################################################
- cd ${CODEBUILD_SRC_DIR}/build/docker/centos6/build
- docker pull ${REGISTRY}/foundationdb/build:centos6-latest || true
- docker build --cache-from ${REGISTRY}/foundationdb/build:centos6-latest
--tag ${REGISTRY}/foundationdb/build:centos6-${DATE_STR}-${COMMIT_HASH}
--tag ${REGISTRY}/foundationdb/build:centos6-latest
--tag foundationdb/build:centos6-${DATE_STR}-${COMMIT_HASH}
--tag foundationdb/build:centos6-latest
.
- ################################################################################
- # CENTOS 6 foundationdb/devel
- ################################################################################
- cd ${CODEBUILD_SRC_DIR}/build/docker/centos6/devel
- docker pull ${REGISTRY}/foundationdb/devel:centos6-latest || true
- docker build --cache-from ${REGISTRY}/foundationdb/devel:centos6-latest
--build-arg REPOSITORY=${REGISTRY}/foundationdb/build
--tag ${REGISTRY}/foundationdb/devel:centos6-${DATE_STR}-${COMMIT_HASH}
--tag ${REGISTRY}/foundationdb/devel:centos6-latest
--tag foundationdb/devel:centos6-${DATE_STR}-${COMMIT_HASH}
--tag foundationdb/devel:centos6-latest
.
- ################################################################################
- # CENTOS 6 foundationdb/distcc
- ################################################################################
- cd ${CODEBUILD_SRC_DIR}/build/docker/centos6/distcc
- docker pull ${REGISTRY}/foundationdb/distcc:centos6-latest || true
- docker build --cache-from ${REGISTRY}/foundationdb/distcc:centos6-latest
--build-arg REPOSITORY=${REGISTRY}/foundationdb/build
--tag ${REGISTRY}/foundationdb/distcc:centos6-${DATE_STR}-${COMMIT_HASH}
--tag ${REGISTRY}/foundationdb/distcc:centos6-latest
--tag foundationdb/distcc:centos6-${DATE_STR}-${COMMIT_HASH}
--tag foundationdb/distcc:centos6-latest
.
post_build:
commands:
- echo "post_build phase"
- echo ${DOCKERHUB_AUTH} | docker login --username foundationdb --password-stdin
- ################################################################################
- # CENTOS 7 PUSH TO ECR
- ################################################################################
- # PUSH TO build ECR
- docker push ${REGISTRY}/foundationdb/build:centos7-${DATE_STR}-${COMMIT_HASH}
- docker push ${REGISTRY}/foundationdb/build:centos7-latest
- docker push ${REGISTRY}/foundationdb/build:latest
- # PUSH TO devel ECR
- docker push ${REGISTRY}/foundationdb/devel:centos7-${DATE_STR}-${COMMIT_HASH}
- docker push ${REGISTRY}/foundationdb/devel:centos7-latest
- docker push ${REGISTRY}/foundationdb/devel:latest
- # PUSH TO distcc ECR
- docker push ${REGISTRY}/foundationdb/distcc:centos7-${DATE_STR}-${COMMIT_HASH}
- docker push ${REGISTRY}/foundationdb/distcc:centos7-latest
- docker push ${REGISTRY}/foundationdb/distcc:latest
- ################################################################################
- # CENTOS 7 PUSH TO DOCKERHUB
- ################################################################################
- # PUSH TO build DOCKERHUB
- docker push foundationdb/build:centos7-${DATE_STR}-${COMMIT_HASH}
- docker push foundationdb/build:centos7-latest
- docker push foundationdb/build:latest
- # PUSH TO devel DOCKERHUB
- docker push foundationdb/devel:centos7-${DATE_STR}-${COMMIT_HASH}
- docker push foundationdb/devel:centos7-latest
- docker push foundationdb/devel:latest
- # PUSH TO distcc DOCKERHUB
- docker push foundationdb/distcc:centos7-${DATE_STR}-${COMMIT_HASH}
- docker push foundationdb/distcc:centos7-latest
- docker push foundationdb/distcc:latest
- ################################################################################
- # CENTOS 6 PUSH TO ECR
- ################################################################################
- # PUSH TO build ECR
- docker push ${REGISTRY}/foundationdb/build:centos6-${DATE_STR}-${COMMIT_HASH}
- docker push ${REGISTRY}/foundationdb/build:centos6-latest
- # PUSH TO devel ECR
- docker push ${REGISTRY}/foundationdb/devel:centos6-${DATE_STR}-${COMMIT_HASH}
- docker push ${REGISTRY}/foundationdb/devel:centos6-latest
- # PUSH TO distcc ECR
- docker push ${REGISTRY}/foundationdb/distcc:centos6-${DATE_STR}-${COMMIT_HASH}
- docker push ${REGISTRY}/foundationdb/distcc:centos6-latest
- ################################################################################
- # CENTOS 6 PUSH TO DOCKERHUB
- ################################################################################
- # PUSH TO build DOCKERHUB
- docker push foundationdb/build:centos6-${DATE_STR}-${COMMIT_HASH}
- docker push foundationdb/build:centos6-latest
- # PUSH TO devel DOCKERHUB
- docker push foundationdb/devel:centos6-${DATE_STR}-${COMMIT_HASH}
- docker push foundationdb/devel:centos6-latest
- # PUSH TO distcc DOCKERHUB
- docker push foundationdb/distcc:centos6-${DATE_STR}-${COMMIT_HASH}
- docker push foundationdb/distcc:centos6-latest

8
build/docker/centos6/build/Dockerfile
View File

@ -37,13 +37,13 @@ RUN sed -i -e '/enabled/d' /etc/yum.repos.d/CentOS-Base.repo && \
lz4-devel \
lz4-static \
mono-devel \
rh-python36 \
rh-python36-python-devel \
rh-ruby24 \
rpm-build \
tcl-devel \
unzip \
wget && \
wget \
rh-python36 \
rh-python36-python-devel \
rh-ruby24 && \
yum clean all && \
rm -rf /var/cache/yum

60
build/docker/centos6/devel/Dockerfile
View File

@ -5,13 +5,16 @@ FROM ${REPOSITORY}:${VERSION}
# add vscode server
RUN yum repolist && \
yum -y install \
bash-completion \
byobu \
cgdb \
emacs-nox \
jq \
the_silver_searcher \
tmux \
tree \
emacs-nox \
vim \
bash-completion \
jq \
cgdb && \
zsh && \
yum clean all && \
rm -rf /var/cache/yum
@ -19,14 +22,25 @@ WORKDIR /tmp
RUN source /opt/rh/devtoolset-8/enable && \
source /opt/rh/rh-python36/enable && \
pip3 install \
lxml \
psutil \
python-dateutil \
subprocess32 \
psutil && \
subprocess32 && \
mkdir fdb-joshua && \
cd fdb-joshua && \
git clone --branch code_pipeline https://github.com/FoundationDB/fdb-joshua . && \
pip3 install /tmp/fdb-joshua && \
cd /tmp && \
curl -Ls https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.9/2020-11-02/bin/linux/amd64/kubectl -o kubectl && \
echo "3dbe69e6deb35fbd6fec95b13d20ac1527544867ae56e3dae17e8c4d638b25b9 kubectl" > kubectl.txt && \
sha256sum -c kubectl.txt && \
mv kubectl /usr/local/bin/kubectl && \
chmod 755 /usr/local/bin/kubectl && \
curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.0.30.zip -o "awscliv2.zip" && \
echo "7ee475f22c1b35cc9e53affbf96a9ffce91706e154a9441d0d39cbf8366b718e awscliv2.zip" > awscliv2.txt && \
sha256sum -c awscliv2.txt && \
unzip -qq awscliv2.zip && \
./aws/install && \
rm -rf /tmp/*
ARG OLD_FDB_BINARY_DIR=/app/deploy/global_data/oldBinaries/
@ -45,17 +59,23 @@ RUN mkdir -p ${OLD_FDB_BINARY_DIR} \
ln -s ${OLD_TLS_LIBRARY_DIR}/FDBGnuTLS.so /usr/lib/foundationdb/plugins/FDBGnuTLS.so
WORKDIR /root
RUN echo -en "\n"\
"source /opt/rh/devtoolset-8/enable\n"\
"source /opt/rh/rh-python36/enable\n"\
"source /opt/rh/rh-ruby24/enable\n"\
"\n"\
"function cmk() {\n"\
" cmake -S ${HOME}/src/foundationdb -B build_output -D USE_CCACHE=1 -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -G Ninja && ninja -C build_output -j 84\n"\
"}\n"\
"function ct() {\n"\
" cd ${HOME}/build_output && ctest -j 32 --output-on-failure\n"\
"}\n"\
"function j() {\n"\
" python3 -m joshua.joshua --cluster-file /etc/foundationdb/cluster-file \"\${@}\"\n"\
"}\n" >> .bashrc
RUN rm -f /root/anaconda-ks.cfg && \
printf '%s\n' \
'source /opt/rh/devtoolset-8/enable' \
'source /opt/rh/rh-python36/enable' \
'source /opt/rh/rh-ruby26/enable' \
'' \
'function cmk() {' \
' cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=1 -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -G Ninja && ninja -C build_output -j 84' \
'}' \
'function ct() {' \
' cd ${HOME}/build_output && ctest -j 32 --output-on-failure' \
'}' \
'function j() {' \
' python3 -m joshua.joshua "${@}"' \
'}' \
'function jsd() {' \
' j start --tarball $(find ${HOME}/build_output/packages -name correctness\*.tar.gz) "${@}"' \
'}' \
'' \
>> .bashrc

36
build/docker/centos7/build/Dockerfile
View File

@ -10,6 +10,7 @@ RUN rpmkeys --import mono-project.com.rpmkey.pgp && \
epel-release \
scl-utils \
yum-utils && \
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo && \
yum install -y \
autoconf \
automake \
@ -19,6 +20,7 @@ RUN rpmkeys --import mono-project.com.rpmkey.pgp && \
devtoolset-8 \
devtoolset-8-libubsan-devel \
devtoolset-8-valgrind-devel \
docker-ce \
dos2unix \
dpkg \
gettext-devel \
@ -32,13 +34,21 @@ RUN rpmkeys --import mono-project.com.rpmkey.pgp && \
lz4-devel \
lz4-static \
mono-devel \
rh-python36 \
rh-python36-python-devel \
rh-ruby26 \
rpm-build \
tcl-devel \
unzip \
wget && \
if [ "$(uname -p)" == "aarch64" ]; then \
yum install -y \
rh-python38 \
rh-python38-python-devel \
rh-ruby27; \
else \
yum install -y \
rh-python36 \
rh-python36-python-devel \
rh-ruby26; \
fi && \
yum clean all && \
rm -rf /var/cache/yum
@ -51,9 +61,10 @@ RUN source /opt/rh/devtoolset-8/enable && \
tar --strip-components 1 --no-same-owner --directory git -xf git.tar.gz && \
cd git && \
make configure && \
./configure \
&& make && \
./configure && \
make && \
make install && \
cd ../ && \
rm -rf /tmp/*
# build/install ninja
@ -69,8 +80,13 @@ RUN source /opt/rh/devtoolset-8/enable && \
rm -rf /tmp/*
# install cmake
RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz -o cmake.tar.gz && \
echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76 cmake.tar.gz" > cmake-sha.txt && \
RUN if [ "$(uname -p)" == "aarch64" ]; then \
curl -Ls https://github.com/Kitware/CMake/releases/download/v3.19.6/cmake-3.19.6-Linux-aarch64.tar.gz -o cmake.tar.gz; \
echo "69ec045c6993907a4f4a77349d0a0668f1bd3ce8bc5f6fbab6dc7a7e2ffc4f80 cmake.tar.gz" > cmake-sha.txt; \
else \
curl -Ls https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz -o cmake.tar.gz; \
echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76 cmake.tar.gz" > cmake-sha.txt; \
fi && \
sha256sum -c cmake-sha.txt && \
mkdir cmake && \
tar --strip-components 1 --no-same-owner --directory cmake -xf cmake.tar.gz && \
@ -185,7 +201,11 @@ RUN source /opt/rh/devtoolset-8/enable && \
# build/install distcc
RUN source /opt/rh/devtoolset-8/enable && \
source /opt/rh/rh-python36/enable && \
if [ "$(uname -p)" == "aarch64" ]; then \
source /opt/rh/rh-python38/enable; \
else \
source /opt/rh/rh-python36/enable; \
fi && \
curl -Ls https://github.com/distcc/distcc/archive/v3.3.5.tar.gz -o distcc.tar.gz && \
echo "13a4b3ce49dfc853a3de550f6ccac583413946b3a2fa778ddf503a9edc8059b0 distcc.tar.gz" > distcc-sha256.txt && \
sha256sum -c distcc-sha256.txt && \

87
build/docker/centos7/devel/Dockerfile
View File

@ -3,15 +3,21 @@ ARG VERSION=centos7-latest
FROM ${REPOSITORY}:${VERSION}
# add vscode server
RUN yum repolist && \
RUN yum-config-manager --add-repo=https://copr.fedorainfracloud.org/coprs/carlwgeorge/ripgrep/repo/epel-7/carlwgeorge-ripgrep-epel-7.repo && \
yum repolist && \
yum -y install \
bash-completion \
byobu \
cgdb \
emacs-nox \
fish \
jq \
ripgrep \
the_silver_searcher \
tmux \
tree \
emacs-nox \
vim \
bash-completion \
jq \
cgdb && \
zsh && \
yum clean all && \
rm -rf /var/cache/yum
@ -19,14 +25,25 @@ WORKDIR /tmp
RUN source /opt/rh/devtoolset-8/enable && \
source /opt/rh/rh-python36/enable && \
pip3 install \
lxml \
psutil \
python-dateutil \
subprocess32 \
psutil && \
subprocess32 && \
mkdir fdb-joshua && \
cd fdb-joshua && \
git clone --branch code_pipeline https://github.com/FoundationDB/fdb-joshua . && \
pip3 install /tmp/fdb-joshua && \
cd /tmp && \
curl -Ls https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.9/2020-11-02/bin/linux/amd64/kubectl -o kubectl && \
echo "3dbe69e6deb35fbd6fec95b13d20ac1527544867ae56e3dae17e8c4d638b25b9 kubectl" > kubectl.txt && \
sha256sum -c kubectl.txt && \
mv kubectl /usr/local/bin/kubectl && \
chmod 755 /usr/local/bin/kubectl && \
curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.0.30.zip -o "awscliv2.zip" && \
echo "7ee475f22c1b35cc9e53affbf96a9ffce91706e154a9441d0d39cbf8366b718e awscliv2.zip" > awscliv2.txt && \
sha256sum -c awscliv2.txt && \
unzip -qq awscliv2.zip && \
./aws/install && \
rm -rf /tmp/*
ARG OLD_FDB_BINARY_DIR=/app/deploy/global_data/oldBinaries/
@ -49,18 +66,44 @@ RUN curl -Ls https://update.code.visualstudio.com/latest/server-linux-x64/stable
mkdir -p .vscode-server/bin/latest && \
tar --strip-components 1 --no-same-owner --directory .vscode-server/bin/latest -xf /tmp/vscode-server-linux-x64.tar.gz && \
touch .vscode-server/bin/latest/0 && \
rm /tmp/*
RUN echo -en "\n"\
"source /opt/rh/devtoolset-8/enable\n"\
"source /opt/rh/rh-python36/enable\n"\
"source /opt/rh/rh-ruby26/enable\n"\
"\n"\
"function cmk() {\n"\
" cmake -S ${HOME}/src/foundationdb -B build_output -D USE_CCACHE=1 -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -G Ninja && ninja -C build_output -j 84\n"\
"}\n"\
"function ct() {\n"\
" cd ${HOME}/build_output && ctest -j 32 --output-on-failure\n"\
"}\n"\
"function j() {\n"\
" python3 -m joshua.joshua --cluster-file /etc/foundationdb/cluster-file \"\${@}\"\n"\
"}\n" >> .bashrc
rm -rf /tmp/*
RUN rm -f /root/anaconda-ks.cfg && \
printf '%s\n' \
'#!/usr/bin/env bash' \
'set -Eeuo pipefail' \
'' \
'mkdir -p ~/.docker' \
'cat > ~/.docker/config.json << EOF' \
'{' \
' "proxies":' \
' {' \
' "default":' \
' {' \
' "httpProxy": "${HTTP_PROXY}",' \
' "httpsProxy": "${HTTPS_PROXY}",' \
' "noProxy": "${NO_PROXY}"' \
' }' \
' }' \
'}' \
'EOF' \
> docker_proxy.sh && \
chmod 755 docker_proxy.sh && \
printf '%s\n' \
'source /opt/rh/devtoolset-8/enable' \
'source /opt/rh/rh-python36/enable' \
'source /opt/rh/rh-ruby26/enable' \
'' \
'function cmk() {' \
' cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=1 -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -G Ninja && ninja -C build_output -j 84' \
'}' \
'function ct() {' \
' cd ${HOME}/build_output && ctest -j 32 --output-on-failure' \
'}' \
'function j() {' \
' python3 -m joshua.joshua "${@}"' \
'}' \
'function jsd() {' \
' j start --tarball $(find ${HOME}/build_output/packages -name correctness\*.tar.gz) "${@}"' \
'}' \
'' \
>> .bashrc

20
build/docker/centos7/ycsb/Dockerfile Normal file
View File

@ -0,0 +1,20 @@
ARG REPOSITORY=foundationdb/build
ARG VERSION=centos7-latest
FROM ${REPOSITORY}:${VERSION}
ENV YCSB_VERSION=ycsb-foundationdb-binding-0.17.0 \
PATH=${PATH}:/usr/bin
RUN cd /opt \
&& eval curl "-Ls https://github.com/brianfrankcooper/YCSB/releases/download/0.17.0/ycsb-foundationdb-binding-0.17.0.tar.gz" \
| tar -xzvf -
RUN rm -Rf /opt/${YCSB_VERSION}/lib/fdb-java-5.2.5.jar
# COPY The Appropriate fdb-java-.jar Aaron from packages
# COPY binary RPM for foundationd-db
# Install Binary
WORKDIR "/opt/${YCSB_VERSION}"
ENTRYPOINT ["bin/ycsb.sh"]

5
cmake/InstallLayout.cmake
View File

@ -380,10 +380,15 @@ set(CPACK_RPM_CLIENTS-VERSIONED_PRE_UNINSTALL_SCRIPT_FILE
if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
set(CPACK_DEBIAN_CLIENTS-DEB_FILE_NAME "${deb-clients-filename}_amd64.deb")
set(CPACK_DEBIAN_SERVER-DEB_FILE_NAME "${deb-server-filename}_amd64.deb")
set(CPACK_DEBIAN_CLIENTS-VERSIONED_FILE_NAME "${deb-clients-filename}.versioned_amd64.deb")
set(CPACK_DEBIAN_SERVER-VERSIONED_FILE_NAME "${deb-server-filename}.versioned_amd64.deb")
else()
set(CPACK_DEBIAN_CLIENTS-DEB_FILE_NAME "${deb-clients-filename}_${CMAKE_SYSTEM_PROCESSOR}.deb")
set(CPACK_DEBIAN_SERVER-DEB_FILE_NAME "${deb-server-filename}_${CMAKE_SYSTEM_PROCESSOR}.deb")
set(CPACK_DEBIAN_CLIENTS-VERSIONED_FILE_NAME "${deb-clients-filename}.versioned_${CMAKE_SYSTEM_PROCESSOR}.deb")
set(CPACK_DEBIAN_SERVER-VERSIONED_FILE_NAME "${deb-server-filename}.versioned_${CMAKE_SYSTEM_PROCESSOR}.deb")
endif()
set(CPACK_DEB_COMPONENT_INSTALL ON)
set(CPACK_DEBIAN_DEBUGINFO_PACKAGE ${GENERATE_DEBUG_PACKAGES})
set(CPACK_DEBIAN_PACKAGE_SECTION "database")

2
cmake/Jemalloc.cmake
View File

@ -3,7 +3,7 @@ add_library(jemalloc INTERFACE)
set(USE_JEMALLOC ON)
# We don't want to use jemalloc on Windows
# Nor on FreeBSD, where jemalloc is the default system allocator
if(USE_SANITIZER OR WIN32 OR (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD"))
if(USE_SANITIZER OR WIN32 OR (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") OR APPLE)
set(USE_JEMALLOC OFF)
return()
endif()

4
contrib/Joshua/scripts/correctnessTest.sh
View File

@ -1,3 +1,7 @@
#!/bin/sh
# Simulation currently has memory leaks. We need to investigate before we can enable leak detection in joshua.
export ASAN_OPTIONS="detect_leaks=0"
OLDBINDIR="${OLDBINDIR:-/app/deploy/global_data/oldBinaries}"
mono bin/TestHarness.exe joshua-run "${OLDBINDIR}" false

7
contrib/monitoring/actor_flamegraph.cpp
View File

@ -24,7 +24,8 @@ struct Error {
struct Actor {
template <class Str>
explicit Actor(std::unordered_map<std::string, unsigned long>& results, unsigned long id, Str&& name) : results(results), id(id), name(std::forward<Str>(name)) {}
explicit Actor(std::unordered_map<std::string, unsigned long>& results, unsigned long id, Str&& name)
: results(results), id(id), name(std::forward<Str>(name)) {}
Actor(const Actor&) = delete;
~Actor() { collect(); }
std::unordered_map<std::string, unsigned long>& results;
@ -40,12 +41,12 @@ struct Actor {
for (auto i = stack.begin(); i != stack.end();) {
int num = 0;
auto name = *i;
for (; i != stack.end() && *i == name ; ++i) {
for (; i != stack.end() && *i == name; ++i) {
++num;
}
ss << name;
if (num > 1) {
ss << " ("<< num << ')';
ss << " (" << num << ')';
}
ss << ';';
}

6
documentation/sphinx/source/api-c.rst
View File

@ -481,7 +481,11 @@ An |database-blurb1| Modifications to a database are performed via transactions.
|length-of| ``snapshot_command``
.. note:: The function is exposing the functionality of the fdbcli command ``snapshot``. Please take a look at the documentation before using (see :ref:`disk-snapshot-backups`).
.. function:: double fdb_database_get_main_thread_busyness(FDBDatabase* database)
Returns a value where 0 indicates that the client is idle and 1 (or larger) indicates that the client is saturated. By default, this value is updated every second.
Transaction
===========

3
documentation/sphinx/source/api-common.rst.inc
View File

@ -136,7 +136,8 @@
Transforms ``param`` using a versionstamp for the transaction. This parameter must be at least 14 bytes long. The final 4 bytes will be interpreted as a 32-bit little-endian integer denoting an index into the parameter at which to perform the transformation, and then trimmed off the key. The 10 bytes in the parameter beginning at the index will be overwritten with the versionstamp. If the index plus 10 bytes points past the end of the parameter, the result will be an error. Sets ``key`` in the database to the transformed parameter.
.. |atomic-versionstamps-1| replace::
A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database. The last 2 bytes are monotonic in the serialization order for transactions.
A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-endian order). The last 2 bytes are monotonic in the serialization order for transactions (serialized in big-endian order).
.. |atomic-versionstamps-2| replace::
A transaction is not permitted to read any transformed key or value previously set within that transaction, and an attempt to do so will result in an ``accessed_unreadable`` error. The range of keys marked unreadable when setting a versionstamped key begins at the transactions's read version if it is known, otherwise a versionstamp of all ``0x00`` bytes is conservatively assumed. The upper bound of the unreadable range is a versionstamp of all ``0xFF`` bytes.

7
documentation/sphinx/source/release-notes/release-notes-620.rst
View File

@ -2,6 +2,13 @@
Release Notes
#############
6.2.33
======
* Fixed an issue where storage servers could shutdown with ``unknown_error``. `(PR #4437) <https://github.com/apple/foundationdb/pull/4437>`_
* Fix backup agent stall when writing to local filesystem with slow metadata operations. `(PR #4428) <https://github.com/apple/foundationdb/pull/4428>`_
* Backup agent no longer uses 4k block caching layer on local output files so that write operations are larger. `(PR #4428) <https://github.com/apple/foundationdb/pull/4428>`_
* Fix accounting error that could cause commits to incorrectly fail with ``proxy_memory_limit_exceeded``. `(PR #4529) <https://github.com/apple/foundationdb/pull/4529>`_
6.2.32
======
* Fix an issue where symbolic links in cmake-built RPMs are broken if you unpack the RPM to a custom directory. `(PR #4380) <https://github.com/apple/foundationdb/pull/4380>`_

32
documentation/sphinx/source/release-notes/release-notes-630.rst
View File

@ -2,30 +2,26 @@
Release Notes
#############
6.3.12
======
* Change the default for --knob_tls_server_handshake_threads to 64. The previous was 1000. This avoids starting 1000 threads by default, but may adversely affect recovery time for large clusters using tls. Users with large tls clusters should consider explicitly setting this knob in their foundationdb.conf file. `(PR #4421) <https://github.com/apple/foundationdb/pull/4421>`_
* Fix accounting error that could cause commits to incorrectly fail with ``proxy_memory_limit_exceeded``. `(PR #4526) <https://github.com/apple/foundationdb/pull/4526>`_
* As an optimization, partial restore using target key ranges now filters backup log data prior to loading it into the database. `(PR #4554) <https://github.com/apple/foundationdb/pull/4554>`_
6.3.11
======
* Added a hint field in the trace event when all replicas of some data are lost. `(PR #4209) <https://github.com/apple/foundationdb/pull/4209>`_
* Rewrote SQLite injected fault handling. `(PR #4212) <https://github.com/apple/foundationdb/pull/4212>`_
* Add a SevWarnAlways trace line to help debug a rare failure. `(PR #4214) <https://github.com/apple/foundationdb/pull/4214>`_
* Use VFSAsyncFile::checkInjectedError to detect injected faults. `(PR #4253) <https://github.com/apple/foundationdb/pull/4253>`_
* Build on Windows using VS 2019 + LLVM/Clang. `(PR #4258) <https://github.com/apple/foundationdb/pull/4258>`_
* RateControl support in AFCCached to enable write op throttling. The feature is disabled by default. `(PR #4229) <https://github.com/apple/foundationdb/pull/4229>`_
* Add knobs for prefix bloom filters and larger block cache for RocksDB. `(PR #4201) <https://github.com/apple/foundationdb/pull/4201>`_
* Adding debug tools to FDB runtime image. `(PR #4247) <https://github.com/apple/foundationdb/pull/4247>`_
* Fix bug in simulated coordinator selection. `(PR #4285) <https://github.com/apple/foundationdb/pull/4285>`_
* Add option to prevent synchronous file deletes on reads for RocksDB. `(PR #4270) <https://github.com/apple/foundationdb/pull/4270>`_
* Report warning when TLS verification fails. `(PR #4299) <https://github.com/apple/foundationdb/pull/4299>`_
* Support multiple worker threads for each version of client that is loaded so that each cluster will be serviced by a client thread. `(PR #4269) <https://github.com/apple/foundationdb/pull/4269>`_
* Reboot simulated process on io_timeout error. `(PR #4345) <https://github.com/apple/foundationdb/pull/4345>`_
* Fix Snapshot backup test failure. `(PR #4372) <https://github.com/apple/foundationdb/pull/4372>`_
* Support multiple worker threads for each client version that is loaded. `(PR #4269) <https://github.com/apple/foundationdb/pull/4269>`_
* fdbcli: Output errors and warnings to stderr. `(PR #4332) <https://github.com/apple/foundationdb/pull/4332>`_
* Do not generate machine id in locality field if it is set by the user. `(PR #4022) <https://github.com/apple/foundationdb/pull/4022>`_
* Make the RocksDB init method idempotent. `(PR #4400) <https://github.com/apple/foundationdb/pull/4400>`_
* Fix bugs turned up by _GLIBCXX_DEBUG. `(PR #4301) <https://github.com/apple/foundationdb/pull/4301>`_
* Add New Unit and Integration Tests, and associated infrastructure. `(PR #4366) <https://github.com/apple/foundationdb/pull/4366>`_
* Do not rely on shared memory to generate a machine id if it is set explicitly. `(Issue #4022) <https://github.com/apple/foundationdb/pull/4022>`_
* Added ``workload.transactions.rejected_for_queued_too_long`` to status to report the number of transaction commits that failed because they were queued too long and could no longer be checked for conflicts. `(PR #4353) <https://github.com/apple/foundationdb/pull/4353>`_
* Add knobs for prefix bloom filters and larger block cache for RocksDB. `(PR #4201) <https://github.com/apple/foundationdb/pull/4201>`_
* Add option to prevent synchronous file deletes on reads for RocksDB. `(PR #4270) <https://github.com/apple/foundationdb/pull/4270>`_
* Build on Windows using VS 2019 + LLVM/Clang. `(PR #4258) <https://github.com/apple/foundationdb/pull/4258>`_
6.3.10
======
* Make fault tolerance metric calculation in HA clusters consistent with 6.2 branch. `(PR #4175) <https://github.com/apple/foundationdb/pull/4175>`_
* Bug fix, stack overflow in redwood storage engine. `(PR #4161) <https://github.com/apple/foundationdb/pull/4161>`_
* Bug fix, getting certain special keys fail. `(PR #4128) <https://github.com/apple/foundationdb/pull/4128>`_

22
documentation/tutorial/tutorial.actor.cpp
View File

@ -394,16 +394,18 @@ ACTOR Future<Void> fdbStatusStresser() {
}
}
std::unordered_map<std::string, std::function<Future<Void>()>> actors = { { "timer", &simpleTimer }, // ./tutorial timer
{ "promiseDemo", &promiseDemo }, // ./tutorial promiseDemo
{ "triggerDemo", &triggerDemo }, // ./tutorial triggerDemo
{ "echoServer", &echoServer }, // ./tutorial -p 6666 echoServer
{ "echoClient", &echoClient }, // ./tutorial -s 127.0.0.1:6666 echoClient
{ "kvStoreServer", &kvStoreServer }, // ./tutorial -p 6666 kvStoreServer
{ "kvSimpleClient", &kvSimpleClient }, // ./tutorial -s 127.0.0.1:6666 kvSimpleClient
{ "multipleClients", &multipleClients }, // ./tutorial -s 127.0.0.1:6666 multipleClients
{ "fdbClient", &fdbClient }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClient
{ "fdbStatusStresser", &fdbStatusStresser } }; // ./tutorial -C $CLUSTER_FILE_PATH fdbStatusStresser
std::unordered_map<std::string, std::function<Future<Void>()>> actors = {
{ "timer", &simpleTimer }, // ./tutorial timer
{ "promiseDemo", &promiseDemo }, // ./tutorial promiseDemo
{ "triggerDemo", &triggerDemo }, // ./tutorial triggerDemo
{ "echoServer", &echoServer }, // ./tutorial -p 6666 echoServer
{ "echoClient", &echoClient }, // ./tutorial -s 127.0.0.1:6666 echoClient
{ "kvStoreServer", &kvStoreServer }, // ./tutorial -p 6666 kvStoreServer
{ "kvSimpleClient", &kvSimpleClient }, // ./tutorial -s 127.0.0.1:6666 kvSimpleClient
{ "multipleClients", &multipleClients }, // ./tutorial -s 127.0.0.1:6666 multipleClients
{ "fdbClient", &fdbClient }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClient
{ "fdbStatusStresser", &fdbStatusStresser }
}; // ./tutorial -C $CLUSTER_FILE_PATH fdbStatusStresser
int main(int argc, char* argv[]) {
bool isServer = false;

55
fdbbackup/FileConverter.actor.cpp
View File

@ -148,13 +148,17 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
FileProgress(Reference<IAsyncFile> f, int index) : fd(f), idx(index), offset(0), eof(false) {}
bool operator<(const FileProgress& rhs) const {
if (rhs.mutations.empty()) return true;
if (mutations.empty()) return false;
if (rhs.mutations.empty())
return true;
if (mutations.empty())
return false;
return mutations[0].version < rhs.mutations[0].version;
}
bool operator<=(const FileProgress& rhs) const {
if (rhs.mutations.empty()) return true;
if (mutations.empty()) return false;
if (rhs.mutations.empty())
return true;
if (mutations.empty())
return false;
return mutations[0].version <= rhs.mutations[0].version;
}
bool empty() { return eof && mutations.empty(); }
@ -169,11 +173,13 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
try {
// Read block header
if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version();
if (reader.consume<int32_t>() != PARTITIONED_MLOG_VERSION)
throw restore_unsupported_file_version();
while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF) break;
if (reader.eof() || *reader.rptr == 0xFF)
break;
// Deserialize messages written in saveMutationsToFile().
msgVersion = bigEndian64(reader.consume<Version>());
@ -181,7 +187,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
int msgSize = bigEndian32(reader.consume<int>());
const uint8_t* message = reader.consume(msgSize);
ArenaReader rd(buf.arena(), StringRef(message, msgSize), AssumeVersion(g_network->protocolVersion()));
ArenaReader rd(
buf.arena(), StringRef(message, msgSize), AssumeVersion(g_network->protocolVersion()));
MutationRef m;
rd >> m;
count++;
@ -194,8 +201,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
break; // skip
}
if (msgVersion >= minVersion) {
mutations.emplace_back(LogMessageVersion(msgVersion, sub), StringRef(message, msgSize),
buf.arena());
mutations.emplace_back(
LogMessageVersion(msgVersion, sub), StringRef(message, msgSize), buf.arena());
inserted++;
}
}
@ -232,7 +239,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
bool hasMutations() {
for (const auto& fp : fileProgress) {
if (!fp->empty()) return true;
if (!fp->empty())
return true;
}
return false;
}
@ -252,7 +260,8 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
// Sorts files according to their first mutation version and removes files without mutations.
void sortAndRemoveEmpty() {
std::sort(fileProgress.begin(), fileProgress.end(),
std::sort(fileProgress.begin(),
fileProgress.end(),
[](const Reference<FileProgress>& a, const Reference<FileProgress>& b) { return (*a) < (*b); });
while (!fileProgress.empty() && fileProgress.back()->empty()) {
fileProgress.pop_back();
@ -319,11 +328,15 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
// Decodes the file until EOF or an mutation >= minVersion and saves these mutations.
// Skip mutations >= maxVersion.
ACTOR static Future<Void> decodeToVersion(Reference<FileProgress> fp, Version minVersion, Version maxVersion,
ACTOR static Future<Void> decodeToVersion(Reference<FileProgress> fp,
Version minVersion,
Version maxVersion,
LogFile file) {
if (fp->empty()) return Void();
if (fp->empty())
return Void();
if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion) return Void();
if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion)
return Void();
state int64_t len;
try {
@ -337,13 +350,15 @@ struct MutationFilesReadProgress : public ReferenceCounted<MutationFilesReadProg
state Standalone<StringRef> buf = makeString(len);
int rLen = wait(fp->fd->read(mutateString(buf), len, fp->offset));
if (len != rLen) throw restore_bad_read();
if (len != rLen)
throw restore_bad_read();
TraceEvent("ReadFile")
.detail("Name", fp->fd->getFilename())
.detail("Length", rLen)
.detail("Offset", fp->offset);
if (fp->decodeBlock(buf, rLen, minVersion, maxVersion)) break;
if (fp->decodeBlock(buf, rLen, minVersion, maxVersion))
break;
}
return Void();
} catch (Error& e) {
@ -402,7 +417,8 @@ struct LogFileWriter {
wait(self->file->appendStringRefWithLen(v));
// At this point we should be in whatever the current block is or the block size is too small
if (self->file->size() > self->blockEnd) throw backup_bad_block_size();
if (self->file->size() > self->blockEnd)
throw backup_bad_block_size();
return Void();
}
@ -439,7 +455,8 @@ ACTOR Future<Void> convert(ConvertParams params) {
state BackupDescription desc = wait(container->describeBackup());
std::cout << "\n" << desc.toString() << "\n";
// std::cout << "Using Protocol Version: 0x" << std::hex << g_network->protocolVersion().version() << std::dec << "\n";
// std::cout << "Using Protocol Version: 0x" << std::hex << g_network->protocolVersion().version() << std::dec <<
// "\n";
std::vector<LogFile> logs = getRelevantLogFiles(listing.logs, params.begin, params.end);
printLogFiles("Range has", logs);
@ -550,7 +567,7 @@ int parseCommandLine(ConvertParams* param, CSimpleOpt* args) {
return FDB_EXIT_SUCCESS;
}
} // namespace file_converter
} // namespace file_converter
int main(int argc, char** argv) {
try {

4
fdbbackup/FileConverter.h
View File

@ -61,6 +61,6 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
{ OPT_HELP, "--help", SO_NONE },
SO_END_OF_OPTIONS };
} // namespace file_converter
} // namespace file_converter
#endif // FDBBACKUP_FILECONVERTER_H
#endif // FDBBACKUP_FILECONVERTER_H

24
fdbbackup/FileDecoder.actor.cpp
View File

@ -189,7 +189,8 @@ std::vector<MutationRef> decode_value(const StringRef& value) {
std::vector<MutationRef> mutations;
while (1) {
if (reader.eof()) break;
if (reader.eof())
break;
// Deserialization of a MutationRef, which was packed by MutationListRef::push_back_deep()
uint32_t type, p1len, p2len;
@ -242,8 +243,7 @@ class DecodeProgress {
public:
DecodeProgress() = default;
template <class U>
DecodeProgress(const LogFile& file, U &&values)
: file(file), keyValues(std::forward<U>(values)) {}
DecodeProgress(const LogFile& file, U&& values) : file(file), keyValues(std::forward<U>(values)) {}
// If there are no more mutations to pull from the file.
// However, we could have unfinished version in the buffer when EOF is true,
@ -289,7 +289,8 @@ public:
int idx = 1; // next kv pair in "keyValues"
int bufSize = kv.kv.size();
for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) {
if (idx == self->keyValues.size()) break;
if (idx == self->keyValues.size())
break;
const auto& nextKV = self->keyValues[idx];
if (kv.version != nextKV.version) {
@ -355,12 +356,14 @@ public:
try {
// Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION
if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION) throw restore_unsupported_file_version();
if (reader.consume<int32_t>() != BACKUP_AGENT_MLOG_VERSION)
throw restore_unsupported_file_version();
// Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte.
while (1) {
// If eof reached or first key len bytes is 0xFF then end of block was reached.
if (reader.eof() || *reader.rptr == 0xFF) break;
if (reader.eof() || *reader.rptr == 0xFF)
break;
// Read key and value. If anything throws then there is a problem.
uint32_t kLen = reader.consumeNetworkUInt32();
@ -379,7 +382,8 @@ public:
// Make sure any remaining bytes in the block are 0xFF
for (auto b : reader.remainder()) {
if (b != 0xFF) throw restore_corrupted_data_padding();
if (b != 0xFF)
throw restore_corrupted_data_padding();
}
// The (version, part) in a block can be out of order, i.e., (3, 0)
@ -445,7 +449,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
state BackupFileList listing = wait(container->dumpFileList());
// remove partitioned logs
listing.logs.erase(std::remove_if(listing.logs.begin(), listing.logs.end(),
listing.logs.erase(std::remove_if(listing.logs.begin(),
listing.logs.end(),
[](const LogFile& file) {
std::string prefix("plogs/");
return file.fileName.substr(0, prefix.size()) == prefix;
@ -464,7 +469,8 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
// Previous file's unfinished version data
state std::vector<VersionedKVPart> left;
for (; i < logs.size(); i++) {
if (logs[i].fileSize == 0) continue;
if (logs[i].fileSize == 0)
continue;
state DecodeProgress progress(logs[i], std::move(left));
wait(progress.openFile(container));

2880
fdbbackup/backup.actor.cpp
View File

File diff suppressed because it is too large Load Diff

221
fdbcli/FlowLineNoise.actor.cpp
View File

@ -29,153 +29,152 @@
#include "flow/ThreadHelper.actor.h"
#if __unixish__
#define HAVE_LINENOISE 1
#include "fdbcli/linenoise/linenoise.h"
#define HAVE_LINENOISE 1
#include "fdbcli/linenoise/linenoise.h"
#else
#define HAVE_LINENOISE 0
#define HAVE_LINENOISE 0
#endif
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/actorcompiler.h" // This must be the last #include.
struct LineNoiseReader final : IThreadPoolReceiver {
void init() override {}
struct Read final : TypedAction<LineNoiseReader, Read> {
std::string prompt;
ThreadReturnPromise<Optional<std::string>> result;
ThreadReturnPromise<Optional<std::string>> result;
double getTimeEstimate() const override { return 0.0; }
explicit Read(std::string const& prompt) : prompt(prompt) {}
};
void action(Read& r) {
try {
r.result.send( read(r.prompt) );
} catch (Error& e) {
r.result.sendError(e);
} catch (...) {
r.result.sendError(unknown_error());
}
}
try {
r.result.send(read(r.prompt));
} catch (Error& e) {
r.result.sendError(e);
} catch (...) {
r.result.sendError(unknown_error());
}
}
private:
Optional<std::string> read(std::string const& prompt) {
#if HAVE_LINENOISE
errno = 0;
char* line = linenoise(prompt.c_str());
if (line) {
std::string s(line);
free(line);
return s;
} else {
if (errno == EAGAIN) // Ctrl-C
return std::string();
return Optional<std::string>();
}
#else
std::string line;
std::fputs( prompt.c_str(), stdout );
if (!std::getline( std::cin, line ).eof()) {
return line;
} else
return Optional<std::string>();
#endif
}
Optional<std::string> read(std::string const& prompt) {
#if HAVE_LINENOISE
errno = 0;
char* line = linenoise(prompt.c_str());
if (line) {
std::string s(line);
free(line);
return s;
} else {
if (errno == EAGAIN) // Ctrl-C
return std::string();
return Optional<std::string>();
}
#else
std::string line;
std::fputs(prompt.c_str(), stdout);
if (!std::getline(std::cin, line).eof()) {
return line;
} else
return Optional<std::string>();
#endif
}
};
LineNoise::LineNoise(
std::function< void(std::string const&, std::vector<std::string>&) > _completion_callback,
std::function< Hint(std::string const&) > _hint_callback,
int maxHistoryLines,
bool multiline )
: threadPool( createGenericThreadPool() )
{
reader = new LineNoiseReader();
LineNoise::LineNoise(std::function<void(std::string const&, std::vector<std::string>&)> _completion_callback,
std::function<Hint(std::string const&)> _hint_callback,
int maxHistoryLines,
bool multiline)
: threadPool(createGenericThreadPool()) {
reader = new LineNoiseReader();
#if HAVE_LINENOISE
// It should be OK to call these functions from this thread, since read() can't be called yet
// The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely invoke the callbacks we've been given
#if HAVE_LINENOISE
// It should be OK to call these functions from this thread, since read() can't be called yet
// The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely
// invoke the callbacks we've been given
// linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables
static std::function< void(std::string const&, std::vector<std::string>&) > completion_callback;
static std::function< Hint(std::string const&) > hint_callback;
completion_callback = _completion_callback;
hint_callback = _hint_callback;
// linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables
static std::function<void(std::string const&, std::vector<std::string>&)> completion_callback;
static std::function<Hint(std::string const&)> hint_callback;
completion_callback = _completion_callback;
hint_callback = _hint_callback;
linenoiseHistorySetMaxLen( maxHistoryLines );
linenoiseSetMultiLine( multiline );
linenoiseSetCompletionCallback( [](const char* line, linenoiseCompletions* lc) {
// This code will run in the thread pool
std::vector<std::string> completions;
onMainThread( [line, &completions]() -> Future<Void> {
completion_callback(line, completions);
return Void();
}).getBlocking();
for( auto const& c : completions )
linenoiseAddCompletion( lc, c.c_str() );
});
linenoiseSetHintsCallback( [](const char* line, int* color, int*bold) -> char* {
Hint h = onMainThread( [line]() -> Future<Hint> {
return hint_callback(line);
}).getBlocking();
if (!h.valid) return nullptr;
*color = h.color;
*bold = h.bold;
return strdup( h.text.c_str() );
});
linenoiseSetFreeHintsCallback( free );
#endif
linenoiseHistorySetMaxLen(maxHistoryLines);
linenoiseSetMultiLine(multiline);
linenoiseSetCompletionCallback([](const char* line, linenoiseCompletions* lc) {
// This code will run in the thread pool
std::vector<std::string> completions;
onMainThread([line, &completions]() -> Future<Void> {
completion_callback(line, completions);
return Void();
}).getBlocking();
for (auto const& c : completions)
linenoiseAddCompletion(lc, c.c_str());
});
linenoiseSetHintsCallback([](const char* line, int* color, int* bold) -> char* {
Hint h = onMainThread([line]() -> Future<Hint> { return hint_callback(line); }).getBlocking();
if (!h.valid)
return nullptr;
*color = h.color;
*bold = h.bold;
return strdup(h.text.c_str());
});
linenoiseSetFreeHintsCallback(free);
#endif
threadPool->addThread(reader);
threadPool->addThread(reader);
}
LineNoise::~LineNoise() {
threadPool.clear();
threadPool->stop();
}
Future<Optional<std::string>> LineNoise::read( std::string const& prompt ) {
auto r = new LineNoiseReader::Read(prompt);
auto f = r->result.getFuture();
threadPool->post(r);
return f;
Future<Optional<std::string>> LineNoise::read(std::string const& prompt) {
auto r = new LineNoiseReader::Read(prompt);
auto f = r->result.getFuture();
threadPool->post(r);
return f;
}
ACTOR Future<Void> waitKeyboardInterrupt(boost::asio::io_service* ios) {
state boost::asio::signal_set signals(*ios, SIGINT);
Promise<Void> result;
signals.async_wait([result](const boost::system::error_code& error, int signal_number) {
if (error) {
result.sendError(io_error());
} else {
result.send(Void());
}
});
state boost::asio::signal_set signals(*ios, SIGINT);
Promise<Void> result;
signals.async_wait([result](const boost::system::error_code& error, int signal_number) {
if (error) {
result.sendError(io_error());
} else {
result.send(Void());
}
});
wait(result.getFuture());
return Void();
wait(result.getFuture());
return Void();
}
Future<Void> LineNoise::onKeyboardInterrupt() {
boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService);
if (!ios) return Never();
return waitKeyboardInterrupt(ios);
boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService);
if (!ios)
return Never();
return waitKeyboardInterrupt(ios);
}
void LineNoise::historyAdd( std::string const& line ) {
#if HAVE_LINENOISE
linenoiseHistoryAdd( line.c_str() );
#endif
void LineNoise::historyAdd(std::string const& line) {
#if HAVE_LINENOISE
linenoiseHistoryAdd(line.c_str());
#endif
}
void LineNoise::historyLoad( std::string const& filename ) {
#if HAVE_LINENOISE
if(linenoiseHistoryLoad(filename.c_str()) != 0) {
throw io_error();
}
#endif
void LineNoise::historyLoad(std::string const& filename) {
#if HAVE_LINENOISE
if (linenoiseHistoryLoad(filename.c_str()) != 0) {
throw io_error();
}
#endif
}
void LineNoise::historySave( std::string const& filename ) {
#if HAVE_LINENOISE
if(linenoiseHistorySave(filename.c_str()) != 0) {
throw io_error();
}
#endif
void LineNoise::historySave(std::string const& filename) {
#if HAVE_LINENOISE
if (linenoiseHistorySave(filename.c_str()) != 0) {
throw io_error();
}
#endif
}

52
fdbcli/FlowLineNoise.h
View File

@ -26,39 +26,37 @@
#include <functional>
struct LineNoise : NonCopyable {
// Wraps the linenoise library so that it can be called from asynchronous Flow code
// Only create one of these at a time; the linenoise library only supports one history
//
// The current implementation does not support calling read concurrently with any other
// function (or itself).
// Wraps the linenoise library so that it can be called from asynchronous Flow code
// Only create one of these at a time; the linenoise library only supports one history
//
// The current implementation does not support calling read concurrently with any other
// function (or itself).
struct Hint {
std::string text;
int color;
bool bold;
bool valid;
Hint() : text(), color(), bold(), valid() {}
Hint( std::string const& text, int color, bool bold ) : text(text), color(color), bold(bold), valid(true) {}
};
struct Hint {
std::string text;
int color;
bool bold;
bool valid;
Hint() : text(), color(), bold(), valid() {}
Hint(std::string const& text, int color, bool bold) : text(text), color(color), bold(bold), valid(true) {}
};
LineNoise(
std::function< void(std::string const&, std::vector<std::string>&) > completion_callback,
std::function< Hint(std::string const&) > hint_callback,
int maxHistoryLines,
bool multiline
);
~LineNoise();
LineNoise(std::function<void(std::string const&, std::vector<std::string>&)> completion_callback,
std::function<Hint(std::string const&)> hint_callback,
int maxHistoryLines,
bool multiline);
~LineNoise();
Future< Optional<std::string> > read( std::string const& prompt ); // Returns "nothing" on EOF
void historyAdd( std::string const& line );
Future<Optional<std::string>> read(std::string const& prompt); // Returns "nothing" on EOF
void historyAdd(std::string const& line);
void historyLoad( std::string const& filename );
void historySave( std::string const& filename );
void historyLoad(std::string const& filename);
void historySave(std::string const& filename);
static Future<Void> onKeyboardInterrupt(); // Returns when Ctrl-C is next pressed (i.e. SIGINT)
static Future<Void> onKeyboardInterrupt(); // Returns when Ctrl-C is next pressed (i.e. SIGINT)
Reference<class IThreadPool> threadPool;
struct LineNoiseReader* reader;
Reference<class IThreadPool> threadPool;
struct LineNoiseReader* reader;
};
#endif

2376
fdbcli/fdbcli.actor.cpp
View File

File diff suppressed because it is too large Load Diff

1604
fdbcli/linenoise/linenoise.c
View File

File diff suppressed because it is too large Load Diff

28
fdbcli/linenoise/linenoise.h
View File

@ -46,24 +46,24 @@ extern "C" {
#endif
typedef struct linenoiseCompletions {
size_t len;
char **cvec;
size_t len;
char** cvec;
} linenoiseCompletions;
typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *);
typedef char*(linenoiseHintsCallback)(const char *, int *color, int *bold);
typedef void(linenoiseFreeHintsCallback)(void *);
void linenoiseSetCompletionCallback(linenoiseCompletionCallback *);
void linenoiseSetHintsCallback(linenoiseHintsCallback *);
void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *);
void linenoiseAddCompletion(linenoiseCompletions *, const char *);
typedef void(linenoiseCompletionCallback)(const char*, linenoiseCompletions*);
typedef char*(linenoiseHintsCallback)(const char*, int* color, int* bold);
typedef void(linenoiseFreeHintsCallback)(void*);
void linenoiseSetCompletionCallback(linenoiseCompletionCallback*);
void linenoiseSetHintsCallback(linenoiseHintsCallback*);
void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback*);
void linenoiseAddCompletion(linenoiseCompletions*, const char*);
char *linenoise(const char *prompt);
void linenoiseFree(void *ptr);
int linenoiseHistoryAdd(const char *line);
char* linenoise(const char* prompt);
void linenoiseFree(void* ptr);
int linenoiseHistoryAdd(const char* line);
int linenoiseHistorySetMaxLen(int len);
int linenoiseHistorySave(const char *filename);
int linenoiseHistoryLoad(const char *filename);
int linenoiseHistorySave(const char* filename);
int linenoiseHistoryLoad(const char* filename);
void linenoiseClearScreen(void);
void linenoiseSetMultiLine(int ml);
void linenoisePrintKeyCodes(void);

6
fdbclient/AsyncFileS3BlobStore.actor.cpp
View File

@ -24,7 +24,8 @@
#include "flow/actorcompiler.h" // has to be last include
Future<int64_t> AsyncFileS3BlobStoreRead::size() const {
if (!m_size.isValid()) m_size = m_bstore->objectSize(m_bucket, m_object);
if (!m_size.isValid())
m_size = m_bstore->objectSize(m_bucket, m_object);
return m_size;
}
@ -48,7 +49,8 @@ ACTOR Future<Void> sendStuff(int id, Reference<IRateControl> t, int bytes) {
TEST_CASE("/backup/throttling") {
// Test will not work in simulation.
if (g_network->isSimulated()) return Void();
if (g_network->isSimulated())
return Void();
state int limit = 100000;
state Reference<IRateControl> t(new SpeedLimit(limit, 1));

23
fdbclient/AsyncFileS3BlobStore.actor.h
View File

@ -46,7 +46,8 @@ static Future<T> joinErrorGroup(Future<T> f, Promise<Void> p) {
wait(success(f) || p.getFuture());
return f.get();
} catch (Error& e) {
if (p.canBeSet()) p.sendError(e);
if (p.canBeSet())
p.sendError(e);
throw;
}
}
@ -117,7 +118,8 @@ public:
}
Future<Void> write(void const* data, int length, int64_t offset) override {
if (offset != m_cursor) throw non_sequential_op();
if (offset != m_cursor)
throw non_sequential_op();
m_cursor += length;
return m_error.getFuture() ||
@ -125,15 +127,16 @@ public:
}
Future<Void> truncate(int64_t size) override {
if (size != m_cursor) return non_sequential_op();
if (size != m_cursor)
return non_sequential_op();
return Void();
}
ACTOR static Future<std::string> doPartUpload(AsyncFileS3BlobStoreWrite* f, Part* p) {
p->finalizeMD5();
std::string upload_id = wait(f->getUploadID());
std::string etag = wait(f->m_bstore->uploadPart(f->m_bucket, f->m_object, upload_id, p->number, &p->content,
p->length, p->md5string));
std::string etag = wait(f->m_bstore->uploadPart(
f->m_bucket, f->m_object, upload_id, p->number, &p->content, p->length, p->md5string));
return etag;
}
@ -142,8 +145,8 @@ public:
if (f->m_parts.size() == 1) {
Reference<Part> part = f->m_parts.back();
part->finalizeMD5();
wait(f->m_bstore->writeEntireFileFromBuffer(f->m_bucket, f->m_object, &part->content, part->length,
part->md5string));
wait(f->m_bstore->writeEntireFileFromBuffer(
f->m_bucket, f->m_object, &part->content, part->length, part->md5string));
return Void();
}
@ -221,7 +224,8 @@ private:
// End the current part and start uploading it, but also wait for a part to finish if too many are in transit.
ACTOR static Future<Void> endCurrentPart(AsyncFileS3BlobStoreWrite* f, bool startNew = false) {
if (f->m_parts.back()->length == 0) return Void();
if (f->m_parts.back()->length == 0)
return Void();
// Wait for an upload slot to be available
wait(f->m_concurrentUploads.take());
@ -241,7 +245,8 @@ private:
}
Future<std::string> getUploadID() {
if (!m_upload_id.isValid()) m_upload_id = m_bstore->beginMultiPartUpload(m_bucket, m_object);
if (!m_upload_id.isValid())
m_upload_id = m_bstore->beginMultiPartUpload(m_bucket, m_object);
return m_upload_id;
}

140
fdbclient/Atomic.h
View File

@ -24,21 +24,25 @@
#include "fdbclient/CommitTransaction.h"
inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand,
Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
int carry = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++) {
for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++) {
int sum = existingValue[i] + otherOperand[i] + carry;
buf[i] = sum;
carry = sum >> 8;
}
for (; i<otherOperand.size(); i++) {
for (; i < otherOperand.size(); i++) {
int sum = otherOperand[i] + carry;
buf[i] = sum;
carry = sum >> 8;
@ -49,14 +53,15 @@ inline ValueRef doLittleEndianAdd(const Optional<ValueRef>& existingValueOptiona
inline ValueRef doAnd(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!otherOperand.size()) return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++)
for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] & otherOperand[i];
for(; i<otherOperand.size(); i++)
for (; i < otherOperand.size(); i++)
buf[i] = 0x0;
return StringRef(buf, i);
@ -71,15 +76,17 @@ inline ValueRef doAndV2(const Optional<ValueRef>& existingValueOptional, const V
inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++)
for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] | otherOperand[i];
for(; i<otherOperand.size(); i++)
for (; i < otherOperand.size(); i++)
buf[i] = otherOperand[i];
return StringRef(buf, i);
@ -87,48 +94,56 @@ inline ValueRef doOr(const Optional<ValueRef>& existingValueOptional, const Valu
inline ValueRef doXor(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
int i = 0;
for(i = 0; i<std::min(existingValue.size(), otherOperand.size()); i++)
for (i = 0; i < std::min(existingValue.size(), otherOperand.size()); i++)
buf[i] = existingValue[i] ^ otherOperand[i];
for(; i<otherOperand.size(); i++)
for (; i < otherOperand.size(); i++)
buf[i] = otherOperand[i];
return StringRef(buf, i);
}
inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
inline ValueRef doAppendIfFits(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand,
Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if(!existingValue.size()) return otherOperand;
if(!otherOperand.size()) return existingValue;
if(existingValue.size() + otherOperand.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) {
TEST( true ) //AppendIfFIts resulted in truncation
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return existingValue;
if (existingValue.size() + otherOperand.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) {
TEST(true) // AppendIfFIts resulted in truncation
return existingValue;
}
uint8_t* buf = new (ar) uint8_t [existingValue.size() + otherOperand.size()];
int i,j;
uint8_t* buf = new (ar) uint8_t[existingValue.size() + otherOperand.size()];
int i, j;
for(i = 0; i<existingValue.size(); i++)
for (i = 0; i < existingValue.size(); i++)
buf[i] = existingValue[i];
for(j = 0; j<otherOperand.size(); j++)
buf[i+j] = otherOperand[j];
for (j = 0; j < otherOperand.size(); j++)
buf[i + j] = otherOperand[j];
return StringRef(buf, i+j);
return StringRef(buf, i + j);
}
inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
if (!existingValue.size()) return otherOperand;
if (!otherOperand.size()) return otherOperand;
if (!existingValue.size())
return otherOperand;
if (!otherOperand.size())
return otherOperand;
int i,j;
int i, j;
for (i = otherOperand.size() - 1; i >= existingValue.size(); i--) {
if (otherOperand[i] != 0) {
@ -139,9 +154,8 @@ inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const Val
for (; i >= 0; i--) {
if (otherOperand[i] > existingValue[i]) {
return otherOperand;
}
else if (otherOperand[i] < existingValue[i]) {
uint8_t* buf = new (ar) uint8_t [otherOperand.size()];
} else if (otherOperand[i] < existingValue[i]) {
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) {
buf[j] = existingValue[j];
}
@ -156,7 +170,8 @@ inline ValueRef doMax(const Optional<ValueRef>& existingValueOptional, const Val
}
inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present()) return otherOperand;
if (!existingValueOptional.present())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.get();
if (existingValue > otherOperand)
@ -166,14 +181,15 @@ inline ValueRef doByteMax(const Optional<ValueRef>& existingValueOptional, const
}
inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!otherOperand.size()) return otherOperand;
if (!otherOperand.size())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef();
int i,j;
int i, j;
for (i = otherOperand.size() - 1; i >= existingValue.size(); i--) {
if (otherOperand[i] != 0) {
uint8_t* buf = new (ar)uint8_t[otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) {
buf[j] = existingValue[j];
}
@ -186,7 +202,7 @@ inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const Val
for (; i >= 0; i--) {
if (otherOperand[i] > existingValue[i]) {
uint8_t* buf = new (ar)uint8_t[otherOperand.size()];
uint8_t* buf = new (ar) uint8_t[otherOperand.size()];
for (j = 0; j < std::min(existingValue.size(), otherOperand.size()); j++) {
buf[j] = existingValue[j];
}
@ -194,8 +210,7 @@ inline ValueRef doMin(const Optional<ValueRef>& existingValueOptional, const Val
buf[j] = 0x0;
}
return StringRef(buf, j);
}
else if (otherOperand[i] < existingValue[i]) {
} else if (otherOperand[i] < existingValue[i]) {
return otherOperand;
}
}
@ -211,7 +226,8 @@ inline ValueRef doMinV2(const Optional<ValueRef>& existingValueOptional, const V
}
inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const ValueRef& otherOperand, Arena& ar) {
if (!existingValueOptional.present()) return otherOperand;
if (!existingValueOptional.present())
return otherOperand;
const ValueRef& existingValue = existingValueOptional.get();
if (existingValue < otherOperand)
@ -221,7 +237,8 @@ inline ValueRef doByteMin(const Optional<ValueRef>& existingValueOptional, const
}
inline Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingValueOptional,
const ValueRef& otherOperand, Arena& ar) {
const ValueRef& otherOperand,
Arena& ar) {
if (!existingValueOptional.present() || existingValueOptional.get() == otherOperand) {
// Clear the value.
return Optional<ValueRef>();
@ -229,19 +246,19 @@ inline Optional<ValueRef> doCompareAndClear(const Optional<ValueRef>& existingVa
return existingValueOptional; // No change required.
}
static void placeVersionstamp( uint8_t* destination, Version version, uint16_t transactionNumber ) {
static void placeVersionstamp(uint8_t* destination, Version version, uint16_t transactionNumber) {
version = bigEndian64(version);
transactionNumber = bigEndian16(transactionNumber);
static_assert( sizeof(version) == 8, "version size mismatch" );
memcpy( destination, &version, sizeof(version) );
static_assert( sizeof(transactionNumber) == 2, "txn num size mismatch");
memcpy( destination + sizeof(version), &transactionNumber, sizeof(transactionNumber) );
static_assert(sizeof(version) == 8, "version size mismatch");
memcpy(destination, &version, sizeof(version));
static_assert(sizeof(transactionNumber) == 2, "txn num size mismatch");
memcpy(destination + sizeof(version), &transactionNumber, sizeof(transactionNumber));
}
/*
* Returns the range corresponding to the specified versionstamp key.
*/
inline KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef &key, Version minVersion, const KeyRef &maxKey) {
* Returns the range corresponding to the specified versionstamp key.
*/
inline KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef& key, Version minVersion, const KeyRef& maxKey) {
KeyRef begin(arena, key);
KeyRef end(arena, key);
@ -253,18 +270,18 @@ inline KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef &key, Vers
pos = littleEndian32(pos);
begin = begin.substr(0, begin.size() - 4);
end = end.substr(0, end.size() - 3);
mutateString(end)[end.size()-1] = 0;
mutateString(end)[end.size() - 1] = 0;
if (pos < 0 || pos + 10 > begin.size())
throw client_invalid_operation();
placeVersionstamp(mutateString(begin)+pos, minVersion, 0);
placeVersionstamp(mutateString(begin) + pos, minVersion, 0);
memset(mutateString(end) + pos, '\xff', 10);
return KeyRangeRef(begin, std::min(end, maxKey));
}
inline void transformVersionstampKey( StringRef& key, Version version, uint16_t transactionNumber ) {
inline void transformVersionstampKey(StringRef& key, Version version, uint16_t transactionNumber) {
if (key.size() < 4)
throw client_invalid_operation();
@ -274,10 +291,13 @@ inline void transformVersionstampKey( StringRef& key, Version version, uint16_t
if (pos < 0 || pos + 10 > key.size())
throw client_invalid_operation();
placeVersionstamp( mutateString(key) + pos, version, transactionNumber );
placeVersionstamp(mutateString(key) + pos, version, transactionNumber);
}
inline void transformVersionstampMutation( MutationRef& mutation, StringRef MutationRef::* param, Version version, uint16_t transactionNumber ) {
inline void transformVersionstampMutation(MutationRef& mutation,
StringRef MutationRef::*param,
Version version,
uint16_t transactionNumber) {
if ((mutation.*param).size() >= 4) {
int32_t pos;
memcpy(&pos, (mutation.*param).end() - sizeof(int32_t), sizeof(int32_t));
@ -285,7 +305,7 @@ inline void transformVersionstampMutation( MutationRef& mutation, StringRef Muta
mutation.*param = (mutation.*param).substr(0, (mutation.*param).size() - 4);
if (pos >= 0 && pos + 10 <= (mutation.*param).size()) {
placeVersionstamp( mutateString(mutation.*param) + pos, version, transactionNumber );
placeVersionstamp(mutateString(mutation.*param) + pos, version, transactionNumber);
}
}

3
fdbclient/AutoPublicAddress.cpp
View File

@ -46,8 +46,7 @@ IPAddress determinePublicIPAutomatically(ClusterConnectionString const& ccs) {
socket.close();
return ip;
}
catch(boost::system::system_error e) {
} catch (boost::system::system_error e) {
fprintf(stderr, "Error determining public address: %s\n", e.what());
throw bind_failed();
}

478
fdbclient/BackupAgent.actor.h
View File

@ -20,10 +20,10 @@
#pragma once
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_BACKUP_AGENT_ACTOR_G_H)
#define FDBCLIENT_BACKUP_AGENT_ACTOR_G_H
#include "fdbclient/BackupAgent.actor.g.h"
#define FDBCLIENT_BACKUP_AGENT_ACTOR_G_H
#include "fdbclient/BackupAgent.actor.g.h"
#elif !defined(FDBCLIENT_BACKUP_AGENT_ACTOR_H)
#define FDBCLIENT_BACKUP_AGENT_ACTOR_H
#define FDBCLIENT_BACKUP_AGENT_ACTOR_H
#include "flow/flow.h"
#include "fdbclient/NativeAPI.actor.h"
@ -42,9 +42,7 @@ public:
static std::string formatTime(int64_t epochs);
static int64_t parseTime(std::string timestamp);
static std::string timeFormat() {
return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM";
}
static std::string timeFormat() { return "YYYY/MM/DD.HH:MI:SS[+/-]HHMM"; }
enum class EnumState {
STATE_ERRORED = 0,
@ -125,8 +123,7 @@ public:
static const char* getStateText(EnumState enState) {
const char* stateText;
switch (enState)
{
switch (enState) {
case EnumState::STATE_ERRORED:
stateText = "has errored";
break;
@ -163,8 +160,7 @@ public:
static const char* getStateName(EnumState enState) {
const char* s;
switch (enState)
{
switch (enState) {
case EnumState::STATE_ERRORED:
s = "Errored";
break;
@ -201,8 +197,7 @@ public:
static bool isRunnable(EnumState enState) {
bool isRunnable = false;
switch (enState)
{
switch (enState) {
case EnumState::STATE_SUBMITTED:
case EnumState::STATE_RUNNING:
case EnumState::STATE_RUNNING_DIFFERENTIAL:
@ -216,13 +211,9 @@ public:
return isRunnable;
}
static const KeyRef getDefaultTag() {
return StringRef(defaultTagName);
}
static const KeyRef getDefaultTag() { return StringRef(defaultTagName); }
static const std::string getDefaultTagName() {
return defaultTagName;
}
static const std::string getDefaultTagName() { return defaultTagName; }
// This is only used for automatic backup name generation
static Standalone<StringRef> getCurrentTime() {
@ -234,7 +225,7 @@ public:
strftime(buffer, 128, "%Y-%m-%d-%H-%M-%S", timeinfo);
std::string time(buffer);
return StringRef(time + format(".%06d", (int)(1e6*(t - curTime))));
return StringRef(time + format(".%06d", (int)(1e6 * (t - curTime))));
}
protected:
@ -252,16 +243,13 @@ public:
void operator=(FileBackupAgent&& r) noexcept {
subspace = std::move(r.subspace);
config = std::move(r.config);
lastRestorable = std::move(r.lastRestorable),
taskBucket = std::move(r.taskBucket);
lastRestorable = std::move(r.lastRestorable), taskBucket = std::move(r.taskBucket);
futureBucket = std::move(r.futureBucket);
}
KeyBackedProperty<Key> lastBackupTimestamp() {
return config.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> lastBackupTimestamp() { return config.pack(LiteralStringRef(__FUNCTION__)); }
Future<Void> run(Database cx, double *pollDelay, int maxConcurrentTasks) {
Future<Void> run(Database cx, double* pollDelay, int maxConcurrentTasks) {
return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
}
@ -273,34 +261,80 @@ public:
// parallel restore
Future<Void> parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true);
Future<Void> submitParallelRestore(Database cx, Key backupTag, Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key bcUrl, Version targetVersion, bool lockDB, UID randomUID, Key addPrefix,
Future<Void> submitParallelRestore(Database cx,
Key backupTag,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key bcUrl,
Version targetVersion,
bool lockDB,
UID randomUID,
Key addPrefix,
Key removePrefix);
Future<Void> atomicParallelRestore(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix,
Key removePrefix);
Future<Void> atomicParallelRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix, Key removePrefix);
// restore() will
// - make sure that url is readable and appears to be a complete backup
// - make sure the requested TargetVersion is valid
// - submit a restore on the given tagName
// - Optionally wait for the restore's completion. Will restore_error if restore fails or is aborted.
// restore() will return the targetVersion which will be either the valid version passed in or the max restorable version for the given url.
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url,
Standalone<VectorRef<KeyRangeRef>> ranges, bool waitForComplete = true,
Version targetVersion = -1, bool verbose = true, Key addPrefix = Key(),
Key removePrefix = Key(), bool lockDB = true, bool incrementalBackupOnly = false,
// restore() will return the targetVersion which will be either the valid version passed in or the max restorable
// version for the given url.
Future<Version> restore(Database cx,
Optional<Database> cxOrig,
Key tagName,
Key url,
Standalone<VectorRef<KeyRangeRef>> ranges,
bool waitForComplete = true,
Version targetVersion = -1,
bool verbose = true,
Key addPrefix = Key(),
Key removePrefix = Key(),
bool lockDB = true,
bool incrementalBackupOnly = false,
Version beginVersion = -1);
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, bool waitForComplete = true,
Version targetVersion = -1, bool verbose = true, KeyRange range = normalKeys,
Key addPrefix = Key(), Key removePrefix = Key(), bool lockDB = true,
bool incrementalBackupOnly = false, Version beginVersion = -1) {
Future<Version> restore(Database cx,
Optional<Database> cxOrig,
Key tagName,
Key url,
bool waitForComplete = true,
Version targetVersion = -1,
bool verbose = true,
KeyRange range = normalKeys,
Key addPrefix = Key(),
Key removePrefix = Key(),
bool lockDB = true,
bool incrementalBackupOnly = false,
Version beginVersion = -1) {
Standalone<VectorRef<KeyRangeRef>> rangeRef;
rangeRef.push_back_deep(rangeRef.arena(), range);
return restore(cx, cxOrig, tagName, url, rangeRef, waitForComplete, targetVersion, verbose, addPrefix,
removePrefix, lockDB, incrementalBackupOnly, beginVersion);
return restore(cx,
cxOrig,
tagName,
url,
rangeRef,
waitForComplete,
targetVersion,
verbose,
addPrefix,
removePrefix,
lockDB,
incrementalBackupOnly,
beginVersion);
}
Future<Version> atomicRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix = Key(), Key removePrefix = Key());
Future<Version> atomicRestore(Database cx, Key tagName, KeyRange range = normalKeys, Key addPrefix = Key(), Key removePrefix = Key()) {
Future<Version> atomicRestore(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> ranges,
Key addPrefix = Key(),
Key removePrefix = Key());
Future<Version> atomicRestore(Database cx,
Key tagName,
KeyRange range = normalKeys,
Key addPrefix = Key(),
Key removePrefix = Key()) {
Standalone<VectorRef<KeyRangeRef>> rangeRef;
rangeRef.push_back_deep(rangeRef.arena(), range);
return atomicRestore(cx, tagName, rangeRef, addPrefix, removePrefix);
@ -315,27 +349,44 @@ public:
// Get a string describing the status of a tag
Future<std::string> restoreStatus(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<std::string> restoreStatus(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return restoreStatus(tr, tagName); });
return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return restoreStatus(tr, tagName); });
}
/** BACKUP METHODS **/
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key outContainer, int snapshotIntervalSeconds,
std::string tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true, bool partitionedLog = false,
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr,
Key outContainer,
int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
bool partitionedLog = false,
bool incrementalBackupOnly = false);
Future<Void> submitBackup(Database cx, Key outContainer, int snapshotIntervalSeconds, std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true,
bool partitionedLog = false, bool incrementalBackupOnly = false) {
Future<Void> submitBackup(Database cx,
Key outContainer,
int snapshotIntervalSeconds,
std::string tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
bool partitionedLog = false,
bool incrementalBackupOnly = false) {
return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr) {
return submitBackup(tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone,
partitionedLog, incrementalBackupOnly);
return submitBackup(tr,
outContainer,
snapshotIntervalSeconds,
tagName,
backupRanges,
stopWhenDone,
partitionedLog,
incrementalBackupOnly);
});
}
Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> discontinueBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return discontinueBackup(tr, tagName); });
return runRYWTransaction(
cx, [=](Reference<ReadYourWritesTransaction> tr) { return discontinueBackup(tr, tagName); });
}
// Terminate an ongoing backup, without waiting for the backup to finish.
@ -347,19 +398,23 @@ public:
// logRangesRange and backupLogKeys will be cleared for this backup.
Future<Void> abortBackup(Reference<ReadYourWritesTransaction> tr, std::string tagName);
Future<Void> abortBackup(Database cx, std::string tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return abortBackup(tr, tagName); });
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return abortBackup(tr, tagName); });
}
Future<std::string> getStatus(Database cx, bool showErrors, std::string tagName);
Future<std::string> getStatusJSON(Database cx, std::string tagName);
Future<Version> getLastRestorable(Reference<ReadYourWritesTransaction> tr, Key tagName, bool snapshot = false);
Future<Optional<Version>> getLastRestorable(Reference<ReadYourWritesTransaction> tr, Key tagName,
bool snapshot = false);
void setLastRestorable(Reference<ReadYourWritesTransaction> tr, Key tagName, Version version);
// stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it
// will return when the backup directory is restorable.
Future<EnumState> waitBackup(Database cx, std::string tagName, bool stopWhenDone = true,
Reference<IBackupContainer>* pContainer = nullptr, UID* pUID = nullptr);
Future<EnumState> waitBackup(Database cx,
std::string tagName,
bool stopWhenDone = true,
Reference<IBackupContainer>* pContainer = nullptr,
UID* pUID = nullptr);
static const Key keyLastRestorable;
@ -383,8 +438,14 @@ public:
Reference<FutureBucket> futureBucket;
};
template<> inline Tuple Codec<FileBackupAgent::ERestoreState>::pack(FileBackupAgent::ERestoreState const &val) { return Tuple().append(val); }
template<> inline FileBackupAgent::ERestoreState Codec<FileBackupAgent::ERestoreState>::unpack(Tuple const &val) { return (FileBackupAgent::ERestoreState)val.getInt(0); }
template <>
inline Tuple Codec<FileBackupAgent::ERestoreState>::pack(FileBackupAgent::ERestoreState const& val) {
return Tuple().append(val);
}
template <>
inline FileBackupAgent::ERestoreState Codec<FileBackupAgent::ERestoreState>::unpack(Tuple const& val) {
return (FileBackupAgent::ERestoreState)val.getInt(0);
}
class DatabaseBackupAgent : public BackupAgentBase {
public:
@ -410,45 +471,74 @@ public:
sourceTagNames = std::move(r.sourceTagNames);
}
Future<Void> run(Database cx, double *pollDelay, int maxConcurrentTasks) {
Future<Void> run(Database cx, double* pollDelay, int maxConcurrentTasks) {
return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks);
}
Future<Void> atomicSwitchover(Database dest, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, Key addPrefix, Key removePrefix, bool forceAction=false);
Future<Void> atomicSwitchover(Database dest,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix,
Key removePrefix,
bool forceAction = false);
Future<Void> unlockBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> unlockBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return unlockBackup(tr, tagName); });
return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return unlockBackup(tr, tagName); });
}
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key addPrefix = StringRef(), Key removePrefix = StringRef(), bool lockDatabase = false, bool databasesInSync=false);
Future<Void> submitBackup(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> backupRanges, bool stopWhenDone = true, Key addPrefix = StringRef(), Key removePrefix = StringRef(), bool lockDatabase = false, bool databasesInSync=false) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return submitBackup(tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, databasesInSync); });
Future<Void> submitBackup(Reference<ReadYourWritesTransaction> tr,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
Key addPrefix = StringRef(),
Key removePrefix = StringRef(),
bool lockDatabase = false,
bool databasesInSync = false);
Future<Void> submitBackup(Database cx,
Key tagName,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
bool stopWhenDone = true,
Key addPrefix = StringRef(),
Key removePrefix = StringRef(),
bool lockDatabase = false,
bool databasesInSync = false) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
return submitBackup(
tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, databasesInSync);
});
}
Future<Void> discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName);
Future<Void> discontinueBackup(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return discontinueBackup(tr, tagName); });
return runRYWTransaction(
cx, [=](Reference<ReadYourWritesTransaction> tr) { return discontinueBackup(tr, tagName); });
}
Future<Void> abortBackup(Database cx, Key tagName, bool partial = false, bool abortOldBackup = false,
bool dstOnly = false, bool waitForDestUID = false);
Future<Void> abortBackup(Database cx,
Key tagName,
bool partial = false,
bool abortOldBackup = false,
bool dstOnly = false,
bool waitForDestUID = false);
Future<std::string> getStatus(Database cx, int errorLimit, Key tagName);
Future<EnumState> getStateValue(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
Future<EnumState> getStateValue(Database cx, UID logUid) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getStateValue(tr, logUid); });
return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return getStateValue(tr, logUid); });
}
Future<UID> getDestUid(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
Future<UID> getDestUid(Database cx, UID logUid) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getDestUid(tr, logUid); });
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return getDestUid(tr, logUid); });
}
Future<UID> getLogUid(Reference<ReadYourWritesTransaction> tr, Key tagName, bool snapshot = false);
Future<UID> getLogUid(Database cx, Key tagName) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr){ return getLogUid(tr, tagName); });
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return getLogUid(tr, tagName); });
}
Future<int64_t> getRangeBytesWritten(Reference<ReadYourWritesTransaction> tr, UID logUid, bool snapshot = false);
@ -495,7 +585,7 @@ struct RCGroup {
Version version;
uint64_t groupKey;
RCGroup() : version(-1), groupKey(ULLONG_MAX) {};
RCGroup() : version(-1), groupKey(ULLONG_MAX){};
template <class Ar>
void serialize(Ar& ar) {
@ -505,25 +595,48 @@ struct RCGroup {
bool copyParameter(Reference<Task> source, Reference<Task> dest, Key key);
Version getVersionFromString(std::string const& value);
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion,
Version endVersion,
Key destUidValue,
int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version endVersion, Key backupUid);
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion = Optional<Version>(), bool checkBackupUid = false, Version backupUid = 0);
Key getApplyKey( Version version, Key backupUid );
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion = Optional<Version>(),
bool checkBackupUid = false,
Version backupUid = 0);
Key getApplyKey(Version version, Key backupUid);
Version getLogKeyVersion(Key key);
std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key);
Future<Void> logError(Database cx, Key keyErrors, const std::string& message);
Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, const std::string& message);
Future<Void> checkVersion(Reference<ReadYourWritesTransaction> const& tr);
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersion> results, Reference<FlowLock> lock,
KeyRangeRef range, bool terminator = true, bool systemAccess = false,
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RangeResultWithVersion> results,
Reference<FlowLock> lock,
KeyRangeRef range,
bool terminator = true,
bool systemAccess = false,
bool lockAware = false);
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Future<Void> active,
Reference<FlowLock> lock, KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy, bool terminator = true,
bool systemAccess = false, bool lockAware = false);
ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key removePrefix, Version beginVersion,
Version* endVersion, RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion, Reference<KeyRangeMap<Version>> keyVersion);
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RCGroup> results,
Future<Void> active,
Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
bool terminator = true,
bool systemAccess = false,
bool lockAware = false);
ACTOR Future<Void> applyMutations(Database cx,
Key uid,
Key addPrefix,
Key removePrefix,
Version beginVersion,
Version* endVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Reference<KeyRangeMap<Version>> keyVersion);
ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData);
using EBackupState = BackupAgentBase::EnumState;
@ -568,9 +681,11 @@ typedef KeyBackedMap<std::string, UidAndAbortedFlagT> TagMap;
// Map of tagName to {UID, aborted_flag} located in the fileRestorePrefixRange keyspace.
class TagUidMap : public KeyBackedMap<std::string, UidAndAbortedFlagT> {
public:
TagUidMap(const StringRef & prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {}
TagUidMap(const StringRef& prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {}
ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap, Reference<ReadYourWritesTransaction> tr, bool snapshot);
ACTOR static Future<std::vector<KeyBackedTag>> getAll_impl(TagUidMap* tagsMap,
Reference<ReadYourWritesTransaction> tr,
bool snapshot);
Future<std::vector<KeyBackedTag>> getAll(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
return getAll_impl(this, tr, snapshot);
@ -587,24 +702,24 @@ static inline KeyBackedTag makeBackupTag(std::string tagName) {
return KeyBackedTag(tagName, fileBackupPrefixRange.begin);
}
static inline Future<std::vector<KeyBackedTag>> getAllRestoreTags(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
static inline Future<std::vector<KeyBackedTag>> getAllRestoreTags(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false) {
return TagUidMap(fileRestorePrefixRange.begin).getAll(tr, snapshot);
}
static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadYourWritesTransaction> tr, bool snapshot = false) {
static inline Future<std::vector<KeyBackedTag>> getAllBackupTags(Reference<ReadYourWritesTransaction> tr,
bool snapshot = false) {
return TagUidMap(fileBackupPrefixRange.begin).getAll(tr, snapshot);
}
class KeyBackedConfig {
public:
static struct {
static TaskParam<UID> uid() {return LiteralStringRef(__FUNCTION__); }
static TaskParam<UID> uid() { return LiteralStringRef(__FUNCTION__); }
} TaskParams;
KeyBackedConfig(StringRef prefix, UID uid = UID()) :
uid(uid),
prefix(prefix),
configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(prefix), uid)) {}
KeyBackedConfig(StringRef prefix, UID uid = UID())
: uid(uid), prefix(prefix), configSpace(uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(prefix), uid)) {}
KeyBackedConfig(StringRef prefix, Reference<Task> task) : KeyBackedConfig(prefix, TaskParams.uid().get(task)) {}
@ -616,31 +731,28 @@ public:
return Void();
}
// Set the validation condition for the task which is that the restore uid's tag's uid is the same as the restore uid.
// Get this uid's tag, then get the KEY for the tag's uid but don't read it. That becomes the validation key
// which TaskBucket will check, and its value must be this restore config's uid.
UID u = uid; // 'this' could be invalid in lambda
// Set the validation condition for the task which is that the restore uid's tag's uid is the same as the
// restore uid. Get this uid's tag, then get the KEY for the tag's uid but don't read it. That becomes the
// validation key which TaskBucket will check, and its value must be this restore config's uid.
UID u = uid; // 'this' could be invalid in lambda
Key p = prefix;
return map(tag().get(tr), [u,p,task](Optional<std::string> const &tag) -> Void {
if(!tag.present())
return map(tag().get(tr), [u, p, task](Optional<std::string> const& tag) -> Void {
if (!tag.present())
throw restore_error();
// Validation contition is that the uidPair key must be exactly {u, false}
TaskBucket::setValidationCondition(task, KeyBackedTag(tag.get(), p).key, Codec<UidAndAbortedFlagT>::pack({u, false}).pack());
TaskBucket::setValidationCondition(
task, KeyBackedTag(tag.get(), p).key, Codec<UidAndAbortedFlagT>::pack({ u, false }).pack());
return Void();
});
}
KeyBackedProperty<std::string> tag() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<std::string> tag() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
UID getUid() { return uid; }
Key getUidAsKey() { return BinaryWriter::toValue(uid, Unversioned()); }
void clear(Reference<ReadYourWritesTransaction> tr) {
tr->clear(configSpace.range());
}
void clear(Reference<ReadYourWritesTransaction> tr) { tr->clear(configSpace.range()); }
// lastError is a pair of error message and timestamp expressed as an int64_t
KeyBackedProperty<std::pair<std::string, Version>> lastError() {
@ -654,15 +766,15 @@ public:
// Updates the error per type map and the last error property
Future<Void> updateErrorInfo(Database cx, Error e, std::string message) {
// Avoid capture of this ptr
auto &copy = *this;
auto& copy = *this;
return runRYWTransaction(cx, [=] (Reference<ReadYourWritesTransaction> tr) mutable {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) mutable {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
return map(tr->getReadVersion(), [=] (Version v) mutable {
copy.lastError().set(tr, {message, v});
copy.lastErrorPerType().set(tr, e.code(), {message, v});
return map(tr->getReadVersion(), [=](Version v) mutable {
copy.lastError().set(tr, { message, v });
copy.lastErrorPerType().set(tr, e.code(), { message, v });
return Void();
});
});
@ -674,10 +786,12 @@ protected:
Subspace configSpace;
};
template<> inline Tuple Codec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const &bc) {
template <>
inline Tuple Codec<Reference<IBackupContainer>>::pack(Reference<IBackupContainer> const& bc) {
return Tuple().append(StringRef(bc->getURL()));
}
template<> inline Reference<IBackupContainer> Codec<Reference<IBackupContainer>>::unpack(Tuple const &val) {
template <>
inline Reference<IBackupContainer> Codec<Reference<IBackupContainer>>::unpack(Tuple const& val) {
return IBackupContainer::openContainer(val.getString(0).toString());
}
@ -695,7 +809,7 @@ public:
Tuple pack() const {
return Tuple().append(begin).append(version).append(StringRef(fileName)).append(fileSize);
}
static RangeSlice unpack(Tuple const &t) {
static RangeSlice unpack(Tuple const& t) {
RangeSlice r;
int i = 0;
r.begin = t.getString(i++);
@ -708,52 +822,34 @@ public:
// Map of range end boundaries to info about the backup file written for that range.
typedef KeyBackedMap<Key, RangeSlice> RangeFileMapT;
RangeFileMapT snapshotRangeFileMap() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
RangeFileMapT snapshotRangeFileMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Number of kv range files that were both committed to persistent storage AND inserted into
// the snapshotRangeFileMap. Note that since insertions could replace 1 or more existing
// map entries this is not necessarily the number of entries currently in the map.
// This value exists to help with sizing of kv range folders for BackupContainers that
// This value exists to help with sizing of kv range folders for BackupContainers that
// require it.
KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedBinaryValue<int64_t> snapshotRangeFileCount() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Coalesced set of ranges already dispatched for writing.
typedef KeyBackedMap<Key, bool> RangeDispatchMapT;
RangeDispatchMapT snapshotRangeDispatchMap() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
RangeDispatchMapT snapshotRangeDispatchMap() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Interval to use for determining the target end version for new snapshots
KeyBackedProperty<int64_t> snapshotIntervalSeconds() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<int64_t> snapshotIntervalSeconds() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// When the current snapshot began
KeyBackedProperty<Version> snapshotBeginVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> snapshotBeginVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// When the current snapshot is desired to end.
// This can be changed at runtime to speed up or slow down a snapshot
KeyBackedProperty<Version> snapshotTargetEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// When the current snapshot is desired to end.
// This can be changed at runtime to speed up or slow down a snapshot
KeyBackedProperty<Version> snapshotTargetEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<int64_t> snapshotBatchSize() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<int64_t> snapshotBatchSize() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Key> snapshotBatchFuture() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> snapshotBatchFuture() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> snapshotBatchDispatchDoneKey() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<int64_t> snapshotDispatchLastShardsBehind() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
@ -764,14 +860,15 @@ public:
}
Future<Void> initNewSnapshot(Reference<ReadYourWritesTransaction> tr, int64_t intervalSeconds = -1) {
BackupConfig &copy = *this; // Capture this by value instead of this ptr
BackupConfig& copy = *this; // Capture this by value instead of this ptr
Future<Version> beginVersion = tr->getReadVersion();
Future<int64_t> defaultInterval = 0;
if(intervalSeconds < 0)
if (intervalSeconds < 0)
defaultInterval = copy.snapshotIntervalSeconds().getOrThrow(tr);
// Make sure read version and possibly the snapshot interval value are ready, then clear/init the snapshot config members
// Make sure read version and possibly the snapshot interval value are ready, then clear/init the snapshot
// config members
return map(success(beginVersion) && success(defaultInterval), [=](Void) mutable {
copy.snapshotRangeFileMap().clear(tr);
copy.snapshotRangeDispatchMap().clear(tr);
@ -779,10 +876,10 @@ public:
copy.snapshotBatchFuture().clear(tr);
copy.snapshotBatchDispatchDoneKey().clear(tr);
if(intervalSeconds < 0)
if (intervalSeconds < 0)
intervalSeconds = defaultInterval.get();
Version endVersion = beginVersion.get() + intervalSeconds * CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
copy.snapshotBeginVersion().set(tr, beginVersion.get());
copy.snapshotTargetEndVersion().set(tr, endVersion);
copy.snapshotRangeFileCount().set(tr, 0);
@ -793,26 +890,18 @@ public:
});
}
KeyBackedBinaryValue<int64_t> rangeBytesWritten() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedBinaryValue<int64_t> rangeBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedBinaryValue<int64_t> logBytesWritten() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedBinaryValue<int64_t> logBytesWritten() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<EBackupState> stateEnum() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<EBackupState> stateEnum() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Reference<IBackupContainer>> backupContainer() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
// Set to true when all backup workers for saving mutation logs have been started.
KeyBackedProperty<bool> allWorkerStarted() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> allWorkerStarted() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Each backup worker adds its (epoch, tag.id) to this property.
KeyBackedProperty<std::vector<std::pair<int64_t, int64_t>>> startedBackupWorkers() {
@ -820,19 +909,13 @@ public:
}
// Set to true if backup worker is enabled.
KeyBackedProperty<bool> backupWorkerEnabled() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> backupWorkerEnabled() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Set to true if partitioned log is enabled (only useful if backup worker is also enabled).
KeyBackedProperty<bool> partitionedLogEnabled() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> partitionedLogEnabled() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Set to true if only requesting incremental backup without base snapshot.
KeyBackedProperty<bool> incrementalBackupOnly() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> incrementalBackupOnly() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Latest version for which all prior versions have saved by backup workers.
KeyBackedProperty<Version> latestBackupWorkerSavedVersion() {
@ -840,28 +923,18 @@ public:
}
// Stop differntial logging if already started or don't start after completing KV ranges
KeyBackedProperty<bool> stopWhenDone() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<bool> stopWhenDone() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// Latest version for which all prior versions have had their log copy tasks completed
KeyBackedProperty<Version> latestLogEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> latestLogEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// The end version of the last complete snapshot
KeyBackedProperty<Version> latestSnapshotEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> latestSnapshotEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
// The end version of the first complete snapshot
KeyBackedProperty<Version> firstSnapshotEndVersion() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Version> firstSnapshotEndVersion() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
KeyBackedProperty<Key> destUidValue() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<Key> destUidValue() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
Future<Optional<Version>> getLatestRestorableVersion(Reference<ReadYourWritesTransaction> tr) {
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
@ -892,24 +965,26 @@ public:
});
}
KeyBackedProperty<std::vector<KeyRange>> backupRanges() {
return configSpace.pack(LiteralStringRef(__FUNCTION__));
}
KeyBackedProperty<std::vector<KeyRange>> backupRanges() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); }
void startMutationLogs(Reference<ReadYourWritesTransaction> tr, KeyRangeRef backupRange, Key destUidValue) {
Key mutationLogsDestKey = destUidValue.withPrefix(backupLogKeys.begin);
tr->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef<UID>(destUidValue, Unversioned())), logRangesEncodeValue(backupRange.end, mutationLogsDestKey));
tr->set(logRangesEncodeKey(backupRange.begin, BinaryReader::fromStringRef<UID>(destUidValue, Unversioned())),
logRangesEncodeValue(backupRange.end, mutationLogsDestKey));
}
Future<Void> logError(Database cx, Error e, std::string details, void *taskInstance = nullptr) {
if(!uid.isValid()) {
Future<Void> logError(Database cx, Error e, std::string details, void* taskInstance = nullptr) {
if (!uid.isValid()) {
TraceEvent(SevError, "FileBackupErrorNoUID").error(e).detail("Description", details);
return Void();
}
TraceEvent t(SevWarn, "FileBackupError");
t.error(e).detail("BackupUID", uid).detail("Description", details).detail("TaskInstance", (uint64_t)taskInstance);
t.error(e)
.detail("BackupUID", uid)
.detail("Description", details)
.detail("TaskInstance", (uint64_t)taskInstance);
// key_not_found could happen
if(e.code() == error_code_key_not_found)
if (e.code() == error_code_key_not_found)
t.backtrace();
return updateErrorInfo(cx, e, details);
@ -925,10 +1000,12 @@ struct StringRefReader {
// Return a pointer to len bytes at the current read position and advance read pos
const uint8_t* consume(unsigned int len) {
if (rptr == end && len != 0) throw end_of_stream();
if (rptr == end && len != 0)
throw end_of_stream();
const uint8_t* p = rptr;
rptr += len;
if (rptr > end) throw failure_error;
if (rptr > end)
throw failure_error;
return p;
}
@ -954,19 +1031,22 @@ struct StringRefReader {
};
namespace fileBackup {
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file, int64_t offset,
ACTOR Future<Standalone<VectorRef<KeyValueRef>>> decodeRangeFileBlock(Reference<IAsyncFile> file,
int64_t offset,
int len);
// Return a block of contiguous padding bytes "\0xff" for backup files, growing if needed.
Value makePadding(int size);
}
} // namespace fileBackup
// For fast restore simulation test
// For testing addPrefix feature in fast restore.
// Transform db content in restoreRanges by removePrefix and then addPrefix.
// Assume: DB is locked
ACTOR Future<Void> transformRestoredDatabase(Database cx, Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix, Key removePrefix);
ACTOR Future<Void> transformRestoredDatabase(Database cx,
Standalone<VectorRef<KeyRangeRef>> backupRanges,
Key addPrefix,
Key removePrefix);
void simulateBlobFailure();

461
fdbclient/BackupAgentBase.actor.cpp
View File

@ -53,7 +53,7 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
return -1;
}
#else
if(strptime(timeOnly.c_str(), "%Y/%m/%d.%H:%M:%S", &out) == nullptr) {
if (strptime(timeOnly.c_str(), "%Y/%m/%d.%H:%M:%S", &out) == nullptr) {
return -1;
}
#endif
@ -61,25 +61,25 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
// Read timezone offset in +/-HHMM format then convert to seconds
int tzHH;
int tzMM;
if(sscanf(timestamp.substr(19, 5).c_str(), "%3d%2d", &tzHH, &tzMM) != 2) {
if (sscanf(timestamp.substr(19, 5).c_str(), "%3d%2d", &tzHH, &tzMM) != 2) {
return -1;
}
if(tzHH < 0) {
if (tzHH < 0) {
tzMM = -tzMM;
}
// tzOffset is the number of seconds EAST of GMT
int tzOffset = tzHH * 60 * 60 + tzMM * 60;
// The goal is to convert the timestamp string to epoch seconds assuming the date/time was expressed in the timezone at the end of the string.
// However, mktime() will ONLY return epoch seconds assuming the date/time is expressed in local time (based on locale / environment)
// mktime() will set out.tm_gmtoff when available
// The goal is to convert the timestamp string to epoch seconds assuming the date/time was expressed in the timezone
// at the end of the string. However, mktime() will ONLY return epoch seconds assuming the date/time is expressed in
// local time (based on locale / environment) mktime() will set out.tm_gmtoff when available
int64_t ts = mktime(&out);
// localTZOffset is the number of seconds EAST of GMT
long localTZOffset;
#ifdef _WIN32
// _get_timezone() returns the number of seconds WEST of GMT
if(_get_timezone(&localTZOffset) != 0) {
if (_get_timezone(&localTZOffset) != 0) {
return -1;
}
// Negate offset to match the orientation of tzOffset
@ -89,7 +89,8 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) {
localTZOffset = out.tm_gmtoff;
#endif
// Add back the difference between the local timezone assumed by mktime() and the intended timezone from the input string
// Add back the difference between the local timezone assumed by mktime() and the intended timezone from the input
// string
ts += (localTZOffset - tzOffset);
return ts;
}
@ -145,7 +146,10 @@ Version getVersionFromString(std::string const& value) {
// Return the ranges of keys that contain the data for the given range
// of versions.
// assert CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE % blocksize = 0. Otherwise calculation of hash will be incorrect
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize) {
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion,
Version endVersion,
Key destUidValue,
int blockSize) {
Standalone<VectorRef<KeyRangeRef>> ret;
Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin);
@ -161,8 +165,9 @@ Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version en
Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
ret.push_back_deep(ret.arena(), KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
ret.push_back_deep(ret.arena(),
KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
}
return ret;
@ -175,7 +180,9 @@ Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version
//TraceEvent("GetLogRanges").detail("BackupUid", backupUid).detail("Prefix", baLogRangePrefix);
for (int64_t vblock = beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE; vblock < (endVersion + CLIENT_KNOBS->APPLY_BLOCK_SIZE - 1) / CLIENT_KNOBS->APPLY_BLOCK_SIZE; ++vblock) {
for (int64_t vblock = beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE;
vblock < (endVersion + CLIENT_KNOBS->APPLY_BLOCK_SIZE - 1) / CLIENT_KNOBS->APPLY_BLOCK_SIZE;
++vblock) {
int64_t tb = vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
uint64_t bv = bigEndian64(std::max(beginVersion, vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE));
uint64_t ev = bigEndian64(std::min(endVersion, (vblock + 1) * CLIENT_KNOBS->APPLY_BLOCK_SIZE));
@ -184,15 +191,16 @@ Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version
Key vblockPrefix = StringRef(&hash, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
ret.push_back_deep(ret.arena(), KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
ret.push_back_deep(ret.arena(),
KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
}
return ret;
}
Key getApplyKey( Version version, Key backupUid ) {
int64_t vblock = (version-1) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
Key getApplyKey(Version version, Key backupUid) {
int64_t vblock = (version - 1) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
uint64_t v = bigEndian64(version);
uint32_t data = vblock & 0xffffffff;
uint8_t hash = (uint8_t)hashlittle(&data, sizeof(uint32_t), 0);
@ -205,24 +213,34 @@ Version getLogKeyVersion(Key key) {
return bigEndian64(*(int64_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t)));
}
//Given a key from one of the ranges returned by get_log_ranges,
//returns(version, part) where version is the database version number of
//the transaction log data in the value, and part is 0 for the first such
//data for a given version, 1 for the second block of data, etc.
// Given a key from one of the ranges returned by get_log_ranges,
// returns(version, part) where version is the database version number of
// the transaction log data in the value, and part is 0 for the first such
// data for a given version, 1 for the second block of data, etc.
std::pair<Version, uint32_t> decodeBKMutationLogKey(Key key) {
return std::make_pair(getLogKeyVersion(key),
bigEndian32(*(int32_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t) + sizeof(int64_t))));
return std::make_pair(
getLogKeyVersion(key),
bigEndian32(*(int32_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t) + sizeof(int64_t))));
}
void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mutationSize, StringRef value, StringRef addPrefix, StringRef removePrefix, Version version, Reference<KeyRangeMap<Version>> key_version) {
void decodeBackupLogValue(Arena& arena,
VectorRef<MutationRef>& result,
int& mutationSize,
StringRef value,
StringRef addPrefix,
StringRef removePrefix,
Version version,
Reference<KeyRangeMap<Version>> key_version) {
try {
uint64_t offset(0);
uint64_t protocolVersion = 0;
memcpy(&protocolVersion, value.begin(), sizeof(uint64_t));
offset += sizeof(uint64_t);
if (protocolVersion <= 0x0FDB00A200090001){
TraceEvent(SevError, "DecodeBackupLogValue").detail("IncompatibleProtocolVersion", protocolVersion)
.detail("ValueSize", value.size()).detail("Value", value);
if (protocolVersion <= 0x0FDB00A200090001) {
TraceEvent(SevError, "DecodeBackupLogValue")
.detail("IncompatibleProtocolVersion", protocolVersion)
.detail("ValueSize", value.size())
.detail("Value", value);
throw incompatible_protocol_version();
}
@ -231,12 +249,12 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
offset += sizeof(uint32_t);
uint32_t consumed = 0;
if(totalBytes + offset > value.size())
if (totalBytes + offset > value.size())
throw restore_missing_data();
int originalOffset = offset;
while (consumed < totalBytes){
while (consumed < totalBytes) {
uint32_t type = 0;
memcpy(&type, value.begin() + offset, sizeof(uint32_t));
offset += sizeof(uint32_t);
@ -247,7 +265,7 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
memcpy(&len2, value.begin() + offset, sizeof(uint32_t));
offset += sizeof(uint32_t);
ASSERT(offset+len1+len2<=value.size() && isValidMutationType(type));
ASSERT(offset + len1 + len2 <= value.size() && isValidMutationType(type));
MutationRef logValue;
Arena tempArena;
@ -265,24 +283,23 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
KeyRef minKey = std::min(r.range().end, range.end);
if (minKey == (removePrefix == StringRef() ? normalKeys.end : strinc(removePrefix))) {
logValue.param1 = std::max(r.range().begin, range.begin);
if(removePrefix.size()) {
if (removePrefix.size()) {
logValue.param1 = logValue.param1.removePrefix(removePrefix);
}
if(addPrefix.size()) {
if (addPrefix.size()) {
logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
}
logValue.param2 = addPrefix == StringRef() ? normalKeys.end : strinc(addPrefix, tempArena);
result.push_back_deep(arena, logValue);
mutationSize += logValue.expectedSize();
}
else {
} else {
logValue.param1 = std::max(r.range().begin, range.begin);
logValue.param2 = minKey;
if(removePrefix.size()) {
if (removePrefix.size()) {
logValue.param1 = logValue.param1.removePrefix(removePrefix);
logValue.param2 = logValue.param2.removePrefix(removePrefix);
}
if(addPrefix.size()) {
if (addPrefix.size()) {
logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
logValue.param2 = logValue.param2.withPrefix(addPrefix, tempArena);
}
@ -291,15 +308,14 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
}
}
}
}
else {
} else {
Version ver = key_version->rangeContaining(logValue.param1).value();
//TraceEvent("ApplyMutation").detail("LogValue", logValue.toString()).detail("Version", version).detail("Ver", ver).detail("Apply", version > ver && ver != invalidVersion);
if (version > ver && ver != invalidVersion) {
if(removePrefix.size()) {
if (removePrefix.size()) {
logValue.param1 = logValue.param1.removePrefix(removePrefix);
}
if(addPrefix.size()) {
if (addPrefix.size()) {
logValue.param1 = logValue.param1.withPrefix(addPrefix, tempArena);
}
result.push_back_deep(arena, logValue);
@ -312,12 +328,20 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
ASSERT(consumed == totalBytes);
if (value.size() != offset) {
TraceEvent(SevError, "BA_DecodeBackupLogValue").detail("UnexpectedExtraDataSize", value.size()).detail("Offset", offset).detail("TotalBytes", totalBytes).detail("Consumed", consumed).detail("OriginalOffset", originalOffset);
TraceEvent(SevError, "BA_DecodeBackupLogValue")
.detail("UnexpectedExtraDataSize", value.size())
.detail("Offset", offset)
.detail("TotalBytes", totalBytes)
.detail("Consumed", consumed)
.detail("OriginalOffset", originalOffset);
throw restore_corrupted_data();
}
}
catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_DecodeBackupLogValue").error(e).GetLastError().detail("ValueSize", value.size()).detail("Value", value);
} catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_DecodeBackupLogValue")
.error(e)
.GetLastError()
.detail("ValueSize", value.size())
.detail("Value", value);
throw;
}
}
@ -327,7 +351,7 @@ static double lastErrorTime = 0;
void logErrorWorker(Reference<ReadYourWritesTransaction> tr, Key keyErrors, std::string message) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
if(now() - lastErrorTime > CLIENT_KNOBS->BACKUP_ERROR_DELAY) {
if (now() - lastErrorTime > CLIENT_KNOBS->BACKUP_ERROR_DELAY) {
TraceEvent("BA_LogError").detail("Key", keyErrors).detail("Message", message);
lastErrorTime = now();
}
@ -335,8 +359,8 @@ void logErrorWorker(Reference<ReadYourWritesTransaction> tr, Key keyErrors, std:
}
Future<Void> logError(Database cx, Key keyErrors, const std::string& message) {
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
logErrorWorker(tr, keyErrors, message);
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) {
logErrorWorker(tr, keyErrors, message);
return Future<Void>(Void());
});
}
@ -345,14 +369,19 @@ Future<Void> logError(Reference<ReadYourWritesTransaction> tr, Key keyErrors, co
return logError(tr->getDatabase(), keyErrors, message);
}
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersion> results, Reference<FlowLock> lock,
KeyRangeRef range, bool terminator, bool systemAccess, bool lockAware) {
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RangeResultWithVersion> results,
Reference<FlowLock> lock,
KeyRangeRef range,
bool terminator,
bool systemAccess,
bool lockAware) {
state KeySelector begin = firstGreaterOrEqual(range.begin);
state KeySelector end = firstGreaterOrEqual(range.end);
state Transaction tr(cx);
state FlowLock::Releaser releaser;
loop{
loop {
try {
state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
(g_network->isSimulated() && !g_simulator.speedUpSimulation)
@ -364,23 +393,26 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
if (lockAware)
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
//add lock
// add lock
releaser.release();
wait(lock->take(TaskPriority::DefaultYield, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT));
releaser = FlowLock::Releaser(*lock, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
wait(lock->take(TaskPriority::DefaultYield,
limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT));
releaser = FlowLock::Releaser(
*lock, limits.bytes + CLIENT_KNOBS->VALUE_SIZE_LIMIT + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT);
state Standalone<RangeResultRef> values = wait(tr.getRange(begin, end, limits));
// When this buggify line is enabled, if there are more than 1 result then use half of the results
if(values.size() > 1 && BUGGIFY) {
if (values.size() > 1 && BUGGIFY) {
values.resize(values.arena(), values.size() / 2);
values.more = true;
// Half of the time wait for this tr to expire so that the next read is at a different version
if(deterministicRandom()->random01() < 0.5)
if (deterministicRandom()->random01() < 0.5)
wait(delay(6.0));
}
releaser.remaining -= values.expectedSize(); //its the responsibility of the caller to release after this point
releaser.remaining -=
values.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0);
results.send(RangeResultWithVersion(values, tr.getReadVersion().get()));
@ -389,28 +421,31 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
begin = firstGreaterThan(values.end()[-1].key);
if (!values.more && !limits.isReached()) {
if(terminator)
if (terminator)
results.sendError(end_of_stream());
return Void();
}
}
catch (Error &e) {
} catch (Error& e) {
if (e.code() == error_code_transaction_too_old) {
// We are using this transaction until it's too old and then resetting to a fresh one,
// so we don't need to delay.
tr.fullReset();
}
else {
} else {
wait(tr.onError(e));
}
}
}
}
ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Future<Void> active, Reference<FlowLock> lock,
KeyRangeRef range, std::function< std::pair<uint64_t, uint32_t>(Key key) > groupBy,
bool terminator, bool systemAccess, bool lockAware)
{
ACTOR Future<Void> readCommitted(Database cx,
PromiseStream<RCGroup> results,
Future<Void> active,
Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy,
bool terminator,
bool systemAccess,
bool lockAware) {
state KeySelector nextKey = firstGreaterOrEqual(range.begin);
state KeySelector end = firstGreaterOrEqual(range.end);
@ -419,7 +454,7 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
state Transaction tr(cx);
state FlowLock::Releaser releaser;
loop{
loop {
try {
state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
(g_network->isSimulated() && !g_simulator.speedUpSimulation)
@ -434,36 +469,37 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
state Standalone<RangeResultRef> rangevalue = wait(tr.getRange(nextKey, end, limits));
// When this buggify line is enabled, if there are more than 1 result then use half of the results
if(rangevalue.size() > 1 && BUGGIFY) {
if (rangevalue.size() > 1 && BUGGIFY) {
rangevalue.resize(rangevalue.arena(), rangevalue.size() / 2);
rangevalue.more = true;
// Half of the time wait for this tr to expire so that the next read is at a different version
if(deterministicRandom()->random01() < 0.5)
if (deterministicRandom()->random01() < 0.5)
wait(delay(6.0));
}
//add lock
// add lock
wait(active);
releaser.release();
wait(lock->take(TaskPriority::DefaultYield, rangevalue.expectedSize() + rcGroup.items.expectedSize()));
releaser = FlowLock::Releaser(*lock, rangevalue.expectedSize() + rcGroup.items.expectedSize());
for (auto & s : rangevalue){
for (auto& s : rangevalue) {
uint64_t groupKey = groupBy(s.key).first;
//TraceEvent("Log_ReadCommitted").detail("GroupKey", groupKey).detail("SkipGroup", skipGroup).detail("NextKey", nextKey.key).detail("End", end.key).detail("Valuesize", value.size()).detail("Index",index++).detail("Size",s.value.size());
if (groupKey != skipGroup){
if (rcGroup.version == -1){
if (groupKey != skipGroup) {
if (rcGroup.version == -1) {
rcGroup.version = tr.getReadVersion().get();
rcGroup.groupKey = groupKey;
}
else if (rcGroup.groupKey != groupKey) {
} else if (rcGroup.groupKey != groupKey) {
//TraceEvent("Log_ReadCommitted").detail("SendGroup0", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength",rcGroup.items[0].value.size());
//state uint32_t len(0);
//for (size_t j = 0; j < rcGroup.items.size(); ++j) {
// state uint32_t len(0);
// for (size_t j = 0; j < rcGroup.items.size(); ++j) {
// len += rcGroup.items[j].value.size();
//}
//TraceEvent("SendGroup").detail("GroupKey", rcGroup.groupKey).detail("Version", rcGroup.version).detail("Length", len).detail("Releaser.remaining", releaser.remaining);
releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point
releaser.remaining -=
rcGroup.items
.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0);
results.send(rcGroup);
nextKey = firstGreaterThan(rcGroup.items.end()[-1].key);
@ -478,39 +514,54 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
}
if (!rangevalue.more) {
if (rcGroup.version != -1){
releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point
if (rcGroup.version != -1) {
releaser.remaining -=
rcGroup.items
.expectedSize(); // its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0);
//TraceEvent("Log_ReadCommitted").detail("SendGroup1", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength", rcGroup.items[0].value.size());
results.send(rcGroup);
}
if(terminator)
if (terminator)
results.sendError(end_of_stream());
return Void();
}
nextKey = firstGreaterThan(rangevalue.end()[-1].key);
}
catch (Error &e) {
} catch (Error& e) {
if (e.code() == error_code_transaction_too_old) {
// We are using this transaction until it's too old and then resetting to a fresh one,
// so we don't need to delay.
tr.fullReset();
}
else {
} else {
wait(tr.onError(e));
}
}
}
}
Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Reference<FlowLock> lock, KeyRangeRef range, std::function< std::pair<uint64_t, uint32_t>(Key key) > groupBy) {
Future<Void> readCommitted(Database cx,
PromiseStream<RCGroup> results,
Reference<FlowLock> lock,
KeyRangeRef range,
std::function<std::pair<uint64_t, uint32_t>(Key key)> groupBy) {
return readCommitted(cx, results, Void(), lock, range, groupBy, true, true, true);
}
ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Reference<FlowLock> lock, Key uid, Key addPrefix, Key removePrefix, RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion, Optional<Version> endVersion, Key rangeBegin, PromiseStream<Future<Void>> addActor, FlowLock* commitLock, Reference<KeyRangeMap<Version>> keyVersion ) {
ACTOR Future<int> dumpData(Database cx,
PromiseStream<RCGroup> results,
Reference<FlowLock> lock,
Key uid,
Key addPrefix,
Key removePrefix,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Optional<Version> endVersion,
Key rangeBegin,
PromiseStream<Future<Void>> addActor,
FlowLock* commitLock,
Reference<KeyRangeMap<Version>> keyVersion) {
state Version lastVersion = invalidVersion;
state bool endOfStream = false;
state int totalBytes = 0;
@ -524,21 +575,27 @@ ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Referenc
lock->release(group.items.expectedSize());
BinaryWriter bw(Unversioned());
for(int i = 0; i < group.items.size(); ++i) {
for (int i = 0; i < group.items.size(); ++i) {
bw.serializeBytes(group.items[i].value);
}
decodeBackupLogValue(req.arena, req.transaction.mutations, mutationSize, bw.toValue(), addPrefix, removePrefix, group.groupKey, keyVersion);
decodeBackupLogValue(req.arena,
req.transaction.mutations,
mutationSize,
bw.toValue(),
addPrefix,
removePrefix,
group.groupKey,
keyVersion);
newBeginVersion = group.groupKey + 1;
if(mutationSize >= CLIENT_KNOBS->BACKUP_LOG_WRITE_BATCH_MAX_SIZE) {
if (mutationSize >= CLIENT_KNOBS->BACKUP_LOG_WRITE_BATCH_MAX_SIZE) {
break;
}
}
catch (Error &e) {
} catch (Error& e) {
if (e.code() == error_code_end_of_stream) {
if(endVersion.present() && endVersion.get() > lastVersion && endVersion.get() > newBeginVersion) {
if (endVersion.present() && endVersion.get() > lastVersion && endVersion.get() > newBeginVersion) {
newBeginVersion = endVersion.get();
}
if(newBeginVersion == invalidVersion)
if (newBeginVersion == invalidVersion)
return totalBytes;
endOfStream = true;
break;
@ -563,31 +620,39 @@ ACTOR Future<int> dumpData(Database cx, PromiseStream<RCGroup> results, Referenc
req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE;
totalBytes += mutationSize;
wait( commitLock->take(TaskPriority::DefaultYield, mutationSize) );
addActor.send( commitLock->releaseWhen( success(commit.getReply(req)), mutationSize ) );
wait(commitLock->take(TaskPriority::DefaultYield, mutationSize));
addActor.send(commitLock->releaseWhen(success(commit.getReply(req)), mutationSize));
if(endOfStream) {
if (endOfStream) {
return totalBytes;
}
}
}
ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Reference<KeyRangeMap<Version>> keyVersion, RequestStream<CommitTransactionRequest> commit, NotifiedVersion* committedVersion, PromiseStream<Future<Void>> addActor, FlowLock* commitLock) {
ACTOR Future<Void> coalesceKeyVersionCache(Key uid,
Version endVersion,
Reference<KeyRangeMap<Version>> keyVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
PromiseStream<Future<Void>> addActor,
FlowLock* commitLock) {
Version lastVersion = -1000;
int64_t removed = 0;
state CommitTransactionRequest req;
state int64_t mutationSize = 0;
Key mapPrefix = uid.withPrefix(applyMutationsKeyVersionMapRange.begin);
for(auto it : keyVersion->ranges()) {
if( lastVersion == -1000 ) {
for (auto it : keyVersion->ranges()) {
if (lastVersion == -1000) {
lastVersion = it.value();
} else {
Version ver = it.value();
if(ver < endVersion && lastVersion < endVersion && ver != invalidVersion && lastVersion != invalidVersion) {
if (ver < endVersion && lastVersion < endVersion && ver != invalidVersion &&
lastVersion != invalidVersion) {
Key removeKey = it.range().begin.withPrefix(mapPrefix);
Key removeEnd = keyAfter(removeKey);
req.transaction.mutations.push_back_deep(req.arena, MutationRef(MutationRef::ClearRange, removeKey, removeEnd));
req.transaction.mutations.push_back_deep(req.arena,
MutationRef(MutationRef::ClearRange, removeKey, removeEnd));
mutationSize += removeKey.size() + removeEnd.size();
removed--;
} else {
@ -596,40 +661,51 @@ ACTOR Future<Void> coalesceKeyVersionCache(Key uid, Version endVersion, Referenc
}
}
if(removed != 0) {
if (removed != 0) {
Key countKey = uid.withPrefix(applyMutationsKeyVersionCountRange.begin);
req.transaction.write_conflict_ranges.push_back_deep(req.arena, singleKeyRange(countKey));
req.transaction.mutations.push_back_deep(req.arena, MutationRef(MutationRef::AddValue, countKey, StringRef((uint8_t*)&removed, 8)));
req.transaction.mutations.push_back_deep(
req.arena, MutationRef(MutationRef::AddValue, countKey, StringRef((uint8_t*)&removed, 8)));
req.transaction.read_snapshot = committedVersion->get();
req.flags = req.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE;
wait( commitLock->take(TaskPriority::DefaultYield, mutationSize) );
addActor.send( commitLock->releaseWhen( success(commit.getReply(req)), mutationSize ) );
wait(commitLock->take(TaskPriority::DefaultYield, mutationSize));
addActor.send(commitLock->releaseWhen(success(commit.getReply(req)), mutationSize));
}
return Void();
}
ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key removePrefix, Version beginVersion, Version* endVersion, RequestStream<CommitTransactionRequest> commit, NotifiedVersion* committedVersion, Reference<KeyRangeMap<Version>> keyVersion ) {
ACTOR Future<Void> applyMutations(Database cx,
Key uid,
Key addPrefix,
Key removePrefix,
Version beginVersion,
Version* endVersion,
RequestStream<CommitTransactionRequest> commit,
NotifiedVersion* committedVersion,
Reference<KeyRangeMap<Version>> keyVersion) {
state FlowLock commitLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
state PromiseStream<Future<Void>> addActor;
state Future<Void> error = actorCollection( addActor.getFuture() );
state Future<Void> error = actorCollection(addActor.getFuture());
state int maxBytes = CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES;
keyVersion->insert(metadataVersionKey, 0);
try {
loop {
if(beginVersion >= *endVersion) {
wait( commitLock.take(TaskPriority::DefaultYield, CLIENT_KNOBS->BACKUP_LOCK_BYTES) );
if (beginVersion >= *endVersion) {
wait(commitLock.take(TaskPriority::DefaultYield, CLIENT_KNOBS->BACKUP_LOCK_BYTES));
commitLock.release(CLIENT_KNOBS->BACKUP_LOCK_BYTES);
if(beginVersion >= *endVersion) {
if (beginVersion >= *endVersion) {
return Void();
}
}
int rangeCount = std::max(1, CLIENT_KNOBS->APPLY_MAX_LOCK_BYTES / maxBytes);
state Version newEndVersion = std::min(*endVersion, ((beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE) + rangeCount) * CLIENT_KNOBS->APPLY_BLOCK_SIZE);
state Version newEndVersion = std::min(*endVersion,
((beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE) + rangeCount) *
CLIENT_KNOBS->APPLY_BLOCK_SIZE);
state Standalone<VectorRef<KeyRangeRef>> ranges = getApplyRanges(beginVersion, newEndVersion, uid);
state size_t idx;
state std::vector<PromiseStream<RCGroup>> results;
@ -643,23 +719,43 @@ ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key remov
rc.push_back(readCommitted(cx, results[i], locks[i], ranges[i], decodeBKMutationLogKey));
}
maxBytes = std::max<int>(maxBytes*CLIENT_KNOBS->APPLY_MAX_DECAY_RATE, CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES);
maxBytes = std::max<int>(maxBytes * CLIENT_KNOBS->APPLY_MAX_DECAY_RATE, CLIENT_KNOBS->APPLY_MIN_LOCK_BYTES);
for (idx = 0; idx < ranges.size(); ++idx) {
int bytes = wait(dumpData(cx, results[idx], locks[idx], uid, addPrefix, removePrefix, commit, committedVersion, idx==ranges.size()-1 ? newEndVersion : Optional<Version>(), ranges[idx].begin, addActor, &commitLock, keyVersion));
maxBytes = std::max<int>(CLIENT_KNOBS->APPLY_MAX_INCREASE_FACTOR*bytes, maxBytes);
if(error.isError()) throw error.getError();
int bytes = wait(dumpData(cx,
results[idx],
locks[idx],
uid,
addPrefix,
removePrefix,
commit,
committedVersion,
idx == ranges.size() - 1 ? newEndVersion : Optional<Version>(),
ranges[idx].begin,
addActor,
&commitLock,
keyVersion));
maxBytes = std::max<int>(CLIENT_KNOBS->APPLY_MAX_INCREASE_FACTOR * bytes, maxBytes);
if (error.isError())
throw error.getError();
}
wait(coalesceKeyVersionCache(uid, newEndVersion, keyVersion, commit, committedVersion, addActor, &commitLock));
wait(coalesceKeyVersionCache(
uid, newEndVersion, keyVersion, commit, committedVersion, addActor, &commitLock));
beginVersion = newEndVersion;
}
} catch( Error &e ) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarnAlways : SevError, "ApplyMutationsError").error(e);
throw;
} catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarnAlways : SevError, "ApplyMutationsError")
.error(e);
throw;
}
}
ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) {
ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion,
bool checkBackupUid,
Version backupUid) {
state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
state Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath);
@ -671,13 +767,15 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (checkBackupUid) {
Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
Optional<Value> v = wait( tr->get( sourceStates.pack(DatabaseBackupAgent::keyFolderId) ) );
if(v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) > backupUid)
Subspace sourceStates =
Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
Optional<Value> v = wait(tr->get(sourceStates.pack(DatabaseBackupAgent::keyFolderId)));
if (v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) > backupUid)
return Void();
}
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
state Standalone<RangeResultRef> backupVersions = wait(
tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
// Make sure version history key does exist and lower the beginVersion if needed
state Version currBeginVersion = invalidVersion;
@ -696,7 +794,7 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
}
state Version currEndVersion = std::numeric_limits<Version>::max();
if(endVersion.present()) {
if (endVersion.present()) {
currEndVersion = std::min(currEndVersion, endVersion.get());
}
@ -725,7 +823,7 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
if (!endVersion.present()) {
// Clear current backup version history
tr->clear(backupLatestVersionsKey);
if(backupVersions.size() == 1) {
if (backupVersions.size() == 1) {
tr->clear(prefixRange(destUidValue.withPrefix(logRangesRange.begin)));
}
} else {
@ -735,19 +833,22 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
// Clear log ranges if needed
if (clearLogRangesRequired) {
if((nextSmallestVersion - currBeginVersion) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE >= std::numeric_limits<uint8_t>::max() || BUGGIFY) {
if ((nextSmallestVersion - currBeginVersion) / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE >=
std::numeric_limits<uint8_t>::max() ||
BUGGIFY) {
Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin);
for(int h = 0; h <= std::numeric_limits<uint8_t>::max(); h++) {
for (int h = 0; h <= std::numeric_limits<uint8_t>::max(); h++) {
uint64_t bv = bigEndian64(Version(0));
uint64_t ev = bigEndian64(nextSmallestVersion);
uint8_t h1 = h;
Key vblockPrefix = StringRef(&h1, sizeof(uint8_t)).withPrefix(baLogRangePrefix);
tr->clear(KeyRangeRef(StringRef((uint8_t*)&bv, sizeof(uint64_t)).withPrefix(vblockPrefix),
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
StringRef((uint8_t*)&ev, sizeof(uint64_t)).withPrefix(vblockPrefix)));
}
} else {
Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(currBeginVersion, nextSmallestVersion, destUidValue);
Standalone<VectorRef<KeyRangeRef>> ranges =
getLogRanges(currBeginVersion, nextSmallestVersion, destUidValue);
for (auto& range : ranges) {
tr->clear(range);
}
@ -763,11 +864,12 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
// Disable committing mutations into blog
tr->clear(prefixRange(destUidValue.withPrefix(logRangesRange.begin)));
}
if(!endVersion.present() && backupVersions.size() == 1) {
Standalone<RangeResultRef> existingDestUidValues = wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
for(auto it : existingDestUidValues) {
if( it.value == destUidValue ) {
if (!endVersion.present() && backupVersions.size() == 1) {
Standalone<RangeResultRef> existingDestUidValues =
wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
for (auto it : existingDestUidValues) {
if (it.value == destUidValue) {
tr->clear(it.key);
}
}
@ -776,7 +878,12 @@ ACTOR static Future<Void> _eraseLogData(Reference<ReadYourWritesTransaction> tr,
return Void();
}
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr, Key logUidValue, Key destUidValue, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) {
Future<Void> eraseLogData(Reference<ReadYourWritesTransaction> tr,
Key logUidValue,
Key destUidValue,
Optional<Version> endVersion,
bool checkBackupUid,
Version backupUid) {
return _eraseLogData(tr, logUidValue, destUidValue, endVersion, checkBackupUid, backupUid);
}
@ -792,58 +899,81 @@ ACTOR Future<Void> cleanupLogMutations(Database cx, Value destUidValue, bool del
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(
KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
state Version readVer = tr->getReadVersion().get();
state Version minVersion = std::numeric_limits<Version>::max();
state Key minVersionLogUid;
state int backupIdx = 0;
for (; backupIdx < backupVersions.size(); backupIdx++) {
state Version currVersion = BinaryReader::fromStringRef<Version>(backupVersions[backupIdx].value, Unversioned());
state Key currLogUid = backupVersions[backupIdx].key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
if( currVersion < minVersion ) {
state Version currVersion =
BinaryReader::fromStringRef<Version>(backupVersions[backupIdx].value, Unversioned());
state Key currLogUid =
backupVersions[backupIdx].key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
if (currVersion < minVersion) {
minVersionLogUid = currLogUid;
minVersion = currVersion;
}
if(!loggedLogUids.count(currLogUid)) {
state Future<Optional<Value>> foundDRKey = tr->get(Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(currLogUid).pack(DatabaseBackupAgent::keyStateStatus));
state Future<Optional<Value>> foundBackupKey = tr->get(Subspace(currLogUid.withPrefix(LiteralStringRef("uid->config/")).withPrefix(fileBackupPrefixRange.begin)).pack(LiteralStringRef("stateEnum")));
if (!loggedLogUids.count(currLogUid)) {
state Future<Optional<Value>> foundDRKey = tr->get(Subspace(databaseBackupPrefixRange.begin)
.get(BackupAgentBase::keySourceStates)
.get(currLogUid)
.pack(DatabaseBackupAgent::keyStateStatus));
state Future<Optional<Value>> foundBackupKey =
tr->get(Subspace(currLogUid.withPrefix(LiteralStringRef("uid->config/"))
.withPrefix(fileBackupPrefixRange.begin))
.pack(LiteralStringRef("stateEnum")));
wait(success(foundDRKey) && success(foundBackupKey));
if(foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("WARNING: Found a tag that looks like both a backup and a DR. This tag is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if(foundDRKey.get().present() && !foundBackupKey.get().present()) {
printf("Found a DR that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if(!foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("Found a Backup that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
if (foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("WARNING: Found a tag that looks like both a backup and a DR. This tag is %.4f hours "
"behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (foundDRKey.get().present() && !foundBackupKey.get().present()) {
printf("Found a DR that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (!foundDRKey.get().present() && foundBackupKey.get().present()) {
printf("Found a Backup that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else {
printf("WARNING: Found an unknown tag that is %.4f hours behind.\n", (readVer - currVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
printf("WARNING: Found an unknown tag that is %.4f hours behind.\n",
(readVer - currVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
}
loggedLogUids.insert(currLogUid);
}
}
if(deleteData) {
if(readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS*CLIENT_KNOBS->CORE_VERSIONSPERSECOND && (!removingLogUid.present() || minVersionLogUid == removingLogUid.get())) {
if (deleteData) {
if (readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS * CLIENT_KNOBS->CORE_VERSIONSPERSECOND &&
(!removingLogUid.present() || minVersionLogUid == removingLogUid.get())) {
removingLogUid = minVersionLogUid;
wait(eraseLogData(tr, minVersionLogUid, destUidValue));
wait(tr->commit());
printf("\nSuccessfully removed the tag that was %.4f hours behind.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if(removingLogUid.present() && minVersionLogUid != removingLogUid.get()) {
printf("\nWARNING: The oldest tag was possibly removed, run again without `--delete_data' to check.\n\n");
printf("\nSuccessfully removed the tag that was %.4f hours behind.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (removingLogUid.present() && minVersionLogUid != removingLogUid.get()) {
printf("\nWARNING: The oldest tag was possibly removed, run again without `--delete_data' to "
"check.\n\n");
} else {
printf("\nWARNING: Did not delete data because the tag is not at least %.4f hours behind. Change `--min_cleanup_seconds' to adjust this threshold.\n\n", CLIENT_KNOBS->MIN_CLEANUP_SECONDS/3600.0);
printf("\nWARNING: Did not delete data because the tag is not at least %.4f hours behind. Change "
"`--min_cleanup_seconds' to adjust this threshold.\n\n",
CLIENT_KNOBS->MIN_CLEANUP_SECONDS / 3600.0);
}
} else if(readVer - minVersion > CLIENT_KNOBS->MIN_CLEANUP_SECONDS*CLIENT_KNOBS->CORE_VERSIONSPERSECOND) {
printf("\nPassing `--delete_data' would delete the tag that is %.4f hours behind.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else if (readVer - minVersion >
CLIENT_KNOBS->MIN_CLEANUP_SECONDS * CLIENT_KNOBS->CORE_VERSIONSPERSECOND) {
printf("\nPassing `--delete_data' would delete the tag that is %.4f hours behind.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
} else {
printf("\nPassing `--delete_data' would not delete the tag that is %.4f hours behind. Change `--min_cleanup_seconds' to adjust the cleanup threshold.\n\n", (readVer - minVersion)/(3600.0*CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
printf("\nPassing `--delete_data' would not delete the tag that is %.4f hours behind. Change "
"`--min_cleanup_seconds' to adjust the cleanup threshold.\n\n",
(readVer - minVersion) / (3600.0 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
}
return Void();
} catch( Error& e) {
} catch (Error& e) {
wait(tr->onError(e));
}
}
@ -856,13 +986,14 @@ ACTOR Future<Void> cleanupBackup(Database cx, bool deleteData) {
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
state Standalone<RangeResultRef> destUids = wait(tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
state Standalone<RangeResultRef> destUids = wait(
tr->getRange(KeyRangeRef(destUidLookupPrefix, strinc(destUidLookupPrefix)), CLIENT_KNOBS->TOO_MANY));
for(auto destUid : destUids) {
for (auto destUid : destUids) {
wait(cleanupLogMutations(cx, destUid.value, deleteData));
}
return Void();
} catch( Error& e) {
} catch (Error& e) {
wait(tr->onError(e));
}
}

119
fdbclient/BackupContainer.actor.cpp
View File

@ -58,7 +58,6 @@ ACTOR Future<Void> appendStringRefWithLen(Reference<IBackupFile> file, Standalon
wait(file->append(s.begin(), s.size()));
return Void();
}
} // namespace IBackupFile_impl
Future<Void> IBackupFile::appendStringRefWithLen(Standalone<StringRef> s) {
@ -67,31 +66,31 @@ Future<Void> IBackupFile::appendStringRefWithLen(Standalone<StringRef> s) {
std::string IBackupContainer::ExpireProgress::toString() const {
std::string s = step + "...";
if(total > 0) {
if (total > 0) {
s += format("%d/%d (%.2f%%)", done, total, double(done) / total * 100);
}
return s;
}
void BackupFileList::toStream(FILE *fout) const {
for(const RangeFile &f : ranges) {
void BackupFileList::toStream(FILE* fout) const {
for (const RangeFile& f : ranges) {
fprintf(fout, "range %" PRId64 " %s\n", f.fileSize, f.fileName.c_str());
}
for(const LogFile &f : logs) {
for (const LogFile& f : logs) {
fprintf(fout, "log %" PRId64 " %s\n", f.fileSize, f.fileName.c_str());
}
for(const KeyspaceSnapshotFile &f : snapshots) {
for (const KeyspaceSnapshotFile& f : snapshots) {
fprintf(fout, "snapshotManifest %" PRId64 " %s\n", f.totalSize, f.fileName.c_str());
}
}
Future<Void> fetchTimes(Reference<ReadYourWritesTransaction> tr, std::map<Version, int64_t> *pVersionTimeMap) {
Future<Void> fetchTimes(Reference<ReadYourWritesTransaction> tr, std::map<Version, int64_t>* pVersionTimeMap) {
std::vector<Future<Void>> futures;
// Resolve each version in the map,
for(auto &p : *pVersionTimeMap) {
for (auto& p : *pVersionTimeMap) {
futures.push_back(map(timeKeeperEpochsFromVersion(p.first, tr), [=](Optional<int64_t> t) {
if(t.present())
if (t.present())
pVersionTimeMap->at(p.first) = t.get();
else
pVersionTimeMap->erase(p.first);
@ -106,22 +105,23 @@ Future<Void> BackupDescription::resolveVersionTimes(Database cx) {
// Populate map with versions needed
versionTimeMap.clear();
for(const KeyspaceSnapshotFile &m : snapshots) {
for (const KeyspaceSnapshotFile& m : snapshots) {
versionTimeMap[m.beginVersion];
versionTimeMap[m.endVersion];
}
if(minLogBegin.present())
if (minLogBegin.present())
versionTimeMap[minLogBegin.get()];
if(maxLogEnd.present())
if (maxLogEnd.present())
versionTimeMap[maxLogEnd.get()];
if(contiguousLogEnd.present())
if (contiguousLogEnd.present())
versionTimeMap[contiguousLogEnd.get()];
if(minRestorableVersion.present())
if (minRestorableVersion.present())
versionTimeMap[minRestorableVersion.get()];
if(maxRestorableVersion.present())
if (maxRestorableVersion.present())
versionTimeMap[maxRestorableVersion.get()];
return runRYWTransaction(cx, [=](Reference<ReadYourWritesTransaction> tr) { return fetchTimes(tr, &versionTimeMap); });
return runRYWTransaction(cx,
[=](Reference<ReadYourWritesTransaction> tr) { return fetchTimes(tr, &versionTimeMap); });
};
std::string BackupDescription::toString() const {
@ -133,46 +133,49 @@ std::string BackupDescription::toString() const {
auto formatVersion = [&](Version v) {
std::string s;
if(!versionTimeMap.empty()) {
if (!versionTimeMap.empty()) {
auto i = versionTimeMap.find(v);
if(i != versionTimeMap.end())
if (i != versionTimeMap.end())
s = format("%lld (%s)", v, BackupAgentBase::formatTime(i->second).c_str());
else
s = format("%lld (unknown)", v);
}
else if(maxLogEnd.present()) {
} else if (maxLogEnd.present()) {
double days = double(maxLogEnd.get() - v) / (CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 60 * 60);
s = format("%lld (maxLogEnd %s%.2f days)", v, days < 0 ? "+" : "-", days);
}
else {
} else {
s = format("%lld", v);
}
return s;
};
for(const KeyspaceSnapshotFile &m : snapshots) {
info.append(format("Snapshot: startVersion=%s endVersion=%s totalBytes=%lld restorable=%s expiredPct=%.2f\n",
formatVersion(m.beginVersion).c_str(), formatVersion(m.endVersion).c_str(), m.totalSize, m.restorable.orDefault(false) ? "true" : "false", m.expiredPct(expiredEndVersion)));
for (const KeyspaceSnapshotFile& m : snapshots) {
info.append(
format("Snapshot: startVersion=%s endVersion=%s totalBytes=%lld restorable=%s expiredPct=%.2f\n",
formatVersion(m.beginVersion).c_str(),
formatVersion(m.endVersion).c_str(),
m.totalSize,
m.restorable.orDefault(false) ? "true" : "false",
m.expiredPct(expiredEndVersion)));
}
info.append(format("SnapshotBytes: %lld\n", snapshotBytes));
if(expiredEndVersion.present())
if (expiredEndVersion.present())
info.append(format("ExpiredEndVersion: %s\n", formatVersion(expiredEndVersion.get()).c_str()));
if(unreliableEndVersion.present())
if (unreliableEndVersion.present())
info.append(format("UnreliableEndVersion: %s\n", formatVersion(unreliableEndVersion.get()).c_str()));
if(minLogBegin.present())
if (minLogBegin.present())
info.append(format("MinLogBeginVersion: %s\n", formatVersion(minLogBegin.get()).c_str()));
if(contiguousLogEnd.present())
if (contiguousLogEnd.present())
info.append(format("ContiguousLogEndVersion: %s\n", formatVersion(contiguousLogEnd.get()).c_str()));
if(maxLogEnd.present())
if (maxLogEnd.present())
info.append(format("MaxLogEndVersion: %s\n", formatVersion(maxLogEnd.get()).c_str()));
if(minRestorableVersion.present())
if (minRestorableVersion.present())
info.append(format("MinRestorableVersion: %s\n", formatVersion(minRestorableVersion.get()).c_str()));
if(maxRestorableVersion.present())
if (maxRestorableVersion.present())
info.append(format("MaxRestorableVersion: %s\n", formatVersion(maxRestorableVersion.get()).c_str()));
if(!extendedDetail.empty())
if (!extendedDetail.empty())
info.append("ExtendedDetail: ").append(extendedDetail);
return info;
@ -189,14 +192,13 @@ std::string BackupDescription::toJSON() const {
auto formatVersion = [&](Version v) {
JsonBuilderObject doc;
doc.setKey("Version", v);
if(!versionTimeMap.empty()) {
if (!versionTimeMap.empty()) {
auto i = versionTimeMap.find(v);
if(i != versionTimeMap.end()) {
if (i != versionTimeMap.end()) {
doc.setKey("Timestamp", BackupAgentBase::formatTime(i->second));
doc.setKey("EpochSeconds", i->second);
}
}
else if(maxLogEnd.present()) {
} else if (maxLogEnd.present()) {
double days = double(v - maxLogEnd.get()) / (CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 60 * 60);
doc.setKey("RelativeDays", days);
}
@ -204,7 +206,7 @@ std::string BackupDescription::toJSON() const {
};
JsonBuilderArray snapshotsArray;
for(const KeyspaceSnapshotFile &m : snapshots) {
for (const KeyspaceSnapshotFile& m : snapshots) {
JsonBuilderObject snapshotDoc;
snapshotDoc.setKey("Start", formatVersion(m.beginVersion));
snapshotDoc.setKey("End", formatVersion(m.endVersion));
@ -217,22 +219,22 @@ std::string BackupDescription::toJSON() const {
doc.setKey("TotalSnapshotBytes", snapshotBytes);
if(expiredEndVersion.present())
if (expiredEndVersion.present())
doc.setKey("ExpiredEnd", formatVersion(expiredEndVersion.get()));
if(unreliableEndVersion.present())
if (unreliableEndVersion.present())
doc.setKey("UnreliableEnd", formatVersion(unreliableEndVersion.get()));
if(minLogBegin.present())
if (minLogBegin.present())
doc.setKey("MinLogBegin", formatVersion(minLogBegin.get()));
if(contiguousLogEnd.present())
if (contiguousLogEnd.present())
doc.setKey("ContiguousLogEnd", formatVersion(contiguousLogEnd.get()));
if(maxLogEnd.present())
if (maxLogEnd.present())
doc.setKey("MaxLogEnd", formatVersion(maxLogEnd.get()));
if(minRestorableVersion.present())
if (minRestorableVersion.present())
doc.setKey("MinRestorablePoint", formatVersion(minRestorableVersion.get()));
if(maxRestorableVersion.present())
if (maxRestorableVersion.present())
doc.setKey("MaxRestorablePoint", formatVersion(maxRestorableVersion.get()));
if(!extendedDetail.empty())
if (!extendedDetail.empty())
doc.setKey("ExtendedDetail", extendedDetail);
return doc.getJson();
@ -255,7 +257,8 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
static std::map<std::string, Reference<IBackupContainer>> m_cache;
Reference<IBackupContainer>& r = m_cache[url];
if (r) return r;
if (r)
return r;
try {
StringRef u(url);
@ -269,9 +272,11 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
Reference<S3BlobStoreEndpoint> bstore =
S3BlobStoreEndpoint::fromString(url, &resource, &lastOpenError, &backupParams);
if (resource.empty()) throw backup_invalid_url();
if (resource.empty())
throw backup_invalid_url();
for (auto c : resource)
if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/') throw backup_invalid_url();
if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/')
throw backup_invalid_url();
r = Reference<IBackupContainer>(new BackupContainerS3BlobStore(bstore, resource, backupParams));
}
#ifdef BUILD_AZURE_BACKUP
@ -291,13 +296,15 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
r->URL = url;
return r;
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw;
if (e.code() == error_code_actor_cancelled)
throw;
TraceEvent m(SevWarn, "BackupContainer");
m.detail("Description", "Invalid container specification. See help.");
m.detail("URL", url);
m.error(e);
if (e.code() == error_code_backup_invalid_url) m.detail("LastOpenError", lastOpenError);
if (e.code() == error_code_backup_invalid_url)
m.detail("LastOpenError", lastOpenError);
throw;
}
@ -344,14 +351,16 @@ ACTOR Future<std::vector<std::string>> listContainers_impl(std::string baseURL)
}
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) throw;
if (e.code() == error_code_actor_cancelled)
throw;
TraceEvent m(SevWarn, "BackupContainer");
m.detail("Description", "Invalid backup container URL prefix. See help.");
m.detail("URL", baseURL);
m.error(e);
if (e.code() == error_code_backup_invalid_url) m.detail("LastOpenError", IBackupContainer::lastOpenError);
if (e.code() == error_code_backup_invalid_url)
m.detail("LastOpenError", IBackupContainer::lastOpenError);
throw;
}
@ -367,8 +376,8 @@ ACTOR Future<Version> timeKeeperVersionFromDatetime(std::string datetime, Databa
state int64_t time = BackupAgentBase::parseTime(datetime);
if (time < 0) {
fprintf(stderr, "ERROR: Incorrect date/time or format. Format is %s.\n",
BackupAgentBase::timeFormat().c_str());
fprintf(
stderr, "ERROR: Incorrect date/time or format. Format is %s.\n", BackupAgentBase::timeFormat().c_str());
throw backup_error();
}

81
fdbclient/BackupContainer.h
View File

@ -31,8 +31,8 @@
class ReadYourWritesTransaction;
Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version const &v, Reference<ReadYourWritesTransaction> const &tr);
Future<Version> timeKeeperVersionFromDatetime(std::string const &datetime, Database const &db);
Future<Optional<int64_t>> timeKeeperEpochsFromVersion(Version const& v, Reference<ReadYourWritesTransaction> const& tr);
Future<Version> timeKeeperVersionFromDatetime(std::string const& datetime, Database const& db);
// Append-only file interface for writing backup data
// Once finish() is called the file cannot be further written to.
@ -43,16 +43,15 @@ public:
IBackupFile(const std::string& fileName) : m_fileName(fileName) {}
virtual ~IBackupFile() {}
// Backup files are append-only and cannot have more than 1 append outstanding at once.
virtual Future<Void> append(const void *data, int len) = 0;
virtual Future<Void> append(const void* data, int len) = 0;
virtual Future<Void> finish() = 0;
inline std::string getFileName() const {
return m_fileName;
}
inline std::string getFileName() const { return m_fileName; }
virtual int64_t size() const = 0;
virtual void addref() = 0;
virtual void delref() = 0;
Future<Void> appendStringRefWithLen(Standalone<StringRef> s);
protected:
std::string m_fileName;
};
@ -78,7 +77,7 @@ struct LogFile {
int totalTags = -1; // Total number of log router tags.
// Order by beginVersion, break ties with endVersion
bool operator< (const LogFile &rhs) const {
bool operator<(const LogFile& rhs) const {
return beginVersion == rhs.beginVersion ? endVersion < rhs.endVersion : beginVersion < rhs.beginVersion;
}
@ -88,9 +87,7 @@ struct LogFile {
return beginVersion >= rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId;
}
bool isPartitionedLog() const {
return tagId >= 0 && tagId < totalTags;
}
bool isPartitionedLog() const { return tagId >= 0 && tagId < totalTags; }
std::string toString() const {
std::stringstream ss;
@ -109,14 +106,14 @@ struct RangeFile {
int64_t fileSize;
// Order by version, break ties with name
bool operator< (const RangeFile &rhs) const {
bool operator<(const RangeFile& rhs) const {
return version == rhs.version ? fileName < rhs.fileName : version < rhs.version;
}
std::string toString() const {
std::stringstream ss;
ss << "version:" << std::to_string(version) << " blockSize:" << std::to_string(blockSize) <<
" fileName:" << fileName << " fileSize:" << std::to_string(fileSize);
ss << "version:" << std::to_string(version) << " blockSize:" << std::to_string(blockSize)
<< " fileName:" << fileName << " fileSize:" << std::to_string(fileSize);
return ss.str();
}
};
@ -126,25 +123,23 @@ struct KeyspaceSnapshotFile {
Version endVersion;
std::string fileName;
int64_t totalSize;
Optional<bool> restorable; // Whether or not the snapshot can be used in a restore, if known
bool isSingleVersion() const {
return beginVersion == endVersion;
}
Optional<bool> restorable; // Whether or not the snapshot can be used in a restore, if known
bool isSingleVersion() const { return beginVersion == endVersion; }
double expiredPct(Optional<Version> expiredEnd) const {
double pctExpired = 0;
if(expiredEnd.present() && expiredEnd.get() > beginVersion) {
if(isSingleVersion()) {
if (expiredEnd.present() && expiredEnd.get() > beginVersion) {
if (isSingleVersion()) {
pctExpired = 1;
}
else {
pctExpired = double(std::min(endVersion, expiredEnd.get()) - beginVersion) / (endVersion - beginVersion);
} else {
pctExpired =
double(std::min(endVersion, expiredEnd.get()) - beginVersion) / (endVersion - beginVersion);
}
}
return pctExpired * 100;
}
// Order by beginVersion, break ties with endVersion
bool operator< (const KeyspaceSnapshotFile &rhs) const {
bool operator<(const KeyspaceSnapshotFile& rhs) const {
return beginVersion == rhs.beginVersion ? endVersion < rhs.endVersion : beginVersion < rhs.beginVersion;
}
};
@ -154,7 +149,7 @@ struct BackupFileList {
std::vector<LogFile> logs;
std::vector<KeyspaceSnapshotFile> snapshots;
void toStream(FILE *fout) const;
void toStream(FILE* fout) const;
};
// The byte counts here only include usable log files and byte counts from kvrange manifests
@ -177,7 +172,7 @@ struct BackupDescription {
Optional<Version> maxRestorableVersion;
// The minimum version which this backup can be used to restore to
Optional<Version> minRestorableVersion;
std::string extendedDetail; // Freeform container-specific info.
std::string extendedDetail; // Freeform container-specific info.
bool partitioned; // If this backup contains partitioned mutation logs.
// Resolves the versions above to timestamps using a given database's TimeKeeper data.
@ -231,11 +226,17 @@ public:
// Open a log file or range file for writing
virtual Future<Reference<IBackupFile>> writeLogFile(Version beginVersion, Version endVersion, int blockSize) = 0;
virtual Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Version fileVersion, int blockSize) = 0;
virtual Future<Reference<IBackupFile>> writeRangeFile(Version snapshotBeginVersion,
int snapshotFileCount,
Version fileVersion,
int blockSize) = 0;
// Open a tagged log file for writing, where tagId is the log router tag's id.
virtual Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize,
uint16_t tagId, int totalTags) = 0;
virtual Future<Reference<IBackupFile>> writeTaggedLogFile(Version beginVersion,
Version endVersion,
int blockSize,
uint16_t tagId,
int totalTags) = 0;
// Write a KeyspaceSnapshotFile of range file names representing a full non overlapping
// snapshot of the key ranges this backup is targeting.
@ -260,22 +261,27 @@ public:
// If force is false, then nothing will be deleted unless there is a restorable snapshot which
// - begins at or after expireEndVersion
// - ends at or before restorableBeginVersion
// If force is true, data is deleted unconditionally which could leave the backup in an unusable state. This is not recommended.
// Returns true if expiration was done.
virtual Future<Void> expireData(Version expireEndVersion, bool force = false, ExpireProgress *progress = nullptr, Version restorableBeginVersion = std::numeric_limits<Version>::max()) = 0;
// If force is true, data is deleted unconditionally which could leave the backup in an unusable state. This is not
// recommended. Returns true if expiration was done.
virtual Future<Void> expireData(Version expireEndVersion,
bool force = false,
ExpireProgress* progress = nullptr,
Version restorableBeginVersion = std::numeric_limits<Version>::max()) = 0;
// Delete entire container. During the process, if pNumDeleted is not null it will be
// updated with the count of deleted files so that progress can be seen.
virtual Future<Void> deleteContainer(int *pNumDeleted = nullptr) = 0;
virtual Future<Void> deleteContainer(int* pNumDeleted = nullptr) = 0;
// Return key details about a backup's contents.
// Unless deepScan is true, use cached metadata, if present, as initial contiguous available log range.
// If logStartVersionOverride is given, log data prior to that version will be ignored for the purposes
// of this describe operation. This can be used to calculate what the restorability of a backup would
// be after deleting all data prior to logStartVersionOverride.
virtual Future<BackupDescription> describeBackup(bool deepScan = false, Version logStartVersionOverride = invalidVersion) = 0;
virtual Future<BackupDescription> describeBackup(bool deepScan = false,
Version logStartVersionOverride = invalidVersion) = 0;
virtual Future<BackupFileList> dumpFileList(Version begin = 0, Version end = std::numeric_limits<Version>::max()) = 0;
virtual Future<BackupFileList> dumpFileList(Version begin = 0,
Version end = std::numeric_limits<Version>::max()) = 0;
// Get exactly the files necessary to restore the key space filtered by the specified key ranges to targetVersion.
// If targetVersion is 'latestVersion', use the minimum restorable version in a snapshot.
@ -283,16 +289,15 @@ public:
// Returns non-present if restoring to the given version is not possible.
virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion,
VectorRef<KeyRangeRef> keyRangesFilter = {},
bool logsOnly = false, Version beginVersion = -1) = 0;
bool logsOnly = false,
Version beginVersion = -1) = 0;
// Get an IBackupContainer based on a container spec string
static Reference<IBackupContainer> openContainer(const std::string& url);
static std::vector<std::string> getURLFormats();
static Future<std::vector<std::string>> listContainers(const std::string& baseURL);
std::string getURL() const {
return URL;
}
std::string getURL() const { return URL; }
static std::string lastOpenError;

41
fdbclient/BackupContainerAzureBlobStore.actor.cpp
View File

@ -33,15 +33,21 @@ public:
AzureClient* client;
public:
ReadFile(AsyncTaskThread& asyncTaskThread, const std::string& containerName, const std::string& blobName,
ReadFile(AsyncTaskThread& asyncTaskThread,
const std::string& containerName,
const std::string& blobName,
AzureClient* client)
: asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {}
void addref() override { ReferenceCounted<ReadFile>::addref(); }
void delref() override { ReferenceCounted<ReadFile>::delref(); }
Future<int> read(void* data, int length, int64_t offset) {
return asyncTaskThread.execAsync([client = this->client, containerName = this->containerName,
blobName = this->blobName, data, length, offset] {
return asyncTaskThread.execAsync([client = this->client,
containerName = this->containerName,
blobName = this->blobName,
data,
length,
offset] {
std::ostringstream oss(std::ios::out | std::ios::binary);
client->download_blob_to_stream(containerName, blobName, offset, length, oss);
auto str = std::move(oss).str();
@ -54,7 +60,8 @@ public:
Future<Void> truncate(int64_t size) override { throw file_not_writable(); }
Future<Void> sync() override { throw file_not_writable(); }
Future<int64_t> size() const override {
return asyncTaskThread.execAsync([client = this->client, containerName = this->containerName,
return asyncTaskThread.execAsync([client = this->client,
containerName = this->containerName,
blobName = this->blobName] {
return static_cast<int64_t>(client->get_blob_properties(containerName, blobName).get().response().size);
});
@ -77,7 +84,9 @@ public:
static constexpr size_t bufferLimit = 1 << 20;
public:
WriteFile(AsyncTaskThread& asyncTaskThread, const std::string& containerName, const std::string& blobName,
WriteFile(AsyncTaskThread& asyncTaskThread,
const std::string& containerName,
const std::string& blobName,
AzureClient* client)
: asyncTaskThread(asyncTaskThread), containerName(containerName), blobName(blobName), client(client) {}
@ -106,8 +115,10 @@ public:
Future<Void> sync() override {
auto movedBuffer = std::move(buffer);
buffer.clear();
return asyncTaskThread.execAsync([client = this->client, containerName = this->containerName,
blobName = this->blobName, buffer = std::move(movedBuffer)] {
return asyncTaskThread.execAsync([client = this->client,
containerName = this->containerName,
blobName = this->blobName,
buffer = std::move(movedBuffer)] {
std::istringstream iss(std::move(buffer));
auto resp = client->append_block_from_stream(containerName, blobName, iss).get();
return Void();
@ -167,11 +178,14 @@ public:
return Void();
}));
return Reference<IBackupFile>(
new BackupFile(fileName, Reference<IAsyncFile>(new WriteFile(self->asyncTaskThread, self->containerName,
fileName, self->client.get()))));
new BackupFile(fileName,
Reference<IAsyncFile>(new WriteFile(
self->asyncTaskThread, self->containerName, fileName, self->client.get()))));
}
static void listFiles(AzureClient* client, const std::string& containerName, const std::string& path,
static void listFiles(AzureClient* client,
const std::string& containerName,
const std::string& path,
std::function<bool(std::string const&)> folderPathFilter,
BackupContainerFileSystem::FilesAndSizesT& result) {
auto resp = client->list_blobs_segmented(containerName, "/", "", path).get().response();
@ -251,8 +265,11 @@ Future<Reference<IBackupFile>> BackupContainerAzureBlobStore::writeFile(const st
}
Future<BackupContainerFileSystem::FilesAndSizesT> BackupContainerAzureBlobStore::listFiles(
const std::string& path, std::function<bool(std::string const&)> folderPathFilter) {
return asyncTaskThread.execAsync([client = this->client.get(), containerName = this->containerName, path = path,
const std::string& path,
std::function<bool(std::string const&)> folderPathFilter) {
return asyncTaskThread.execAsync([client = this->client.get(),
containerName = this->containerName,
path = path,
folderPathFilter = folderPathFilter] {
FilesAndSizesT result;
BackupContainerAzureBlobStoreImpl::listFiles(client, containerName, path, folderPathFilter, result);

Some files were not shown because too many files have changed in this diff Show More