diff --git a/bindings/bindingtester/__init__.py b/bindings/bindingtester/__init__.py index 8f93b26b09..75454625c0 100644 --- a/bindings/bindingtester/__init__.py +++ b/bindings/bindingtester/__init__.py @@ -26,7 +26,7 @@ sys.path[:0] = [os.path.join(os.path.dirname(__file__), '..', '..', 'bindings', import util -FDB_API_VERSION = 610 +FDB_API_VERSION = 620 LOGGING = { 'version': 1, diff --git a/bindings/bindingtester/bindingtester.py b/bindings/bindingtester/bindingtester.py index 559244233b..b14b0007f5 100755 --- a/bindings/bindingtester/bindingtester.py +++ b/bindings/bindingtester/bindingtester.py @@ -158,7 +158,7 @@ def choose_api_version(selected_api_version, tester_min_version, tester_max_vers api_version = min_version elif random.random() < 0.9: api_version = random.choice([v for v in [13, 14, 16, 21, 22, 23, 100, 200, 300, 400, 410, 420, 430, - 440, 450, 460, 500, 510, 520, 600, 610] if v >= min_version and v <= max_version]) + 440, 450, 460, 500, 510, 520, 600, 610, 620] if v >= min_version and v <= max_version]) else: api_version = random.randint(min_version, max_version) diff --git a/bindings/bindingtester/known_testers.py b/bindings/bindingtester/known_testers.py index fee09f5adf..2c5211a3df 100644 --- a/bindings/bindingtester/known_testers.py +++ b/bindings/bindingtester/known_testers.py @@ -20,7 +20,7 @@ import os -MAX_API_VERSION = 610 +MAX_API_VERSION = 620 COMMON_TYPES = ['null', 'bytes', 'string', 'int', 'uuid', 'bool', 'float', 'double', 'tuple'] ALL_TYPES = COMMON_TYPES + ['versionstamp'] diff --git a/bindings/bindingtester/spec/bindingApiTester.md b/bindings/bindingtester/spec/bindingApiTester.md index 872710a671..2f67c9dc1f 100644 --- a/bindings/bindingtester/spec/bindingApiTester.md +++ b/bindings/bindingtester/spec/bindingApiTester.md @@ -277,6 +277,12 @@ futures must apply the following rules to the result: internal stack machine state as the last seen version. Pushes the byte string "GOT_COMMITTED_VERSION" onto the stack. +#### GET_APPROXIMATE_SIZE + + Calls get_approximate_size and pushes the byte string "GOT_APPROXIMATE_SIZE" + onto the stack. Note bindings may issue GET_RANGE calls with different + limits, so these bindings can obtain different sizes back. + #### WAIT_FUTURE Pops the top item off the stack and pushes it back on. If the top item on diff --git a/bindings/bindingtester/tests/api.py b/bindings/bindingtester/tests/api.py index ccad9d39b3..be6350ece6 100644 --- a/bindings/bindingtester/tests/api.py +++ b/bindings/bindingtester/tests/api.py @@ -156,6 +156,7 @@ class ApiTest(Test): resets = ['ON_ERROR', 'RESET', 'CANCEL'] read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY'] write_conflicts = ['WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT'] + txn_sizes = ['GET_APPROXIMATE_SIZE'] op_choices += reads op_choices += mutations @@ -168,6 +169,7 @@ class ApiTest(Test): op_choices += read_conflicts op_choices += write_conflicts op_choices += resets + op_choices += txn_sizes idempotent_atomic_ops = [u'BIT_AND', u'BIT_OR', u'MAX', u'MIN', u'BYTE_MIN', u'BYTE_MAX'] atomic_ops = idempotent_atomic_ops + [u'ADD', u'BIT_XOR', u'APPEND_IF_FITS'] @@ -434,6 +436,10 @@ class ApiTest(Test): self.can_set_version = True self.can_use_key_selectors = True + elif op == 'GET_APPROXIMATE_SIZE': + instructions.append(op) + self.add_strings(1) + elif op == 'TUPLE_PACK' or op == 'TUPLE_RANGE': tup = self.random.random_tuple(10) instructions.push_args(len(tup), *tup) diff --git a/bindings/bindingtester/tests/scripted.py b/bindings/bindingtester/tests/scripted.py index 5fedd317d3..279fc10eae 100644 --- a/bindings/bindingtester/tests/scripted.py +++ b/bindings/bindingtester/tests/scripted.py @@ -34,7 +34,7 @@ fdb.api_version(FDB_API_VERSION) class ScriptedTest(Test): - TEST_API_VERSION = 610 + TEST_API_VERSION = 620 def __init__(self, subspace): super(ScriptedTest, self).__init__(subspace, ScriptedTest.TEST_API_VERSION, ScriptedTest.TEST_API_VERSION) diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt index a5897a08d4..b30a78bd84 100644 --- a/bindings/c/CMakeLists.txt +++ b/bindings/c/CMakeLists.txt @@ -60,16 +60,20 @@ if(NOT WIN32) if(OPEN_FOR_IDE) add_library(fdb_c_performance_test OBJECT test/performance_test.c test/test.h) add_library(fdb_c_ryw_benchmark OBJECT test/ryw_benchmark.c test/test.h) + add_library(fdb_c_txn_size_test OBJECT test/txn_size_test.c test/test.h) add_library(mako OBJECT ${MAKO_SRCS}) else() add_executable(fdb_c_performance_test test/performance_test.c test/test.h) add_executable(fdb_c_ryw_benchmark test/ryw_benchmark.c test/test.h) + add_executable(fdb_c_txn_size_test test/txn_size_test.c test/test.h) add_executable(mako ${MAKO_SRCS}) strip_debug_symbols(fdb_c_performance_test) strip_debug_symbols(fdb_c_ryw_benchmark) + strip_debug_symbols(fdb_c_txn_size_test) endif() target_link_libraries(fdb_c_performance_test PRIVATE fdb_c) target_link_libraries(fdb_c_ryw_benchmark PRIVATE fdb_c) + target_link_libraries(fdb_c_txn_size_test PRIVATE fdb_c) # do not set RPATH for mako set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE) target_link_libraries(mako PRIVATE fdb_c) diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index efb498b971..f0ea313b7b 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -18,7 +18,7 @@ * limitations under the License. */ -#define FDB_API_VERSION 610 +#define FDB_API_VERSION 620 #define FDB_INCLUDE_LEGACY_TYPES #include "fdbclient/MultiVersionTransaction.h" @@ -215,10 +215,15 @@ fdb_error_t fdb_future_get_error_v22( FDBFuture* f, const char** description ) { } extern "C" DLLEXPORT -fdb_error_t fdb_future_get_version( FDBFuture* f, int64_t* out_version ) { +fdb_error_t fdb_future_get_version_v619( FDBFuture* f, int64_t* out_version ) { CATCH_AND_RETURN( *out_version = TSAV(Version, f)->get(); ); } +extern "C" DLLEXPORT +fdb_error_t fdb_future_get_int64( FDBFuture* f, int64_t* out_value ) { + CATCH_AND_RETURN( *out_value = TSAV(int64_t, f)->get(); ); +} + extern "C" DLLEXPORT fdb_error_t fdb_future_get_key( FDBFuture* f, uint8_t const** out_key, int* out_key_length ) { @@ -584,6 +589,11 @@ fdb_error_t fdb_transaction_get_committed_version( FDBTransaction* tr, *out_version = TXN(tr)->getCommittedVersion(); ); } +extern "C" DLLEXPORT +FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* tr) { + return (FDBFuture*)TXN(tr)->getApproximateSize().extractPtr(); +} + extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_versionstamp( FDBTransaction* tr ) { @@ -670,6 +680,7 @@ fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version // Versioned API changes -- descending order by version (new changes at top) // FDB_API_CHANGED( function, ver ) means there is a new implementation as of ver, and a function function_(ver-1) is the old implementation // FDB_API_REMOVED( function, ver ) means the function was removed as of ver, and function_(ver-1) is the old implementation + FDB_API_REMOVED( fdb_future_get_version, 620 ); FDB_API_REMOVED( fdb_create_cluster, 610 ); FDB_API_REMOVED( fdb_cluster_create_database, 610 ); FDB_API_REMOVED( fdb_cluster_set_option, 610 ); diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h index fd04c84592..0e049e4119 100644 --- a/bindings/c/foundationdb/fdb_c.h +++ b/bindings/c/foundationdb/fdb_c.h @@ -28,10 +28,10 @@ #endif #if !defined(FDB_API_VERSION) -#error You must #define FDB_API_VERSION prior to including fdb_c.h (current version is 610) +#error You must #define FDB_API_VERSION prior to including fdb_c.h (current version is 620) #elif FDB_API_VERSION < 13 #error API version no longer supported (upgrade to 13) -#elif FDB_API_VERSION > 610 +#elif FDB_API_VERSION > 620 #error Requested API version requires a newer version of this header #endif @@ -120,8 +120,13 @@ extern "C" { fdb_future_get_error( FDBFuture* f ); #endif +#if FDB_API_VERSION < 620 DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_version( FDBFuture* f, int64_t* out_version ); +#endif + + DLLEXPORT WARN_UNUSED_RESULT fdb_error_t + fdb_future_get_int64( FDBFuture* f, int64_t* out ); DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_key( FDBFuture* f, uint8_t const** out_key, @@ -225,6 +230,13 @@ extern "C" { fdb_transaction_get_committed_version( FDBTransaction* tr, int64_t* out_version ); + // This function intentionally returns an FDBFuture instead of an integer directly, + // so that calling this API can see the effect of previous mutations on the transaction. + // Specifically, mutations are applied asynchronously by the main thread. In order to + // see them, this call has to be serviced by the main thread too. + DLLEXPORT WARN_UNUSED_RESULT FDBFuture* + fdb_transaction_get_approximate_size(FDBTransaction* tr); + DLLEXPORT WARN_UNUSED_RESULT FDBFuture* fdb_transaction_get_versionstamp( FDBTransaction* tr ); DLLEXPORT WARN_UNUSED_RESULT FDBFuture* diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c index bfccdb9594..9b49e57fc3 100755 --- a/bindings/c/test/mako/mako.c +++ b/bindings/c/test/mako/mako.c @@ -290,13 +290,22 @@ int64_t run_op_getreadversion(FDBTransaction *transaction) { } while (err && retry--); if (err) { - fprintf(stderr, "ERROR: fdb_transaction_get_version: %s\n", fdb_get_error(err)); + fprintf(stderr, "ERROR: fdb_transaction_get_read_version: %s\n", fdb_get_error(err)); return -1; } +#if FDB_API_VERSION < 620 err = fdb_future_get_version(f, &rv); +#else + err = fdb_future_get_int64(f, &rv); +#endif + if (err) { +#if FDB_API_VERSION < 620 fprintf(stderr, "ERROR: fdb_future_get_version: %s\n", fdb_get_error(err)); +#else + fprintf(stderr, "ERROR: fdb_future_get_int64: %s\n", fdb_get_error(err)); +#endif } fdb_future_destroy(f); return rv; diff --git a/bindings/c/test/mako/mako.h b/bindings/c/test/mako/mako.h index e22ee21d83..b8515092df 100755 --- a/bindings/c/test/mako/mako.h +++ b/bindings/c/test/mako/mako.h @@ -3,7 +3,7 @@ #pragma once #ifndef FDB_API_VERSION -#define FDB_API_VERSION 610 +#define FDB_API_VERSION 620 #endif #include diff --git a/bindings/c/test/performance_test.c b/bindings/c/test/performance_test.c index 01ccc95ce4..edfb9a96d1 100644 --- a/bindings/c/test/performance_test.c +++ b/bindings/c/test/performance_test.c @@ -603,7 +603,7 @@ void runTests(struct ResultSet *rs) { int main(int argc, char **argv) { srand(time(NULL)); struct ResultSet *rs = newResultSet(); - checkError(fdb_select_api_version(610), "select API version", rs); + checkError(fdb_select_api_version(620), "select API version", rs); printf("Running performance test at client version: %s\n", fdb_get_client_version()); valueStr = (uint8_t*)malloc((sizeof(uint8_t))*valueSize); diff --git a/bindings/c/test/ryw_benchmark.c b/bindings/c/test/ryw_benchmark.c index 55d62b1d50..1777150894 100644 --- a/bindings/c/test/ryw_benchmark.c +++ b/bindings/c/test/ryw_benchmark.c @@ -224,7 +224,7 @@ void runTests(struct ResultSet *rs) { checkError(fdb_future_block_until_ready(f), "block for read version", rs); int64_t version; - checkError(fdb_future_get_version(f, &version), "get version", rs); + checkError(fdb_future_get_int64(f, &version), "get version", rs); fdb_future_destroy(f); insertData(tr); @@ -244,7 +244,7 @@ void runTests(struct ResultSet *rs) { int main(int argc, char **argv) { srand(time(NULL)); struct ResultSet *rs = newResultSet(); - checkError(fdb_select_api_version(610), "select API version", rs); + checkError(fdb_select_api_version(620), "select API version", rs); printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version()); keys = generateKeys(numKeys, keySize); diff --git a/bindings/c/test/test.h b/bindings/c/test/test.h index cecb76b10c..895691c265 100644 --- a/bindings/c/test/test.h +++ b/bindings/c/test/test.h @@ -29,7 +29,7 @@ #include #ifndef FDB_API_VERSION -#define FDB_API_VERSION 610 +#define FDB_API_VERSION 620 #endif #include diff --git a/bindings/c/test/txn_size_test.c b/bindings/c/test/txn_size_test.c new file mode 100644 index 0000000000..73ee9a82e6 --- /dev/null +++ b/bindings/c/test/txn_size_test.c @@ -0,0 +1,110 @@ +/* + * txn_size_test.c + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "test.h" + +#include +#include +#include + +#include +#include + +pthread_t netThread; +const int numKeys = 100; +uint8_t** keys = NULL; + +#define KEY_SIZE 16 +#define VALUE_SIZE 100 +uint8_t valueStr[VALUE_SIZE]; + +fdb_error_t getSize(struct ResultSet* rs, FDBTransaction* tr, int64_t* out_size) { + fdb_error_t e; + FDBFuture* future = fdb_transaction_get_approximate_size(tr); + + e = maybeLogError(fdb_future_block_until_ready(future), "waiting for get future", rs); + if (e) { + fdb_future_destroy(future); + return e; + } + + e = maybeLogError(fdb_future_get_int64(future, out_size), "getting future value", rs); + if (e) { + fdb_future_destroy(future); + return e; + } + + fdb_future_destroy(future); + return 0; +} + +void runTests(struct ResultSet *rs) { + int64_t sizes[numKeys]; + int i = 0, j = 0; + FDBDatabase *db = openDatabase(rs, &netThread); + FDBTransaction *tr = NULL; + fdb_error_t e = fdb_database_create_transaction(db, &tr); + checkError(e, "create transaction", rs); + memset(sizes, 0, numKeys * sizeof(uint32_t)); + + fdb_transaction_set(tr, keys[i], KEY_SIZE, valueStr, VALUE_SIZE); + e = getSize(rs, tr, sizes + i); + checkError(e, "transaction get size", rs); + printf("size %d: %u\n", i, sizes[i]); + i++; + + fdb_transaction_set(tr, keys[i], KEY_SIZE, valueStr, VALUE_SIZE); + e = getSize(rs, tr, sizes + i); + checkError(e, "transaction get size", rs); + printf("size %d: %u\n", i, sizes[i]); + i++; + + fdb_transaction_clear(tr, keys[i], KEY_SIZE); + e = getSize(rs, tr, sizes + i); + checkError(e, "transaction get size", rs); + printf("size %d: %u\n", i, sizes[i]); + i++; + + fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i+1], KEY_SIZE); + e = getSize(rs, tr, sizes + i); + checkError(e, "transaction get size", rs); + printf("size %d: %u\n", i, sizes[i]); + i++; + + for (j = 0; j + 1 < i; j++) { + assert(sizes[j] < sizes[j + 1]); + } + printf("Test passed!\n"); +} + +int main(int argc, char **argv) { + srand(time(NULL)); + struct ResultSet *rs = newResultSet(); + checkError(fdb_select_api_version(620), "select API version", rs); + printf("Running performance test at client version: %s\n", fdb_get_client_version()); + + keys = generateKeys(numKeys, KEY_SIZE); + runTests(rs); + + freeResultSet(rs); + freeKeys(keys, numKeys); + + return 0; +} diff --git a/bindings/c/test/workloads/SimpleWorkload.cpp b/bindings/c/test/workloads/SimpleWorkload.cpp index 2872bc05af..3d61f85cb9 100644 --- a/bindings/c/test/workloads/SimpleWorkload.cpp +++ b/bindings/c/test/workloads/SimpleWorkload.cpp @@ -18,7 +18,7 @@ * limitations under the License. */ -#define FDB_API_VERSION 610 +#define FDB_API_VERSION 620 #include "foundationdb/fdb_c.h" #undef DLLEXPORT #include "workloads.h" @@ -258,7 +258,7 @@ struct SimpleWorkload : FDBWorkload { insertsPerTx = context->getOption("insertsPerTx", 100ul); opsPerTx = context->getOption("opsPerTx", 100ul); runFor = context->getOption("runFor", 10.0); - auto err = fdb_select_api_version(610); + auto err = fdb_select_api_version(620); if (err) { context->trace(FDBSeverity::Info, "SelectAPIVersionFailed", { { "Error", std::string(fdb_get_error(err)) } }); diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.actor.cpp index 99af1a665e..810f088e1c 100644 --- a/bindings/flow/fdb_flow.actor.cpp +++ b/bindings/flow/fdb_flow.actor.cpp @@ -20,12 +20,13 @@ #include "fdb_flow.h" -#include "flow/DeterministicRandom.h" -#include "flow/SystemMonitor.h" - #include #include +#include "flow/DeterministicRandom.h" +#include "flow/SystemMonitor.h" +#include "flow/actorcompiler.h" // This must be the last #include. + using namespace FDB; THREAD_FUNC networkThread(void* fdb) { @@ -34,7 +35,7 @@ THREAD_FUNC networkThread(void* fdb) { } ACTOR Future _test() { - API *fdb = FDB::API::selectAPIVersion(610); + API *fdb = FDB::API::selectAPIVersion(620); auto db = fdb->createDatabase(); state Reference tr = db->createTransaction(); @@ -77,7 +78,7 @@ ACTOR Future _test() { } void fdb_flow_test() { - API *fdb = FDB::API::selectAPIVersion(610); + API *fdb = FDB::API::selectAPIVersion(620); fdb->setupNetwork(); startThread(networkThread, fdb); @@ -147,6 +148,7 @@ namespace FDB { void setOption(FDBTransactionOption option, Optional value = Optional()) override; + Future getApproximateSize() override; Future onError(Error const& e) override; void cancel() override; @@ -290,7 +292,7 @@ namespace FDB { return backToFuture( fdb_transaction_get_read_version( tr ), [](Reference f){ Version value; - throw_on_error( fdb_future_get_version( f->f, &value ) ); + throw_on_error( fdb_future_get_int64( f->f, &value ) ); return value; } ); @@ -408,6 +410,14 @@ namespace FDB { } } + Future TransactionImpl::getApproximateSize() { + return backToFuture(fdb_transaction_get_approximate_size(tr), [](Reference f) { + int64_t size = 0; + throw_on_error(fdb_future_get_int64(f->f, &size)); + return size; + }); + } + Future TransactionImpl::onError(Error const& e) { return backToFuture< Void >( fdb_transaction_on_error( tr, e.code() ), [](Reference f) { throw_on_error( fdb_future_get_error( f->f ) ); @@ -422,4 +432,5 @@ namespace FDB { void TransactionImpl::reset() { fdb_transaction_reset( tr ); } -} + +} // namespace FDB diff --git a/bindings/flow/fdb_flow.h b/bindings/flow/fdb_flow.h index 3e84daecad..90f77e67cc 100644 --- a/bindings/flow/fdb_flow.h +++ b/bindings/flow/fdb_flow.h @@ -23,7 +23,7 @@ #include -#define FDB_API_VERSION 610 +#define FDB_API_VERSION 620 #include #undef DLLEXPORT @@ -112,6 +112,7 @@ namespace FDB { virtual Future commit() = 0; virtual Version getCommittedVersion() = 0; + virtual Future getApproximateSize() = 0; virtual Future> getVersionstamp() = 0; }; diff --git a/bindings/flow/tester/DirectoryTester.actor.cpp b/bindings/flow/tester/DirectoryTester.actor.cpp index 60c6cb88e2..a04317792b 100644 --- a/bindings/flow/tester/DirectoryTester.actor.cpp +++ b/bindings/flow/tester/DirectoryTester.actor.cpp @@ -19,6 +19,7 @@ */ #include "Tester.actor.h" +#include "flow/actorcompiler.h" // This must be the last #include. using namespace FDB; diff --git a/bindings/flow/tester/Tester.actor.cpp b/bindings/flow/tester/Tester.actor.cpp index fa309a59e8..69feab7b0d 100644 --- a/bindings/flow/tester/Tester.actor.cpp +++ b/bindings/flow/tester/Tester.actor.cpp @@ -18,16 +18,18 @@ * limitations under the License. */ -#include "fdbrpc/fdbrpc.h" -#include "flow/DeterministicRandom.h" -#include "bindings/flow/Tuple.h" -#include "bindings/flow/FDBLoanerTypes.h" - #include "Tester.actor.h" +#include #ifdef __linux__ #include #endif +#include "bindings/flow/Tuple.h" +#include "bindings/flow/FDBLoanerTypes.h" +#include "fdbrpc/fdbrpc.h" +#include "flow/DeterministicRandom.h" +#include "flow/actorcompiler.h" // This must be the last #include. + // Otherwise we have to type setupNetwork(), FDB::open(), etc. using namespace FDB; @@ -292,7 +294,7 @@ ACTOR Future printFlowTesterStack(FlowTesterStack* stack) { state int idx; for (idx = stack->data.size() - 1; idx >= 0; --idx) { Standalone value = wait(stack->data[idx].value); - // printf("==========stack item:%d, index:%d, value:%s\n", idx, stack->data[idx].index, printable(value).c_str()); + // printf("==========stack item:%d, index:%d, value:%s\n", idx, stack->data[idx].index, value.printable().c_str()); } return Void(); } @@ -703,6 +705,20 @@ struct GetCommittedVersionFunc : InstructionFunc { const char* GetCommittedVersionFunc::name = "GET_COMMITTED_VERSION"; REGISTER_INSTRUCTION_FUNC(GetCommittedVersionFunc); +// GET_APPROXIMATE_SIZE +struct GetApproximateSizeFunc : InstructionFunc { + static const char* name; + + ACTOR static Future call(Reference data, Reference instruction) { + int64_t _ = wait(instruction->tr->getApproximateSize()); + (void) _; // disable unused variable warning + data->stack.pushTuple(LiteralStringRef("GOT_APPROXIMATE_SIZE")); + return Void(); + } +}; +const char* GetApproximateSizeFunc::name = "GET_APPROXIMATE_SIZE"; +REGISTER_INSTRUCTION_FUNC(GetApproximateSizeFunc); + // GET_VERSIONSTAMP struct GetVersionstampFunc : InstructionFunc { static const char* name; @@ -1551,19 +1567,21 @@ struct UnitTestsFunc : InstructionFunc { const uint64_t noRetryLimit = -1; const uint64_t maxRetryDelay = 100; const uint64_t sizeLimit = 100000; + const uint64_t maxFieldLength = 1000; data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_LOCATION_CACHE_SIZE, Optional(StringRef((const uint8_t*)&locationCacheSize, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MAX_WATCHES, Optional(StringRef((const uint8_t*)&maxWatches, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_DATACENTER_ID, Optional(LiteralStringRef("dc_id"))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MACHINE_ID, Optional(LiteralStringRef("machine_id"))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_ENABLE); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_DISABLE); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, Optional(StringRef((const uint8_t*)&maxFieldLength, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, Optional(StringRef((const uint8_t*)&timeout, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, Optional(StringRef((const uint8_t*)&noTimeout, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_MAX_RETRY_DELAY, Optional(StringRef((const uint8_t*)&maxRetryDelay, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, Optional(StringRef((const uint8_t*)&sizeLimit, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, Optional(StringRef((const uint8_t*)&retryLimit, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, Optional(StringRef((const uint8_t*)&noRetryLimit, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_ENABLE); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_DISABLE); state Reference tr = data->db->createTransaction(); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_PRIORITY_SYSTEM_IMMEDIATE); @@ -1574,6 +1592,7 @@ struct UnitTestsFunc : InstructionFunc { tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_SYSTEM_KEYS); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, Optional(StringRef((const uint8_t*)&maxFieldLength, 8))); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TIMEOUT, Optional(StringRef((const uint8_t*)&timeout, 8))); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_RETRY_LIMIT, Optional(StringRef((const uint8_t*)&retryLimit, 8))); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_MAX_RETRY_DELAY, Optional(StringRef((const uint8_t*)&maxRetryDelay, 8))); @@ -1635,7 +1654,7 @@ ACTOR static Future doInstructions(Reference data) { op = op.substr(0, op.size() - 9); // printf("[==========]%ld/%ld:%s:%s: isDatabase:%d, isSnapshot:%d, stack count:%ld\n", - // idx, data->instructions.size(), printable(StringRef(data->instructions[idx].key)).c_str(), printable(StringRef(data->instructions[idx].value)).c_str(), + // idx, data->instructions.size(), StringRef(data->instructions[idx].key).printable().c_str(), StringRef(data->instructions[idx].value).printable().c_str(), // isDatabase, isSnapshot, data->stack.data.size()); //wait(printFlowTesterStack(&(data->stack))); @@ -1771,7 +1790,7 @@ ACTOR void _test_versionstamp() { try { g_network = newNet2(false); - API *fdb = FDB::API::selectAPIVersion(610); + API *fdb = FDB::API::selectAPIVersion(620); fdb->setupNetwork(); startThread(networkThread, fdb); diff --git a/bindings/go/README.md b/bindings/go/README.md index 27b11a68d9..6e2a90a684 100644 --- a/bindings/go/README.md +++ b/bindings/go/README.md @@ -9,7 +9,7 @@ This package requires: - [Mono](http://www.mono-project.com/) (macOS or Linux) or [Visual Studio](https://www.visualstudio.com/) (Windows) (build-time only) - FoundationDB C API 2.0.x-6.1.x (part of the [FoundationDB client packages](https://apple.github.io/foundationdb/downloads.html#c)) -Use of this package requires the selection of a FoundationDB API version at runtime. This package currently supports FoundationDB API versions 200-610. +Use of this package requires the selection of a FoundationDB API version at runtime. This package currently supports FoundationDB API versions 200-620. To install this package, you can run the "fdb-go-install.sh" script (for versions 5.0.x and greater): diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go index 5b5d988e6b..0530c2e146 100644 --- a/bindings/go/src/_stacktester/stacktester.go +++ b/bindings/go/src/_stacktester/stacktester.go @@ -590,6 +590,9 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { panic(e) } sm.store(idx, []byte("GOT_COMMITTED_VERSION")) + case op == "GET_APPROXIMATE_SIZE": + _ = sm.currentTransaction().GetApproximateSize().MustGet() + sm.store(idx, []byte("GOT_APPROXIMATE_SIZE")) case op == "GET_VERSIONSTAMP": sm.store(idx, sm.currentTransaction().GetVersionstamp()) case op == "GET_KEY": @@ -793,13 +796,14 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { db.Options().SetMaxWatches(10001) db.Options().SetDatacenterId("dc_id") db.Options().SetMachineId("machine_id") + db.Options().SetSnapshotRywEnable() + db.Options().SetSnapshotRywDisable() + db.Options().SetTransactionLoggingMaxFieldLength(1000) db.Options().SetTransactionTimeout(100000) db.Options().SetTransactionTimeout(0) db.Options().SetTransactionMaxRetryDelay(100) db.Options().SetTransactionRetryLimit(10) db.Options().SetTransactionRetryLimit(-1) - db.Options().SetSnapshotRywEnable() - db.Options().SetSnapshotRywDisable() if !fdb.IsAPIVersionSelected() { log.Fatal("API version should be selected") @@ -836,6 +840,7 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { tr.Options().SetReadYourWritesDisable() tr.Options().SetReadSystemKeys() tr.Options().SetAccessSystemKeys() + tr.Options().SetTransactionLoggingMaxFieldLength(1000) tr.Options().SetTimeout(60 * 1000) tr.Options().SetRetryLimit(50) tr.Options().SetMaxRetryDelay(100) diff --git a/bindings/go/src/fdb/cluster.go b/bindings/go/src/fdb/cluster.go index d6342170ab..275a036338 100644 --- a/bindings/go/src/fdb/cluster.go +++ b/bindings/go/src/fdb/cluster.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 610 +// #define FDB_API_VERSION 620 // #include import "C" diff --git a/bindings/go/src/fdb/database.go b/bindings/go/src/fdb/database.go index 92ae56300d..aca709e3d4 100644 --- a/bindings/go/src/fdb/database.go +++ b/bindings/go/src/fdb/database.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 610 +// #define FDB_API_VERSION 620 // #include import "C" diff --git a/bindings/go/src/fdb/doc.go b/bindings/go/src/fdb/doc.go index b12cb4a229..f6d0228ef5 100644 --- a/bindings/go/src/fdb/doc.go +++ b/bindings/go/src/fdb/doc.go @@ -46,7 +46,7 @@ A basic interaction with the FoundationDB API is demonstrated below: func main() { // Different API versions may expose different runtime behaviors. - fdb.MustAPIVersion(610) + fdb.MustAPIVersion(620) // Open the default database from the system cluster db := fdb.MustOpenDefault() diff --git a/bindings/go/src/fdb/errors.go b/bindings/go/src/fdb/errors.go index b828dd4c26..9380736b4e 100644 --- a/bindings/go/src/fdb/errors.go +++ b/bindings/go/src/fdb/errors.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 610 +// #define FDB_API_VERSION 620 // #include import "C" diff --git a/bindings/go/src/fdb/fdb.go b/bindings/go/src/fdb/fdb.go index 6e983d2ddc..3910a980c3 100644 --- a/bindings/go/src/fdb/fdb.go +++ b/bindings/go/src/fdb/fdb.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 610 +// #define FDB_API_VERSION 620 // #include // #include import "C" @@ -108,7 +108,7 @@ func (opt NetworkOptions) setOpt(code int, param []byte) error { // library, an error will be returned. APIVersion must be called prior to any // other functions in the fdb package. // -// Currently, this package supports API versions 200 through 610. +// Currently, this package supports API versions 200 through 620. // // Warning: When using the multi-version client API, setting an API version that // is not supported by a particular client library will prevent that client from @@ -116,7 +116,7 @@ func (opt NetworkOptions) setOpt(code int, param []byte) error { // the API version of your application after upgrading your client until the // cluster has also been upgraded. func APIVersion(version int) error { - headerVersion := 610 + headerVersion := 620 networkMutex.Lock() defer networkMutex.Unlock() @@ -128,7 +128,7 @@ func APIVersion(version int) error { return errAPIVersionAlreadySet } - if version < 200 || version > 610 { + if version < 200 || version > 620 { return errAPIVersionNotSupported } diff --git a/bindings/go/src/fdb/futures.go b/bindings/go/src/fdb/futures.go index 7416628c92..4894ee40ea 100644 --- a/bindings/go/src/fdb/futures.go +++ b/bindings/go/src/fdb/futures.go @@ -23,7 +23,7 @@ package fdb // #cgo LDFLAGS: -lfdb_c -lm -// #define FDB_API_VERSION 610 +// #define FDB_API_VERSION 620 // #include // #include // @@ -331,7 +331,7 @@ func (f *futureInt64) Get() (int64, error) { f.BlockUntilReady() var ver C.int64_t - if err := C.fdb_future_get_version(f.ptr, &ver); err != 0 { + if err := C.fdb_future_get_int64(f.ptr, &ver); err != 0 { return 0, Error{int(err)} } diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go index 782b108fda..3435613de6 100644 --- a/bindings/go/src/fdb/generated.go +++ b/bindings/go/src/fdb/generated.go @@ -280,6 +280,23 @@ func (o DatabaseOptions) SetDatacenterId(param string) error { return o.setOpt(22, []byte(param)) } +// Snapshot read operations will see the results of writes done in the same transaction. This is the default behavior. +func (o DatabaseOptions) SetSnapshotRywEnable() error { + return o.setOpt(26, nil) +} + +// Snapshot read operations will not see the results of writes done in the same transaction. This was the default behavior prior to API version 300. +func (o DatabaseOptions) SetSnapshotRywDisable() error { + return o.setOpt(27, nil) +} + +// Sets the maximum escaped length of key and value fields to be logged to the trace file via the LOG_TRANSACTION option. This sets the ``transaction_logging_max_field_length`` option of each transaction created by this database. See the transaction option description for more information. +// +// Parameter: Maximum length of escaped key and value fields. +func (o DatabaseOptions) SetTransactionLoggingMaxFieldLength(param int64) error { + return o.setOpt(405, int64ToBytes(param)) +} + // Set a timeout in milliseconds which, when elapsed, will cause each transaction automatically to be cancelled. This sets the ``timeout`` option of each transaction created by this database. See the transaction option description for more information. Using this option requires that the API version is 610 or higher. // // Parameter: value in milliseconds of timeout @@ -308,16 +325,6 @@ func (o DatabaseOptions) SetTransactionSizeLimit(param int64) error { return o.setOpt(503, int64ToBytes(param)) } -// Snapshot read operations will see the results of writes done in the same transaction. This is the default behavior. -func (o DatabaseOptions) SetSnapshotRywEnable() error { - return o.setOpt(26, nil) -} - -// Snapshot read operations will not see the results of writes done in the same transaction. This was the default behavior prior to API version 300. -func (o DatabaseOptions) SetSnapshotRywDisable() error { - return o.setOpt(27, nil) -} - // The transaction, if not self-conflicting, may be committed a second time after commit succeeds, in the event of a fault func (o TransactionOptions) SetCausalWriteRisky() error { return o.setOpt(10, nil) @@ -412,6 +419,13 @@ func (o TransactionOptions) SetLogTransaction() error { return o.setOpt(404, nil) } +// Sets the maximum escaped length of key and value fields to be logged to the trace file via the LOG_TRANSACTION option, after which the field will be truncated. A negative value disables truncation. +// +// Parameter: Maximum length of escaped key and value fields. +func (o TransactionOptions) SetTransactionLoggingMaxFieldLength(param int64) error { + return o.setOpt(405, int64ToBytes(param)) +} + // Set a timeout in milliseconds which, when elapsed, will cause the transaction automatically to be cancelled. Valid parameter values are ``[0, INT_MAX]``. If set to 0, will disable all timeouts. All pending and any future uses of the transaction will throw an exception. The transaction can be used again after it is reset. Prior to API version 610, like all other transaction options, the timeout must be reset after a call to ``onError``. If the API version is 610 or greater, the timeout is not reset after an ``onError`` call. This allows the user to specify a longer timeout on specific transactions than the default timeout specified through the ``transaction_timeout`` database option without the shorter database timeout cancelling transactions that encounter a retryable error. Note that at all API versions, it is safe and legal to set the timeout each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option. // // Parameter: value in milliseconds of timeout diff --git a/bindings/go/src/fdb/range.go b/bindings/go/src/fdb/range.go index 78b95f35fd..8273fe37fe 100644 --- a/bindings/go/src/fdb/range.go +++ b/bindings/go/src/fdb/range.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 610 +// #define FDB_API_VERSION 620 // #include import "C" diff --git a/bindings/go/src/fdb/transaction.go b/bindings/go/src/fdb/transaction.go index adfb5b62ec..274aeee867 100644 --- a/bindings/go/src/fdb/transaction.go +++ b/bindings/go/src/fdb/transaction.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 610 +// #define FDB_API_VERSION 620 // #include import "C" @@ -372,6 +372,16 @@ func (t Transaction) GetVersionstamp() FutureKey { return &futureKey{future: newFuture(C.fdb_transaction_get_versionstamp(t.ptr))} } +func (t *transaction) getApproximateSize() FutureInt64 { + return &futureInt64{ + future: newFuture(C.fdb_transaction_get_approximate_size(t.ptr)), + } +} + +func (t Transaction) GetApproximateSize() FutureInt64 { + return t.getApproximateSize() +} + // Reset rolls back a transaction, completely resetting it to its initial // state. This is logically equivalent to destroying the transaction and // creating a new one. diff --git a/bindings/java/CMakeLists.txt b/bindings/java/CMakeLists.txt index 5b460e9656..42c96c93a5 100644 --- a/bindings/java/CMakeLists.txt +++ b/bindings/java/CMakeLists.txt @@ -25,11 +25,11 @@ set(JAVA_BINDING_SRCS src/main/com/apple/foundationdb/FDB.java src/main/com/apple/foundationdb/FDBDatabase.java src/main/com/apple/foundationdb/FDBTransaction.java + src/main/com/apple/foundationdb/FutureInt64.java src/main/com/apple/foundationdb/FutureKey.java src/main/com/apple/foundationdb/FutureResult.java src/main/com/apple/foundationdb/FutureResults.java src/main/com/apple/foundationdb/FutureStrings.java - src/main/com/apple/foundationdb/FutureVersion.java src/main/com/apple/foundationdb/FutureVoid.java src/main/com/apple/foundationdb/JNIUtil.java src/main/com/apple/foundationdb/KeySelector.java @@ -141,9 +141,33 @@ target_include_directories(java_workloads PUBLIC ${JNI_INCLUDE_DIRS}) set(CMAKE_JAVA_COMPILE_FLAGS "-source" "1.8" "-target" "1.8") set(CMAKE_JNI_TARGET TRUE) -set(JAR_VERSION "${FDB_MAJOR}.${FDB_MINOR}.${FDB_REVISION}") + +# build a manifest file +set(JAR_VERSION "${FDB_MAJOR}.${FDB_MINOR}.${FDB_PATCH}") +string(TIMESTAMP BND_LAST_MODIFIED_SEC "%s") +set(MANIFEST_TEXT "Manifest-Version: 1.0 +Bnd-LastModified: ${BND_LAST_MODIFIED_SEC}000 +Build-Jdk: ${Java_VERSION_STRING} +Built-By: CMake ${CMAKE_VERSION} +Bundle-Description: FoundationDB Java bindings and API +Bundle-ManifestVersion: 2 +Bundle-Name: fdb-java +Bundle-SymbolicName: com.apple.foundationdb +Bundle-Version: ${JAR_VERSION} +Created-By: CMake ${CMAKE_VERSION} +Implementation-Title: fdb-java +Implementation-Version: ${CURRENT_GIT_VERSION} +Specification-Title: FoundationDB Java bindings and API +Specification-Version: ${JAR_VERSION} +") + +# write the manifest file +file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/resources/META-INF) +set(MANIFEST_FILE ${CMAKE_CURRENT_BINARY_DIR}/resources/META-INF/MANIFEST.MF) +file(WRITE ${MANIFEST_FILE} ${MANIFEST_TEXT}) + add_jar(fdb-java ${JAVA_BINDING_SRCS} ${GENERATED_JAVA_FILES} ${CMAKE_SOURCE_DIR}/LICENSE - OUTPUT_DIR ${PROJECT_BINARY_DIR}/lib VERSION ${CMAKE_PROJECT_VERSION}) + OUTPUT_DIR ${PROJECT_BINARY_DIR}/lib VERSION ${CMAKE_PROJECT_VERSION} MANIFEST ${MANIFEST_FILE}) add_dependencies(fdb-java fdb_java_options fdb_java) add_jar(foundationdb-tests SOURCES ${JAVA_TESTS_SRCS} INCLUDE_JARS fdb-java) add_dependencies(foundationdb-tests fdb_java_options) @@ -201,7 +225,7 @@ if(NOT OPEN_FOR_IDE) add_dependencies(copy_lib unpack_jar) set(target_jar ${jar_destination}/fdb-java-${CMAKE_PROJECT_VERSION}.jar) add_custom_command(OUTPUT ${target_jar} - COMMAND ${Java_JAR_EXECUTABLE} cf ${target_jar} . + COMMAND ${Java_JAR_EXECUTABLE} cfm ${target_jar} ${unpack_dir}/META-INF/MANIFEST.MF . WORKING_DIRECTORY ${unpack_dir} DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/lib_copied COMMENT "Build ${jar_destination}/fdb-java-${CMAKE_PROJECT_VERSION}.jar") diff --git a/bindings/java/JavaWorkload.cpp b/bindings/java/JavaWorkload.cpp index 0d0509451d..286197997f 100644 --- a/bindings/java/JavaWorkload.cpp +++ b/bindings/java/JavaWorkload.cpp @@ -19,7 +19,7 @@ */ #include -#define FDB_API_VERSION 610 +#define FDB_API_VERSION 620 #include #include @@ -370,7 +370,7 @@ struct JVM { jmethodID selectMethod = env->GetStaticMethodID(fdbClass, "selectAPIVersion", "(IZ)Lcom/apple/foundationdb/FDB;"); checkException(); - env->CallStaticObjectMethod(fdbClass, selectMethod, jint(610), jboolean(false)); + env->CallStaticObjectMethod(fdbClass, selectMethod, jint(620), jboolean(false)); checkException(); } diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp index 66191d9755..5a49987a85 100644 --- a/bindings/java/fdbJNI.cpp +++ b/bindings/java/fdbJNI.cpp @@ -21,7 +21,7 @@ #include #include -#define FDB_API_VERSION 610 +#define FDB_API_VERSION 620 #include @@ -243,21 +243,21 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1releaseM fdb_future_release_memory(var); } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FutureVersion_FutureVersion_1get(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FutureInt64_FutureInt64_1get(JNIEnv *jenv, jobject, jlong future) { + if (!future) { throwParamNotNull(jenv); return 0; } FDBFuture *f = (FDBFuture *)future; - int64_t version = 0; - fdb_error_t err = fdb_future_get_version(f, &version); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + int64_t value = 0; + fdb_error_t err = fdb_future_get_int64(f, &value); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return 0; } - return (jlong)version; + return (jlong)value; } JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureStrings_1get(JNIEnv *jenv, jobject, jlong future) { @@ -805,6 +805,15 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 return (jlong)version; } +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getApproximateSize(JNIEnv *jenv, jobject, jlong tPtr) { + if (!tPtr) { + throwParamNotNull(jenv); + return 0; + } + FDBFuture* f = fdb_transaction_get_approximate_size((FDBTransaction*)tPtr); + return (jlong)f; +} + JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getVersionstamp(JNIEnv *jenv, jobject, jlong tPtr) { if (!tPtr) { throwParamNotNull(jenv); diff --git a/bindings/java/src/main/com/apple/foundationdb/FDB.java b/bindings/java/src/main/com/apple/foundationdb/FDB.java index 232414a267..e20fa90432 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDB.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDB.java @@ -35,7 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger; * This call is required before using any other part of the API. The call allows * an error to be thrown at this point to prevent client code from accessing a later library * with incorrect assumptions from the current version. The API version documented here is version - * {@code 610}.

+ * {@code 620}.

* FoundationDB encapsulates multiple versions of its interface by requiring * the client to explicitly specify the version of the API it uses. The purpose * of this design is to allow you to upgrade the server, client libraries, or @@ -193,8 +193,8 @@ public class FDB { } if(version < 510) throw new IllegalArgumentException("API version not supported (minimum 510)"); - if(version > 610) - throw new IllegalArgumentException("API version not supported (maximum 610)"); + if(version > 620) + throw new IllegalArgumentException("API version not supported (maximum 620)"); Select_API_version(version); FDB fdb = new FDB(version, controlRuntime); diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java index 0e507c914d..d6f1e4f935 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java @@ -216,7 +216,7 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC public CompletableFuture getReadVersion() { pointerReadLock.lock(); try { - return new FutureVersion(Transaction_getReadVersion(getPtr()), executor); + return new FutureInt64(Transaction_getReadVersion(getPtr()), executor); } finally { pointerReadLock.unlock(); } @@ -514,6 +514,16 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC } } + @Override + public CompletableFuture getApproximateSize() { + pointerReadLock.lock(); + try { + return new FutureInt64(Transaction_getApproximateSize(getPtr()), executor); + } finally { + pointerReadLock.unlock(); + } + } + @Override public CompletableFuture watch(byte[] key) throws FDBException { pointerReadLock.lock(); @@ -642,6 +652,7 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC private native long Transaction_commit(long cPtr); private native long Transaction_getCommittedVersion(long cPtr); private native long Transaction_getVersionstamp(long cPtr); + private native long Transaction_getApproximateSize(long cPtr); private native long Transaction_onError(long cPtr, int errorCode); private native void Transaction_dispose(long cPtr); private native void Transaction_reset(long cPtr); diff --git a/bindings/java/src/main/com/apple/foundationdb/FutureVersion.java b/bindings/java/src/main/com/apple/foundationdb/FutureInt64.java similarity index 74% rename from bindings/java/src/main/com/apple/foundationdb/FutureVersion.java rename to bindings/java/src/main/com/apple/foundationdb/FutureInt64.java index c5f1322094..0e1b280b71 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FutureVersion.java +++ b/bindings/java/src/main/com/apple/foundationdb/FutureInt64.java @@ -1,9 +1,9 @@ /* - * FutureVersion.java + * FutureInt64.java * * This source file is part of the FoundationDB open source project * - * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,16 +22,16 @@ package com.apple.foundationdb; import java.util.concurrent.Executor; -class FutureVersion extends NativeFuture { - FutureVersion(long cPtr, Executor executor) { +class FutureInt64 extends NativeFuture { + FutureInt64(long cPtr, Executor executor) { super(cPtr); registerMarshalCallback(executor); } @Override protected Long getIfDone_internal(long cPtr) throws FDBException { - return FutureVersion_get(cPtr); + return FutureInt64_get(cPtr); } - private native long FutureVersion_get(long cPtr) throws FDBException; + private native long FutureInt64_get(long cPtr) throws FDBException; } diff --git a/bindings/java/src/main/com/apple/foundationdb/Transaction.java b/bindings/java/src/main/com/apple/foundationdb/Transaction.java index c3a8c6b671..f82c68e790 100644 --- a/bindings/java/src/main/com/apple/foundationdb/Transaction.java +++ b/bindings/java/src/main/com/apple/foundationdb/Transaction.java @@ -260,6 +260,15 @@ public interface Transaction extends AutoCloseable, ReadTransaction, Transaction */ CompletableFuture getVersionstamp(); + /** + * Returns a future that will contain the approximated size of the commit, which is the + * summation of mutations, read conflict ranges, and write conflict ranges. This can be + * called multiple times before transaction commit. + * + * @return a future that will contain the approximated size of the commit. + */ + CompletableFuture getApproximateSize(); + /** * Resets a transaction and returns a delayed signal for error recovery. If the error * encountered by the {@code Transaction} could not be recovered from, the returned diff --git a/bindings/java/src/main/overview.html.in b/bindings/java/src/main/overview.html.in index d25a128335..d594b769e3 100644 --- a/bindings/java/src/main/overview.html.in +++ b/bindings/java/src/main/overview.html.in @@ -13,7 +13,7 @@ and then added to your classpath.

Getting started

To start using FoundationDB from Java, create an instance of the {@link com.apple.foundationdb.FDB FoundationDB API interface} with the version of the -API that you want to use (this release of the FoundationDB Java API supports versions between {@code 510} and {@code 610}). +API that you want to use (this release of the FoundationDB Java API supports versions between {@code 510} and {@code 620}). With this API object you can then open {@link com.apple.foundationdb.Cluster Cluster}s and {@link com.apple.foundationdb.Database Database}s and start using {@link com.apple.foundationdb.Transaction Transaction}s. @@ -29,7 +29,7 @@ import com.apple.foundationdb.tuple.Tuple; public class Example { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(610); + FDB fdb = FDB.selectAPIVersion(620); try(Database db = fdb.open()) { // Run an operation on the database diff --git a/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java b/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java index 018e594532..969b12b89a 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java @@ -27,7 +27,7 @@ import com.apple.foundationdb.Database; import com.apple.foundationdb.FDB; public abstract class AbstractTester { - public static final int API_VERSION = 610; + public static final int API_VERSION = 620; protected static final int NUM_RUNS = 25; protected static final Charset ASCII = Charset.forName("ASCII"); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java index 6e2b6e9318..57278728dc 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java @@ -282,6 +282,11 @@ public class AsyncStackTester { return AsyncUtil.DONE; } + else if(op == StackOperation.GET_APPROXIMATE_SIZE) { + return inst.tr.getApproximateSize().thenAcceptAsync(size -> { + inst.push("GOT_APPROXIMATE_SIZE".getBytes()); + }, FDB.DEFAULT_EXECUTOR); + } else if(op == StackOperation.GET_VERSIONSTAMP) { try { inst.push(inst.tr.getVersionstamp()); @@ -481,13 +486,14 @@ public class AsyncStackTester { db.options().setMaxWatches(10001); db.options().setDatacenterId("dc_id"); db.options().setMachineId("machine_id"); + db.options().setSnapshotRywEnable(); + db.options().setSnapshotRywDisable(); + db.options().setTransactionLoggingMaxFieldLength(1000); db.options().setTransactionTimeout(100000); db.options().setTransactionTimeout(0); db.options().setTransactionMaxRetryDelay(100); db.options().setTransactionRetryLimit(10); db.options().setTransactionRetryLimit(-1); - db.options().setSnapshotRywEnable(); - db.options().setSnapshotRywDisable(); tr.options().setPrioritySystemImmediate(); tr.options().setPriorityBatch(); @@ -496,6 +502,7 @@ public class AsyncStackTester { tr.options().setReadYourWritesDisable(); tr.options().setReadSystemKeys(); tr.options().setAccessSystemKeys(); + tr.options().setTransactionLoggingMaxFieldLength(1000); tr.options().setTimeout(60*1000); tr.options().setRetryLimit(50); tr.options().setMaxRetryDelay(100); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java b/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java index 0576d0e359..86963c4496 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java @@ -33,7 +33,7 @@ public class BlockingBenchmark { private static final int PARALLEL = 100; public static void main(String[] args) throws InterruptedException { - FDB fdb = FDB.selectAPIVersion(610); + FDB fdb = FDB.selectAPIVersion(620); // The cluster file DOES NOT need to be valid, although it must exist. // This is because the database is never really contacted in this test. diff --git a/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java b/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java index 64c90f5b30..fcc77ae854 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java @@ -48,7 +48,7 @@ public class ConcurrentGetSetGet { } public static void main(String[] args) { - try(Database database = FDB.selectAPIVersion(610).open()) { + try(Database database = FDB.selectAPIVersion(620).open()) { new ConcurrentGetSetGet().apply(database); } } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java b/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java index 066bbf1bdf..40a781756a 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java @@ -33,7 +33,7 @@ import com.apple.foundationdb.directory.DirectorySubspace; public class DirectoryTest { public static void main(String[] args) throws Exception { try { - FDB fdb = FDB.selectAPIVersion(610); + FDB fdb = FDB.selectAPIVersion(620); try(Database db = fdb.open()) { runTests(db); } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/Example.java b/bindings/java/src/test/com/apple/foundationdb/test/Example.java index d41e19474a..5a31f7c566 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/Example.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/Example.java @@ -26,7 +26,7 @@ import com.apple.foundationdb.tuple.Tuple; public class Example { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(610); + FDB fdb = FDB.selectAPIVersion(620); try(Database db = fdb.open()) { // Run an operation on the database diff --git a/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java b/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java index 1af785de08..78bc725450 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java @@ -31,7 +31,7 @@ public class IterableTest { public static void main(String[] args) throws InterruptedException { final int reps = 1000; try { - FDB fdb = FDB.selectAPIVersion(610); + FDB fdb = FDB.selectAPIVersion(620); try(Database db = fdb.open()) { runTests(reps, db); } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java b/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java index cb9d38467d..9018339175 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java @@ -34,7 +34,7 @@ import com.apple.foundationdb.tuple.ByteArrayUtil; public class LocalityTests { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(610); + FDB fdb = FDB.selectAPIVersion(620); try(Database database = fdb.open(args[0])) { try(Transaction tr = database.createTransaction()) { String[] keyAddresses = LocalityUtil.getAddressesForKey(tr, "a".getBytes()).join(); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java b/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java index b6ba3cb925..b6c5cfdfaf 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java @@ -43,7 +43,7 @@ public class ParallelRandomScan { private static final int PARALLELISM_STEP = 5; public static void main(String[] args) throws InterruptedException { - FDB api = FDB.selectAPIVersion(610); + FDB api = FDB.selectAPIVersion(620); try(Database database = api.open(args[0])) { for(int i = PARALLELISM_MIN; i <= PARALLELISM_MAX; i += PARALLELISM_STEP) { runTest(database, i, ROWS, DURATION_MS); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java b/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java index 21a38cdf82..1ce5f657c3 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java @@ -34,7 +34,7 @@ import com.apple.foundationdb.Transaction; import com.apple.foundationdb.async.AsyncIterable; public class RangeTest { - private static final int API_VERSION = 610; + private static final int API_VERSION = 620; public static void main(String[] args) { System.out.println("About to use version " + API_VERSION); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java index f4660c273e..44b1ee7b77 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java @@ -34,7 +34,7 @@ public class SerialInsertion { private static final int NODES = 1000000; public static void main(String[] args) { - FDB api = FDB.selectAPIVersion(610); + FDB api = FDB.selectAPIVersion(620); try(Database database = api.open()) { long start = System.currentTimeMillis(); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java index ad9d42b900..49e51af299 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java @@ -39,7 +39,7 @@ public class SerialIteration { private static final int THREAD_COUNT = 1; public static void main(String[] args) throws InterruptedException { - FDB api = FDB.selectAPIVersion(610); + FDB api = FDB.selectAPIVersion(620); try(Database database = api.open(args[0])) { for(int i = 1; i <= THREAD_COUNT; i++) { runThreadedTest(database, i); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java index 43d13d9dc6..c733f54e04 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java @@ -30,7 +30,7 @@ public class SerialTest { public static void main(String[] args) throws InterruptedException { final int reps = 1000; try { - FDB fdb = FDB.selectAPIVersion(610); + FDB fdb = FDB.selectAPIVersion(620); try(Database db = fdb.open()) { runTests(reps, db); } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java b/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java index d7532f6bfe..16acc7c1a7 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java @@ -39,7 +39,7 @@ public class SnapshotTransactionTest { private static final Subspace SUBSPACE = new Subspace(Tuple.from("test", "conflict_ranges")); public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(610); + FDB fdb = FDB.selectAPIVersion(620); try(Database db = fdb.open()) { snapshotReadShouldNotConflict(db); snapshotShouldNotAddConflictRange(db); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java b/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java index 7fdc7b9ff5..8d13aadde1 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java @@ -54,6 +54,7 @@ enum StackOperation { GET_KEY, GET_READ_VERSION, GET_COMMITTED_VERSION, + GET_APPROXIMATE_SIZE, GET_VERSIONSTAMP, SET_READ_VERSION, ON_ERROR, diff --git a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java index 20c9770bc2..96eac7a843 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java @@ -261,6 +261,10 @@ public class StackTester { inst.context.lastVersion = inst.tr.getCommittedVersion(); inst.push("GOT_COMMITTED_VERSION".getBytes()); } + else if(op == StackOperation.GET_APPROXIMATE_SIZE) { + Long size = inst.tr.getApproximateSize().join(); + inst.push("GOT_APPROXIMATE_SIZE".getBytes()); + } else if(op == StackOperation.GET_VERSIONSTAMP) { inst.push(inst.tr.getVersionstamp()); } @@ -434,13 +438,14 @@ public class StackTester { db.options().setMaxWatches(10001); db.options().setDatacenterId("dc_id"); db.options().setMachineId("machine_id"); + db.options().setSnapshotRywEnable(); + db.options().setSnapshotRywDisable(); + db.options().setTransactionLoggingMaxFieldLength(1000); db.options().setTransactionTimeout(100000); db.options().setTransactionTimeout(0); db.options().setTransactionMaxRetryDelay(100); db.options().setTransactionRetryLimit(10); db.options().setTransactionRetryLimit(-1); - db.options().setSnapshotRywEnable(); - db.options().setSnapshotRywDisable(); tr.options().setPrioritySystemImmediate(); tr.options().setPriorityBatch(); @@ -449,6 +454,7 @@ public class StackTester { tr.options().setReadYourWritesDisable(); tr.options().setReadSystemKeys(); tr.options().setAccessSystemKeys(); + tr.options().setTransactionLoggingMaxFieldLength(1000); tr.options().setTimeout(60*1000); tr.options().setRetryLimit(50); tr.options().setMaxRetryDelay(100); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java b/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java index ab0ed4f91a..599272f73c 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java @@ -50,7 +50,7 @@ public class TupleTest { public static void main(String[] args) throws NoSuchFieldException { final int reps = 1000; try { - FDB fdb = FDB.selectAPIVersion(610); + FDB fdb = FDB.selectAPIVersion(620); addMethods(); comparisons(); emptyTuple(); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java b/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java index 670d94629a..48a74f4d09 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java @@ -32,7 +32,7 @@ import com.apple.foundationdb.tuple.Versionstamp; public class VersionstampSmokeTest { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(610); + FDB fdb = FDB.selectAPIVersion(620); try(Database db = fdb.open()) { db.run(tr -> { tr.clear(Tuple.from("prefix").range()); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java b/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java index 640b1e3749..31076f1305 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java @@ -34,7 +34,7 @@ import com.apple.foundationdb.Transaction; public class WatchTest { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(610); + FDB fdb = FDB.selectAPIVersion(620); try(Database database = fdb.open(args[0])) { database.options().setLocationCacheSize(42); try(Transaction tr = database.createTransaction()) { diff --git a/bindings/python/fdb/__init__.py b/bindings/python/fdb/__init__.py index abbc7ec18c..82ebb0d6e7 100644 --- a/bindings/python/fdb/__init__.py +++ b/bindings/python/fdb/__init__.py @@ -52,7 +52,7 @@ def get_api_version(): def api_version(ver): - header_version = 610 + header_version = 620 if '_version' in globals(): if globals()['_version'] != ver: diff --git a/bindings/python/fdb/impl.py b/bindings/python/fdb/impl.py index 77c7a74d13..c7d33f0fe9 100644 --- a/bindings/python/fdb/impl.py +++ b/bindings/python/fdb/impl.py @@ -405,7 +405,7 @@ class TransactionRead(_FDBBase): def get_read_version(self): """Get the read version of the transaction.""" - return FutureVersion(self.capi.fdb_transaction_get_read_version(self.tpointer)) + return FutureInt64(self.capi.fdb_transaction_get_read_version(self.tpointer)) def get(self, key): key = keyToBytes(key) @@ -541,6 +541,10 @@ class Transaction(TransactionRead): self.capi.fdb_transaction_get_committed_version(self.tpointer, ctypes.byref(version)) return version.value + def get_approximate_size(self): + """Get the approximate commit size of the transaction.""" + return FutureInt64(self.capi.fdb_transaction_get_approximate_size(self.tpointer)) + def get_versionstamp(self): return Key(self.capi.fdb_transaction_get_versionstamp(self.tpointer)) @@ -687,12 +691,12 @@ class FutureVoid(Future): return None -class FutureVersion(Future): +class FutureInt64(Future): def wait(self): self.block_until_ready() - version = ctypes.c_int64() - self.capi.fdb_future_get_version(self.fpointer, ctypes.byref(version)) - return version.value + value = ctypes.c_int64() + self.capi.fdb_future_get_int64(self.fpointer, ctypes.byref(value)) + return value.value class FutureKeyValueArray(Future): @@ -1359,9 +1363,9 @@ def init_c_api(): _capi.fdb_future_get_error.restype = int _capi.fdb_future_get_error.errcheck = check_error_code - _capi.fdb_future_get_version.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int64)] - _capi.fdb_future_get_version.restype = ctypes.c_int - _capi.fdb_future_get_version.errcheck = check_error_code + _capi.fdb_future_get_int64.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int64)] + _capi.fdb_future_get_int64.restype = ctypes.c_int + _capi.fdb_future_get_int64.errcheck = check_error_code _capi.fdb_future_get_key.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.POINTER(ctypes.c_byte)), ctypes.POINTER(ctypes.c_int)] @@ -1453,6 +1457,9 @@ def init_c_api(): _capi.fdb_transaction_get_committed_version.restype = ctypes.c_int _capi.fdb_transaction_get_committed_version.errcheck = check_error_code + _capi.fdb_transaction_get_approximate_size.argtypes = [ctypes.c_void_p] + _capi.fdb_transaction_get_approximate_size.restype = ctypes.c_void_p + _capi.fdb_transaction_get_versionstamp.argtypes = [ctypes.c_void_p] _capi.fdb_transaction_get_versionstamp.restype = ctypes.c_void_p diff --git a/bindings/python/tests/size_limit_tests.py b/bindings/python/tests/size_limit_tests.py index 6445e02e2d..9c71942999 100644 --- a/bindings/python/tests/size_limit_tests.py +++ b/bindings/python/tests/size_limit_tests.py @@ -22,7 +22,7 @@ import fdb import sys if __name__ == '__main__': - fdb.api_version(610) + fdb.api_version(620) @fdb.transactional def setValue(tr, key, value): @@ -34,8 +34,6 @@ def setValueWithLimit(tr, key, value, limit): tr[key] = value def test_size_limit_option(db): - db.options.set_transaction_timeout(2000) # 2 seconds - db.options.set_transaction_retry_limit(3) value = b'a' * 1024 setValue(db, b't1', value) @@ -68,9 +66,33 @@ def test_size_limit_option(db): except fdb.FDBError as e: assert(e.code == 2101) # Transaction exceeds byte limit (2101) +@fdb.transactional +def test_get_approximate_size(tr): + tr[b'key1'] = b'value1' + s1 = tr.get_approximate_size().wait() + + tr[b'key2'] = b'value2' + s2 = tr.get_approximate_size().wait() + assert(s1 < s2) + + tr.clear(b'key3') + s3 = tr.get_approximate_size().wait() + assert(s2 < s3) + + tr.add_read_conflict_key(b'key3+') + s4 = tr.get_approximate_size().wait() + assert(s3 < s4) + + tr.add_write_conflict_key(b'key4') + s5 = tr.get_approximate_size().wait() + assert(s4 < s5) + # Expect a cluster file as input. This test will write to the FDB cluster, so # be aware of potential side effects. if __name__ == '__main__': clusterFile = sys.argv[1] db = fdb.open(clusterFile) + db.options.set_transaction_timeout(2000) # 2 seconds + db.options.set_transaction_retry_limit(3) test_size_limit_option(db) + test_get_approximate_size(db) diff --git a/bindings/python/tests/tester.py b/bindings/python/tests/tester.py index 95aa36ea3e..9fb1829ce7 100644 --- a/bindings/python/tests/tester.py +++ b/bindings/python/tests/tester.py @@ -48,7 +48,7 @@ from cancellation_timeout_tests import test_retry_limits from cancellation_timeout_tests import test_db_retry_limits from cancellation_timeout_tests import test_combinations -from size_limit_tests import test_size_limit_option +from size_limit_tests import test_size_limit_option, test_get_approximate_size random.seed(0) @@ -128,9 +128,13 @@ class Instruction: def test_db_options(db): + db.options.set_location_cache_size(100001) db.options.set_max_watches(100001) db.options.set_datacenter_id("dc_id") db.options.set_machine_id("machine_id") + db.options.set_snapshot_ryw_enable() + db.options.set_snapshot_ryw_disable() + db.options.set_transaction_logging_max_field_length(1000) db.options.set_transaction_timeout(100000) db.options.set_transaction_timeout(0) db.options.set_transaction_timeout(0) @@ -138,8 +142,6 @@ def test_db_options(db): db.options.set_transaction_size_limit(100000) db.options.set_transaction_retry_limit(10) db.options.set_transaction_retry_limit(-1) - db.options.set_snapshot_ryw_enable() - db.options.set_snapshot_ryw_disable() @fdb.transactional @@ -151,6 +153,7 @@ def test_options(tr): tr.options.set_read_your_writes_disable() tr.options.set_read_system_keys() tr.options.set_access_system_keys() + tr.options.set_transaction_logging_max_field_length(1000) tr.options.set_timeout(60 * 1000) tr.options.set_retry_limit(50) tr.options.set_max_retry_delay(100) @@ -475,6 +478,9 @@ class Tester: elif inst.op == six.u("GET_COMMITTED_VERSION"): self.last_version = inst.tr.get_committed_version() inst.push(b"GOT_COMMITTED_VERSION") + elif inst.op == six.u("GET_APPROXIMATE_SIZE"): + approximate_size = inst.tr.get_approximate_size().wait() + inst.push(b"GOT_APPROXIMATE_SIZE") elif inst.op == six.u("GET_VERSIONSTAMP"): inst.push(inst.tr.get_versionstamp()) elif inst.op == six.u("TUPLE_PACK"): @@ -545,8 +551,6 @@ class Tester: inst.push(b"WAITED_FOR_EMPTY") elif inst.op == six.u("UNIT_TESTS"): try: - db.options.set_location_cache_size(100001) - test_db_options(db) test_options(db) test_watches(db) @@ -560,6 +564,7 @@ class Tester: test_predicates() test_size_limit_option(db) + test_get_approximate_size(db) except fdb.FDBError as e: print("Unit tests failed: %s" % e.description) diff --git a/bindings/ruby/lib/fdb.rb b/bindings/ruby/lib/fdb.rb index c2431d8bf3..8fa688066c 100644 --- a/bindings/ruby/lib/fdb.rb +++ b/bindings/ruby/lib/fdb.rb @@ -36,7 +36,7 @@ module FDB end end def self.api_version(version) - header_version = 610 + header_version = 620 if self.is_api_version_selected?() if @@chosen_version != version raise "FDB API already loaded at version #{@@chosen_version}." diff --git a/bindings/ruby/lib/fdbimpl.rb b/bindings/ruby/lib/fdbimpl.rb index dcbe246f89..b1deb1123c 100644 --- a/bindings/ruby/lib/fdbimpl.rb +++ b/bindings/ruby/lib/fdbimpl.rb @@ -85,7 +85,7 @@ module FDB attach_function :fdb_future_set_callback, [ :pointer, :fdb_future_callback, :pointer ], :fdb_error attach_function :fdb_future_get_error, [ :pointer ], :fdb_error - attach_function :fdb_future_get_version, [ :pointer, :pointer ], :fdb_error + attach_function :fdb_future_get_int64, [ :pointer, :pointer ], :fdb_error attach_function :fdb_future_get_key, [ :pointer, :pointer, :pointer ], :fdb_error attach_function :fdb_future_get_value, [ :pointer, :pointer, :pointer, :pointer ], :fdb_error attach_function :fdb_future_get_keyvalue_array, [ :pointer, :pointer, :pointer, :pointer ], :fdb_error @@ -114,6 +114,7 @@ module FDB attach_function :fdb_transaction_watch, [ :pointer, :pointer, :int ], :pointer attach_function :fdb_transaction_commit, [ :pointer ], :pointer attach_function :fdb_transaction_get_committed_version, [ :pointer, :pointer ], :fdb_error + attach_function :fdb_transaction_get_approximate_size, [ :pointer ], :pointer attach_function :fdb_transaction_get_versionstamp, [ :pointer ], :pointer attach_function :fdb_transaction_on_error, [ :pointer, :fdb_error ], :pointer attach_function :fdb_transaction_reset, [ :pointer ], :void @@ -443,11 +444,11 @@ module FDB end end - class Version < LazyFuture + class Int64Future < LazyFuture def getter - version = FFI::MemoryPointer.new :int64 - FDBC.check_error FDBC.fdb_future_get_version(@fpointer, version) - @value = version.read_long_long + val = FFI::MemoryPointer.new :int64 + FDBC.check_error FDBC.fdb_future_get_int64(@fpointer, val) + @value = val.read_long_long end private :getter end @@ -687,7 +688,7 @@ module FDB end def get_read_version - Version.new(FDBC.fdb_transaction_get_read_version @tpointer) + Int64Future.new(FDBC.fdb_transaction_get_read_version @tpointer) end def get(key) @@ -904,6 +905,10 @@ module FDB version.read_long_long end + def get_approximate_size + Int64Future.new(FDBC.fdb_transaction_get_approximate_size @tpointer) + end + def get_versionstamp Key.new(FDBC.fdb_transaction_get_versionstamp(@tpointer)) end diff --git a/bindings/ruby/tests/tester.rb b/bindings/ruby/tests/tester.rb index 829ecf8a5f..ccf863797d 100755 --- a/bindings/ruby/tests/tester.rb +++ b/bindings/ruby/tests/tester.rb @@ -381,6 +381,9 @@ class Tester when "GET_COMMITTED_VERSION" @last_version = inst.tr.get_committed_version inst.push("GOT_COMMITTED_VERSION") + when "GET_APPROXIMATE_SIZE" + size = inst.tr.get_approximate_size.to_i + inst.push("GOT_APPROXIMATE_SIZE") when "GET_VERSIONSTAMP" inst.push(inst.tr.get_versionstamp) when "TUPLE_PACK" @@ -456,14 +459,15 @@ class Tester @db.options.set_max_watches(10001) @db.options.set_datacenter_id("dc_id") @db.options.set_machine_id("machine_id") + @db.options.set_snapshot_ryw_enable() + @db.options.set_snapshot_ryw_disable() + @db.options.set_transaction_logging_max_field_length(1000) @db.options.set_transaction_timeout(100000) @db.options.set_transaction_timeout(0) @db.options.set_transaction_max_retry_delay(100) @db.options.set_transaction_size_limit(100000) @db.options.set_transaction_retry_limit(10) @db.options.set_transaction_retry_limit(-1) - @db.options.set_snapshot_ryw_enable() - @db.options.set_snapshot_ryw_disable() @db.transact do |tr| tr.options.set_priority_system_immediate @@ -473,6 +477,7 @@ class Tester tr.options.set_read_your_writes_disable tr.options.set_read_system_keys tr.options.set_access_system_keys + tr.options.set_transaction_logging_max_field_length(1000) tr.options.set_timeout(60*1000) tr.options.set_retry_limit(50) tr.options.set_max_retry_delay(100) diff --git a/build/cmake/package_tester/fdb_c_app/app.c b/build/cmake/package_tester/fdb_c_app/app.c index 549bd935d3..80d05f591a 100644 --- a/build/cmake/package_tester/fdb_c_app/app.c +++ b/build/cmake/package_tester/fdb_c_app/app.c @@ -1,7 +1,7 @@ -#define FDB_API_VERSION 610 +#define FDB_API_VERSION 620 #include int main(int argc, char* argv[]) { - fdb_select_api_version(610); + fdb_select_api_version(620); return 0; } diff --git a/build/cmake/package_tester/modules/tests.sh b/build/cmake/package_tester/modules/tests.sh index 5dd39b21e0..8eef8c4e07 100644 --- a/build/cmake/package_tester/modules/tests.sh +++ b/build/cmake/package_tester/modules/tests.sh @@ -65,7 +65,7 @@ then python setup.py install successOr "Installing python bindings failed" popd - python -c 'import fdb; fdb.api_version(610)' + python -c 'import fdb; fdb.api_version(620)' successOr "Loading python bindings failed" # Test cmake and pkg-config integration: https://github.com/apple/foundationdb/issues/1483 diff --git a/cmake/AddFdbTest.cmake b/cmake/AddFdbTest.cmake index 3835f82350..a20fef78ed 100644 --- a/cmake/AddFdbTest.cmake +++ b/cmake/AddFdbTest.cmake @@ -84,13 +84,13 @@ function(add_fdb_test) if (NOT "${ADD_FDB_TEST_TEST_NAME}" STREQUAL "") set(test_name ${ADD_FDB_TEST_TEST_NAME}) endif() - math(EXPR test_idx "${CURRENT_TEST_INDEX} + 1") + math(EXPR test_idx "${CURRENT_TEST_INDEX} + ${NUM_TEST_FILES}") set(CURRENT_TEST_INDEX "${test_idx}" PARENT_SCOPE) # set( PARENT_SCOPE) doesn't set the # value in this scope (only in the parent scope). So # if the value was undefined before, it will still be # undefined. - math(EXPR assigned_id "${test_idx} - 1") + math(EXPR assigned_id "${test_idx} - ${NUM_TEST_FILES}") if(ADD_FDB_TEST_UNIT) message(STATUS "ADDING UNIT TEST ${assigned_id} ${test_name}") @@ -106,6 +106,10 @@ function(add_fdb_test) if (ENABLE_BUGGIFY) set(BUGGIFY_OPTION "-B") endif() + set(VALGRIND_OPTION "") + if (USE_VALGRIND) + set(VALGRIND_OPTION "--use-valgrind") + endif() list(TRANSFORM ADD_FDB_TEST_TEST_FILES PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/") add_test(NAME ${test_name} COMMAND $ ${TestRunner} @@ -120,6 +124,7 @@ function(add_fdb_test) --seed ${SEED} --test-number ${assigned_id} ${BUGGIFY_OPTION} + ${VALGRIND_OPTION} ${ADD_FDB_TEST_TEST_FILES} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) get_filename_component(test_dir_full ${first_file} DIRECTORY) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 0d7d6a67a8..ebd81fa5a1 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -1,5 +1,4 @@ set(USE_GPERFTOOLS OFF CACHE BOOL "Use gperfools for profiling") -set(PORTABLE_BINARY OFF CACHE BOOL "Create a binary that runs on older OS versions") set(USE_VALGRIND OFF CACHE BOOL "Compile for valgrind usage") set(ALLOC_INSTRUMENTATION OFF CACHE BOOL "Instrument alloc") set(WITH_UNDODB OFF CACHE BOOL "Use rr or undodb") @@ -8,6 +7,13 @@ set(FDB_RELEASE OFF CACHE BOOL "This is a building of a final release") set(USE_LD "LD" CACHE STRING "The linker to use for building: can be LD (system default, default choice), GOLD, or LLD") set(USE_LIBCXX OFF CACHE BOOL "Use libc++") set(USE_CCACHE OFF CACHE BOOL "Use ccache for compilation if available") +set(RELATIVE_DEBUG_PATHS OFF CACHE BOOL "Use relative file paths in debug info") +set(STATIC_LINK_LIBCXX ON CACHE BOOL "Statically link libstdcpp/libc++") + +set(rel_debug_paths OFF) +if(RELATIVE_DEBUG_PATHS) + set(rel_debug_paths ON) +endif() if(USE_GPERFTOOLS) find_package(Gperftools REQUIRED) @@ -103,6 +109,10 @@ else() set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld -Wl,--disable-new-dtags") endif() + if(rel_debug_paths) + add_compile_options("-fdebug-prefix-map=${CMAKE_SOURCE_DIR}=." "-fdebug-prefix-map=${CMAKE_BINARY_DIR}=.") + endif() + # we always compile with debug symbols. CPack will strip them out # and create a debuginfo rpm add_compile_options(-ggdb -fno-omit-frame-pointer) @@ -121,6 +131,11 @@ else() set(CMAKE_SHARED_LINKER_FLAGS "-static-libstdc++ -static-libgcc ${CMAKE_SHARED_LINKER_FLAGS}") set(CMAKE_EXE_LINKER_FLAGS "-static-libstdc++ -static-libgcc ${CMAKE_EXE_LINKER_FLAGS}") endif() + if(STATIC_LINK_LIBCXX) + if (NOT USE_LIBCXX AND NOT APPLE) + add_link_options(-static-libstdc++ -static-libgcc) + endif() + endif() # Instruction sets we require to be supported by the CPU add_compile_options( -maes diff --git a/cmake/FlowCommands.cmake b/cmake/FlowCommands.cmake index d4ddd41c05..97e3781eb3 100644 --- a/cmake/FlowCommands.cmake +++ b/cmake/FlowCommands.cmake @@ -158,7 +158,7 @@ function(add_flow_target) message(FATAL_ERROR "No sources provided") endif() if(OPEN_FOR_IDE) - set(sources ${AFT_SRCS} ${AFT_DISABLE_ACTOR_WRITHOUT_WAIT_WARNING} ${AFT_ADDL_SRCS}) + set(sources ${AFT_SRCS} ${AFT_DISABLE_ACTOR_WITHOUT_WAIT_WARNING} ${AFT_ADDL_SRCS}) add_library(${AFT_NAME} OBJECT ${sources}) else() foreach(src IN LISTS AFT_SRCS AFT_DISABLE_ACTOR_WITHOUT_WAIT_WARNING) diff --git a/cmake/Sandbox.conf.cmake b/cmake/Sandbox.conf.cmake index d06ec0ca27..b8c5929248 100644 --- a/cmake/Sandbox.conf.cmake +++ b/cmake/Sandbox.conf.cmake @@ -30,6 +30,7 @@ logdir = ${CMAKE_BINARY_DIR}/sandbox/logs # class = # memory = 8GiB # storage_memory = 1GiB +# cache_memory = 2GiB # metrics_cluster = # metrics_prefix = diff --git a/documentation/sphinx/source/api-c.rst b/documentation/sphinx/source/api-c.rst index 1e5d0243dc..bd8eb33189 100644 --- a/documentation/sphinx/source/api-c.rst +++ b/documentation/sphinx/source/api-c.rst @@ -49,6 +49,8 @@ .. |max-retry-delay-database-option| replace:: FIXME .. |transaction-size-limit-database-option| replace:: FIXME .. |timeout-database-option| replace:: FIXME +.. |transaction-logging-max-field-length-database-option| replace:: FIXME +.. |transaction-logging-max-field-length-transaction-option| replace:: FIXME .. include:: api-common.rst.inc @@ -129,7 +131,7 @@ API versioning Prior to including ``fdb_c.h``, you must define the ``FDB_API_VERSION`` macro. This, together with the :func:`fdb_select_api_version()` function, allows programs written against an older version of the API to compile and run with newer versions of the C library. The current version of the FoundationDB C API is |api-version|. :: - #define FDB_API_VERSION 610 + #define FDB_API_VERSION 620 #include .. function:: fdb_error_t fdb_select_api_version(int version) @@ -289,9 +291,9 @@ See :ref:`developer-guide-programming-with-futures` for further (language-indepe |future-get-return1|. -.. function:: fdb_error_t fdb_future_get_version(FDBFuture* future, int64_t* out_version) +.. function:: fdb_error_t fdb_future_get_int64(FDBFuture* future, int64_t* out) - Extracts a version from an :type:`FDBFuture` into a caller-provided variable of type ``int64_t``. |future-warning| + Extracts a 64-bit integer from an :type:`FDBFuture*` into a caller-provided variable of type ``int64_t``. |future-warning| |future-get-return1| |future-get-return2|. @@ -445,7 +447,7 @@ Applications must provide error handling and an appropriate retry loop around th .. function:: FDBFuture* fdb_transaction_get_read_version(FDBTransaction* transaction) - |future-return0| the transaction snapshot read version. |future-return1| call :func:`fdb_future_get_version()` to extract the version into an int64_t that you provide, |future-return2| + |future-return0| the transaction snapshot read version. |future-return1| call :func:`fdb_future_get_int64()` to extract the version into an int64_t that you provide, |future-return2| The transaction obtains a snapshot read version automatically at the time of the first call to :func:`fdb_transaction_get_*()` (including this one) and (unless causal consistency has been deliberately compromised by transaction options) is guaranteed to represent all transactions which were reported committed before that call. @@ -718,6 +720,12 @@ Applications must provide error handling and an appropriate retry loop around th Most applications will not call this function. +.. function:: FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* transaction) + + |future-return0| the approximate transaction size so far in the returned future, which is the summation of the estimated size of mutations, read conflict ranges, and write conflict ranges. |future-return1| call :func:`fdb_future_get_int64()` to extract the size, |future-return2| + + This can be called multiple times before the transaction is committed. + .. function:: FDBFuture* fdb_transaction_get_versionstamp(FDBTransaction* transaction) |future-return0| the versionstamp which was used by any versionstamp operations in this transaction. |future-return1| call :func:`fdb_future_get_key()` to extract the key, |future-return2| diff --git a/documentation/sphinx/source/api-common.rst.inc b/documentation/sphinx/source/api-common.rst.inc index 3c99c45382..87f7fdb688 100644 --- a/documentation/sphinx/source/api-common.rst.inc +++ b/documentation/sphinx/source/api-common.rst.inc @@ -147,7 +147,7 @@ .. |atomic-versionstamps-tuple-warning-value| replace:: At this time, versionstamped values are not compatible with the Tuple layer except in Java, Python, and Go. Note that this implies versionstamped values may not be used with the Subspace and Directory layers except in those languages. -.. |api-version| replace:: 610 +.. |api-version| replace:: 620 .. |streaming-mode-blurb1| replace:: When using |get-range-func| and similar interfaces, API clients can request large ranges of the database to iterate over. Making such a request doesn't necessarily mean that the client will consume all of the data in the range - sometimes the client doesn't know how far it intends to iterate in advance. FoundationDB tries to balance latency and bandwidth by requesting data for iteration in batches. @@ -326,6 +326,10 @@ If this option has been set more times with this database than the disable option, snapshot reads will *not* see the effects of prior writes in the same transaction. Disabling this option is equivalent to calling |snapshot-ryw-disable-transaction-option| on each transaction created by this database. +.. |option-db-tr-transaction-logging-max-field-length-blurb| replace:: + + Sets the maximum escaped length of key and value fields to be logged to the trace file via the LOG_TRANSACTION option. This is equivalent to calling |transaction-logging-max-field-length-transaction-option| on each transaction created by this database. + .. |transaction-options-blurb| replace:: Transaction options alter the behavior of FoundationDB transactions. FoundationDB defaults to extremely safe transaction behavior, and we have worked hard to make the performance excellent with the default setting, so you should not often need to use transaction options. @@ -411,7 +415,7 @@ .. |option-set-timeout-blurb3| replace:: - Prior to API version 610, like all other transaction options, a timeout must be reset after a call to |on-error-func|. Note that resetting this option resets only the timeout *duration*, not the starting point from which the time is measured. If the API version is 610 or newer, then the timeout is not reset. This allows the user to specify a timeout for specific transactions that is longer than the timeout specified by |timeout-database-option|. Note that at all API versions, it is safe and legal to call this option after each call to |on-error-func|, so most code written assuming the older behavior can be upgraded without requiring any modification. This also means that there is no need to introduce logic to conditionally set this option within retry loops. One can set the default timeout for all transactions by calling |timeout-database-option|. + Prior to API version 610, like all other transaction options, a timeout must be reset after a call to |on-error-func|. Note that resetting this option resets only the timeout *duration*, not the starting point from which the time is measured. If the API version is 610 or newer, then the timeout is not reset. This allows the user to specify a timeout for specific transactions that is longer than the timeout specified by |timeout-database-option|. Note that at all API versions, it is safe and legal to call this option after each call to |on-error-func|, so most code written assuming the older behavior can be upgraded without requiring any modification. This also means that there is no need to introduce logic to conditionally set this option within retry loops. One can set the default timeout for all transactions by calling |timeout-database-option|. .. |option-next-write-no-write-conflict-range-blurb| replace:: @@ -421,6 +425,10 @@ Care needs to be taken when using this option on a transaction that is shared between multiple threads. When setting this option, write conflict ranges will be disabled on the next write operation, regardless of what thread it is on. +.. |option-set-transaction-logging-max-field-length-blurb| replace:: + + Sets the maximum escaped length of key and value fields to be logged to the trace file via the LOG_TRANSACTION option, after which the field will be truncated. A negative value disables truncation. One can set the default max field length for all transactions by calling |transaction-logging-max-field-length-database-option|. + .. |future-blurb1| replace:: Many FoundationDB API functions return "future" objects. A brief overview of futures is included in the :doc:`class scheduling tutorial `. Most future objects behave just like a normal object, but block when you use them for the first time if the asynchronous function which returned the future has not yet completed its action. A future object is considered ready when either a value is available, or when an error has occurred. diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst index 68ae70b1fa..a12e9b2cd4 100644 --- a/documentation/sphinx/source/api-python.rst +++ b/documentation/sphinx/source/api-python.rst @@ -25,6 +25,7 @@ .. |timeout-database-option| replace:: :func:`Database.options.set_transaction_timeout` .. |max-retry-delay-database-option| replace:: :func:`Database.options.set_transaction_max_retry_delay` .. |transaction-size-limit-database-option| replace:: :func:`Database.options.set_transaction_size_limit` +.. |transaction-logging-max-field-length-database-option| replace:: :func:`Database.options.set_transaction_logging_max_field_length` .. |snapshot-ryw-enable-database-option| replace:: :func:`Database.options.set_snapshot_ryw_enable` .. |snapshot-ryw-disable-database-option| replace:: :func:`Database.options.set_snapshot_ryw_disable` .. |future-type-string| replace:: a :ref:`future ` @@ -35,6 +36,7 @@ .. |size-limit-transaction-option| replace:: :func:`Transaction.options.set_size_limit` .. |snapshot-ryw-enable-transaction-option| replace:: :func:`Transaction.options.set_snapshot_ryw_enable` .. |snapshot-ryw-disable-transaction-option| replace:: :func:`Transaction.options.set_snapshot_ryw_disable` +.. |transaction-logging-max-field-length-transaction-option| replace:: :func:`Transaction.options.set_transaction_logging_max_field_length` .. |lazy-iterator-object| replace:: generator .. |key-meth| replace:: :meth:`Subspace.key` .. |directory-subspace| replace:: :ref:`DirectorySubspace ` @@ -100,7 +102,7 @@ Opening a database After importing the ``fdb`` module and selecting an API version, you probably want to open a :class:`Database` using :func:`open`:: import fdb - fdb.api_version(610) + fdb.api_version(620) db = fdb.open() .. function:: open( cluster_file=None, event_model=None ) @@ -384,6 +386,10 @@ Database options |option-db-tr-size-limit-blurb| +.. method:: Database.options.set_transaction_logging_max_field_length(size_limit) + + |option-db-tr-transaction-logging-max-field-length-blurb| + .. method:: Database.options.set_snapshot_ryw_enable() |option-db-snapshot-ryw-enable-blurb| @@ -855,6 +861,10 @@ Transaction options |option-set-timeout-blurb3| +.. method:: Transaction.options.set_transaction_logging_max_field_length(size_limit) + + |option-set-transaction-logging-max-field-length-blurb| + .. _api-python-future: Future objects diff --git a/documentation/sphinx/source/api-ruby.rst b/documentation/sphinx/source/api-ruby.rst index cc35ad68b8..d4264a4d84 100644 --- a/documentation/sphinx/source/api-ruby.rst +++ b/documentation/sphinx/source/api-ruby.rst @@ -25,6 +25,7 @@ .. |transaction-size-limit-database-option| replace:: :func:`Database.options.set_transaction_size_limit` .. |snapshot-ryw-enable-database-option| replace:: :meth:`Database.options.set_snapshot_ryw_enable` .. |snapshot-ryw-disable-database-option| replace:: :meth:`Database.options.set_snapshot_ryw_disable` +.. |transaction-logging-max-field-length-database-option| replace:: :meth:`Database.options.set_transaction_logging_max_field_length` .. |future-type-string| replace:: a :class:`Future` .. |read-your-writes-disable-option| replace:: :meth:`Transaction.options.set_read_your_writes_disable` .. |retry-limit-transaction-option| replace:: :meth:`Transaction.options.set_retry_limit` @@ -33,6 +34,7 @@ .. |size-limit-transaction-option| replace:: :meth:`Transaction.options.set_size_limit` .. |snapshot-ryw-enable-transaction-option| replace:: :meth:`Transaction.options.set_snapshot_ryw_enable` .. |snapshot-ryw-disable-transaction-option| replace:: :meth:`Transaction.options.set_snapshot_ryw_disable` +.. |transaction-logging-max-field-length-transaction-option| replace:: :meth:`Transaction.options.set_transaction_logging_max_field_length` .. |lazy-iterator-object| replace:: :class:`Enumerator` .. |key-meth| replace:: :meth:`Subspace.key` .. |directory-subspace| replace:: :class:`DirectorySubspace` @@ -89,7 +91,7 @@ Opening a database After requiring the ``FDB`` gem and selecting an API version, you probably want to open a :class:`Database` using :func:`open`:: require 'fdb' - FDB.api_version 610 + FDB.api_version 620 db = FDB.open .. function:: open( cluster_file=nil ) -> Database @@ -380,6 +382,10 @@ Database options |option-db-tr-size-limit-blurb| +.. method:: Database.options.set_transaction_logging_max_field_length(size_limit) -> nil + + |option-db-tr-transaction-logging-max-field-length-blurb| + .. method:: Database.options.set_snapshot_ryw_enable() -> nil |option-db-snapshot-ryw-enable-blurb| @@ -797,6 +803,10 @@ Transaction options |option-set-timeout-blurb3| +.. method:: Transaction.options.set_transaction_logging_max_field_length(size_limit) -> nil + + |option-set-transaction-logging-max-field-length-blurb| + .. _transact: The transact method diff --git a/documentation/sphinx/source/class-scheduling-go.rst b/documentation/sphinx/source/class-scheduling-go.rst index 73ec379ea4..a0d103769b 100644 --- a/documentation/sphinx/source/class-scheduling-go.rst +++ b/documentation/sphinx/source/class-scheduling-go.rst @@ -29,7 +29,7 @@ Before using the API, we need to specify the API version. This allows programs t .. code-block:: go - fdb.MustAPIVersion(610) + fdb.MustAPIVersion(620) Next, we open a FoundationDB database. The API will connect to the FoundationDB cluster indicated by the :ref:`default cluster file `. @@ -78,7 +78,7 @@ If this is all working, it looks like we are ready to start building a real appl func main() { // Different API versions may expose different runtime behaviors. - fdb.MustAPIVersion(610) + fdb.MustAPIVersion(620) // Open the default database from the system cluster db := fdb.MustOpenDefault() @@ -666,7 +666,7 @@ Here's the code for the scheduling tutorial: } func main() { - fdb.MustAPIVersion(610) + fdb.MustAPIVersion(620) db := fdb.MustOpenDefault() db.Options().SetTransactionTimeout(60000) // 60,000 ms = 1 minute db.Options().SetTransactionRetryLimit(100) diff --git a/documentation/sphinx/source/class-scheduling-java.rst b/documentation/sphinx/source/class-scheduling-java.rst index aef201c8c8..75db289e1a 100644 --- a/documentation/sphinx/source/class-scheduling-java.rst +++ b/documentation/sphinx/source/class-scheduling-java.rst @@ -30,7 +30,7 @@ Before using the API, we need to specify the API version. This allows programs t private static final Database db; static { - fdb = FDB.selectAPIVersion(610); + fdb = FDB.selectAPIVersion(620); db = fdb.open(); } @@ -66,7 +66,7 @@ If this is all working, it looks like we are ready to start building a real appl private static final Database db; static { - fdb = FDB.selectAPIVersion(610); + fdb = FDB.selectAPIVersion(620); db = fdb.open(); } @@ -441,7 +441,7 @@ Here's the code for the scheduling tutorial: private static final Database db; static { - fdb = FDB.selectAPIVersion(610); + fdb = FDB.selectAPIVersion(620); db = fdb.open(); db.options().setTransactionTimeout(60000); // 60,000 ms = 1 minute db.options().setRetryLimit(100); diff --git a/documentation/sphinx/source/class-scheduling-ruby.rst b/documentation/sphinx/source/class-scheduling-ruby.rst index 4a38e65898..345678887c 100644 --- a/documentation/sphinx/source/class-scheduling-ruby.rst +++ b/documentation/sphinx/source/class-scheduling-ruby.rst @@ -23,7 +23,7 @@ Open a Ruby interactive interpreter and import the FoundationDB API module:: Before using the API, we need to specify the API version. This allows programs to maintain compatibility even if the API is modified in future versions:: - > FDB.api_version 610 + > FDB.api_version 620 => nil Next, we open a FoundationDB database. The API will connect to the FoundationDB cluster indicated by the :ref:`default cluster file `. :: @@ -46,7 +46,7 @@ If this is all working, it looks like we are ready to start building a real appl .. code-block:: ruby require 'fdb' - FDB.api_version 610 + FDB.api_version 620 @db = FDB.open @db['hello'] = 'world' print 'hello ', @db['hello'] @@ -373,7 +373,7 @@ Here's the code for the scheduling tutorial: require 'fdb' - FDB.api_version 610 + FDB.api_version 620 #################################### ## Initialization ## diff --git a/documentation/sphinx/source/class-scheduling.rst b/documentation/sphinx/source/class-scheduling.rst index b4558f2ff2..c13678a23f 100644 --- a/documentation/sphinx/source/class-scheduling.rst +++ b/documentation/sphinx/source/class-scheduling.rst @@ -30,7 +30,7 @@ Open a Python interactive interpreter and import the FoundationDB API module:: Before using the API, we need to specify the API version. This allows programs to maintain compatibility even if the API is modified in future versions:: - >>> fdb.api_version(610) + >>> fdb.api_version(620) Next, we open a FoundationDB database. The API will connect to the FoundationDB cluster indicated by the :ref:`default cluster file `. :: @@ -48,7 +48,7 @@ When this command returns without exception, the modification is durably stored If this is all working, it looks like we are ready to start building a real application. For reference, here's the full code for "hello world":: import fdb - fdb.api_version(610) + fdb.api_version(620) db = fdb.open() db[b'hello'] = b'world' print 'hello', db[b'hello'] @@ -91,7 +91,7 @@ FoundationDB includes a few tools that make it easy to model data using this app opening a :ref:`directory ` in the database:: import fdb - fdb.api_version(610) + fdb.api_version(620) db = fdb.open() scheduling = fdb.directory.create_or_open(db, ('scheduling',)) @@ -337,7 +337,7 @@ Here's the code for the scheduling tutorial:: import fdb import fdb.tuple - fdb.api_version(610) + fdb.api_version(620) #################################### diff --git a/documentation/sphinx/source/configuration.rst b/documentation/sphinx/source/configuration.rst index 02ef379328..873ad61522 100644 --- a/documentation/sphinx/source/configuration.rst +++ b/documentation/sphinx/source/configuration.rst @@ -237,6 +237,7 @@ Contains settings applicable to all processes (e.g. fdbserver, backup_agent). Th # class = # memory = 8GiB # storage_memory = 1GiB + # cache_memory = 2GiB # locality_machineid = # locality_zoneid = # locality_data_hall = @@ -256,8 +257,9 @@ Contains default parameters for all fdbserver processes on this machine. These s * ``logsize``: Roll over to a new log file after the current log file reaches the specified size. The default value is 10MiB. * ``maxlogssize``: Delete the oldest log file when the total size of all log files exceeds the specified size. If set to 0B, old log files will not be deleted. The default value is 100MiB. * ``class``: Process class specifying the roles that will be taken in the cluster. Recommended options are ``storage``, ``transaction``, ``stateless``. See :ref:`guidelines-process-class-config` for process class config recommendations. -* ``memory``: Maximum memory used by the process. The default value is 8GiB. When specified without a unit, MiB is assumed. This parameter does not change the memory allocation of the program. Rather, it sets a hard limit beyond which the process will kill itself and be restarted. The default value of 8GiB is double the intended memory usage in the default configuration (providing an emergency buffer to deal with memory leaks or similar problems). It is *not* recommended to decrease the value of this parameter below its default value. It may be *increased* if you wish to allocate a very large amount of storage engine memory or cache. In particular, when the ``storage_memory`` parameter is increased, the ``memory`` parameter should be increased by an equal amount. -* ``storage_memory``: Maximum memory used for data storage. This parameter is used *only* with memory storage engine, not the ssd storage engine. The default value is 1GiB. When specified without a unit, MB is assumed. Clusters will be restricted to using this amount of memory per process for purposes of data storage. Memory overhead associated with storing the data is counted against this total. If you increase the ``storage_memory``, you should also increase the ``memory`` parameter by the same amount. +* ``memory``: Maximum memory used by the process. The default value is 8GiB. When specified without a unit, MiB is assumed. This parameter does not change the memory allocation of the program. Rather, it sets a hard limit beyond which the process will kill itself and be restarted. The default value of 8GiB is double the intended memory usage in the default configuration (providing an emergency buffer to deal with memory leaks or similar problems). It is *not* recommended to decrease the value of this parameter below its default value. It may be *increased* if you wish to allocate a very large amount of storage engine memory or cache. In particular, when the ``storage_memory`` or ``cache_memory`` parameters are increased, the ``memory`` parameter should be increased by an equal amount. +* ``storage_memory``: Maximum memory used for data storage. This parameter is used *only* with memory storage engine, not the ssd storage engine. The default value is 1GiB. When specified without a unit, MB is assumed. Clusters will be restricted to using this amount of memory per process for purposes of data storage. Memory overhead associated with storing the data is counted against this total. If you increase the ``storage_memory`` parameter, you should also increase the ``memory`` parameter by the same amount. +* ``cache_memory``: Maximum memory used for caching pages from disk. The default value is 2GiB. When specified without a unit, MiB is assumed. If you increase the ``cache_memory`` parameter, you should also increase the ``memory`` parameter by the same amount. * ``locality_machineid``: Machine identifier key. All processes on a machine should share a unique id. By default, processes on a machine determine a unique id to share. This does not generally need to be set. * ``locality_zoneid``: Zone identifier key. Processes that share a zone id are considered non-unique for the purposes of data replication. If unset, defaults to machine id. * ``locality_dcid``: Datacenter identifier key. All processes physically located in a datacenter should share the id. No default value. If you are depending on datacenter based replication this must be set on all processes. @@ -316,19 +318,19 @@ Single datacenter modes +==============================+==+=================+=================+================+ | Best for | | 1-2 machines | 3-4 machines | 5+ machines | +------------------------------+--+-----------------+-----------------+----------------+ -| Replication | | 1 copy | 2 copy | 3 copy | +| Total Replicas | | 1 copy | 2 copies | 3 copies | +------------------------------+--+-----------------+-----------------+----------------+ -| # live machines | | | | | +| Live machines required | | | | | | to make progress | | 1 | 2 | 3 | +------------------------------+--+-----------------+-----------------+----------------+ -| Minimum # of machines | | | | | +| Required machines | | | | | | for fault tolerance | | impossible | 3 | 4 | +------------------------------+--+-----------------+-----------------+----------------+ -| Ideal # of | | | | | +| Ideal number of | | | | | | coordination servers | | 1 | 3 | 5 | +------------------------------+--+-----------------+-----------------+----------------+ -| # simultaneous failures | | | | | -| after which data may be lost | | any machine | 2+ machines | 3+ machines | +| Simultaneous failures | | | | | +| after which data may be lost | | any process | 2+ machines | 3+ machines | +------------------------------+--+-----------------+-----------------+----------------+ In the three single datacenter redundancy modes, FoundationDB replicates data across the required number of machines in the cluster, but without aiming for datacenter redundancy. Although machines may be placed in more than one datacenter, the cluster will not be tolerant of datacenter-correlated failures. diff --git a/documentation/sphinx/source/data-modeling.rst b/documentation/sphinx/source/data-modeling.rst index 51867006af..8fcde06958 100644 --- a/documentation/sphinx/source/data-modeling.rst +++ b/documentation/sphinx/source/data-modeling.rst @@ -51,6 +51,8 @@ .. |max-retry-delay-database-option| replace:: FIXME .. |transaction-size-limit-database-option| replace:: FIXME .. |timeout-database-option| replace:: FIXME +.. |transaction-logging-max-field-length-transaction-option| replace:: FIXME +.. |transaction-logging-max-field-length-database-option| replace:: FIXME .. include:: api-common.rst.inc diff --git a/documentation/sphinx/source/developer-guide.rst b/documentation/sphinx/source/developer-guide.rst index e8d6335b6f..0d7f24f661 100644 --- a/documentation/sphinx/source/developer-guide.rst +++ b/documentation/sphinx/source/developer-guide.rst @@ -51,6 +51,8 @@ .. |max-retry-delay-database-option| replace:: FIXME .. |transaction-size-limit-database-option| replace:: FIXME .. |timeout-database-option| replace:: FIXME +.. |transaction-logging-max-field-length-transaction-option| replace:: FIXME +.. |transaction-logging-max-field-length-database-option| replace:: FIXME .. include:: api-common.rst.inc diff --git a/documentation/sphinx/source/hierarchical-documents-java.rst b/documentation/sphinx/source/hierarchical-documents-java.rst index 1bd5312c14..137dfa8f1f 100644 --- a/documentation/sphinx/source/hierarchical-documents-java.rst +++ b/documentation/sphinx/source/hierarchical-documents-java.rst @@ -69,7 +69,7 @@ Here’s a basic implementation of the recipe. private static final long EMPTY_ARRAY = -1; static { - fdb = FDB.selectAPIVersion(610); + fdb = FDB.selectAPIVersion(620); db = fdb.open(); docSpace = new Subspace(Tuple.from("D")); } diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc index a548194cb3..cd63961396 100644 --- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc +++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc @@ -36,6 +36,7 @@ "roles":[ { "query_queue_max":0, + "local_rate":0, "input_bytes":{ "hz":0.0, "counter":0, @@ -230,7 +231,8 @@ "storage_server_min_free_space", "storage_server_min_free_space_ratio", "log_server_min_free_space", - "log_server_min_free_space_ratio" + "log_server_min_free_space_ratio", + "storage_server_durability_lag" ] }, "description":"The database is not being saturated by the workload." @@ -249,7 +251,8 @@ "storage_server_min_free_space", "storage_server_min_free_space_ratio", "log_server_min_free_space", - "log_server_min_free_space_ratio" + "log_server_min_free_space_ratio", + "storage_server_durability_lag" ] }, "description":"The database is not being saturated by the workload." @@ -337,7 +340,9 @@ "storage_servers_error", "status_incomplete", "layer_status_incomplete", - "database_availability_timeout" + "database_availability_timeout", + "consistencycheck_suspendkey_fetch_timeout", + "consistencycheck_disabled" ] }, "issues":[ diff --git a/documentation/sphinx/source/mr-status.rst b/documentation/sphinx/source/mr-status.rst index dfc063e911..9e11906e71 100644 --- a/documentation/sphinx/source/mr-status.rst +++ b/documentation/sphinx/source/mr-status.rst @@ -128,4 +128,5 @@ min_free_space Running out of space (approaching 100MB limi min_free_space_ratio Running out of space (approaching 5% limit). log_server_min_free_space Log server running out of space (approaching 100MB limit). log_server_min_free_space_ratio Log server running out of space (approaching 5% limit). +storage_server_durability_lag Storage server durable version falling behind. =================================== ==================================================== diff --git a/documentation/sphinx/source/multimaps-java.rst b/documentation/sphinx/source/multimaps-java.rst index 3ac0ad4b91..11febcb738 100644 --- a/documentation/sphinx/source/multimaps-java.rst +++ b/documentation/sphinx/source/multimaps-java.rst @@ -74,7 +74,7 @@ Here’s a simple implementation of multimaps with multisets as described: private static final int N = 100; static { - fdb = FDB.selectAPIVersion(610); + fdb = FDB.selectAPIVersion(620); db = fdb.open(); multi = new Subspace(Tuple.from("M")); } diff --git a/documentation/sphinx/source/priority-queues-java.rst b/documentation/sphinx/source/priority-queues-java.rst index 391c8192b5..50a2b1e037 100644 --- a/documentation/sphinx/source/priority-queues-java.rst +++ b/documentation/sphinx/source/priority-queues-java.rst @@ -74,7 +74,7 @@ Here's a basic implementation of the model: private static final Random randno; static{ - fdb = FDB.selectAPIVersion(610); + fdb = FDB.selectAPIVersion(620); db = fdb.open(); pq = new Subspace(Tuple.from("P")); diff --git a/documentation/sphinx/source/queues-java.rst b/documentation/sphinx/source/queues-java.rst index 70825aab63..7d39a27fce 100644 --- a/documentation/sphinx/source/queues-java.rst +++ b/documentation/sphinx/source/queues-java.rst @@ -73,7 +73,7 @@ The following is a simple implementation of the basic pattern: private static final Random randno; static{ - fdb = FDB.selectAPIVersion(610); + fdb = FDB.selectAPIVersion(620); db = fdb.open(); queue = new Subspace(Tuple.from("Q")); randno = new Random(); diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst index 9817574b41..600f2f4d3b 100644 --- a/documentation/sphinx/source/release-notes.rst +++ b/documentation/sphinx/source/release-notes.rst @@ -7,15 +7,26 @@ Release Notes Features -------- +* Improved team collection for data distribution that builds a balanced number of teams per server and gurantees that each server has at least one team. `(PR #1785) `_. + +* CMake is now our official build system. The Makefile based build system is deprecated. + +* Added local ratekeeper, to throttle reads at a per-storage-process level. `(PR #1447) `_. Performance ----------- +* Use CRC32 checksum for SQLite pages. `(PR #1582) `_. +* Added a 96-byte fast allocator, so storage queue nodes use less memory. `(PR #1336) `_. +* Handle large packets better. `(PR #1684) `_. + Fixes ----- +* Set the priority of redundant teams to remove as PRIORITY_TEAM_REDUNDANT, instead of PRIORITY_TEAM_UNHEALTHY. `(PR #1802) `_. * During an upgrade, the multi-version client now persists database default options and transaction options that aren't reset on retry (e.g. transaction timeout). In order for these options to function correctly during an upgrade, a 6.2 or later client should be used as the primary client. `(PR #1767) `_. * If a cluster is upgraded during an ``onError`` call, the cluster could return a ``cluster_version_changed`` error. `(PR #1734) `_. +* Do not set doBuildTeams in StorageServerTracker unless a storage server's interface changes, in order to avoid unnecessary work. `(PR #1779) `_. Status ------ @@ -24,16 +35,30 @@ Status * Added ``cluster.page_cache`` section to status. In this section, added two new statistics ``storage_hit_rate`` and ``log_hit_rate`` that indicate the fraction of recent page reads that were served by cache. `(PR #1823) `_. * Added transaction start counts by priority to ``cluster.workload.transactions``. The new counters are named ``started_immediate_priority``, ``started_default_priority``, and ``started_batch_priority``. `(PR #1836) `_. * Remove ``cluster.datacenter_version_difference`` and replace it with ``cluster.datacenter_lag`` that has subfields ``versions`` and ``seconds``. `(PR #1800) `_. +* Added ``local_rate`` to the ``roles`` section to record the throttling rate of the local ratekeeper `(PR #1712) `_. Bindings -------- +* Add a transaction size limit for both database option and transaction option. `(PR #1725) `_. +* Added a new API to get the approximated transaction size before commit, e.g., ``fdb_transaction_get_approximate_size`` in the C binding. `(PR #1756) `_. +* C: ``fdb_future_get_version`` has been renamed to ``fdb_future_get_int64``. `(PR #1756) `_. +* C: Applications linking to libfdb_c can now use ``pkg-config foundationdb-client`` or ``find_package(FoundationDB-Client ...)`` (for cmake) to get the proper flags for compiling and linking. `(PR #1636) `_. * Go: The Go bindings now require Go version 1.11 or later. * Go: Fix issue with finalizers running too early that could lead to undefined behavior. `(PR #1451) `_. +* Added transaction option to control the field length of keys and values in debug transaction logging in order to avoid truncation. `(PR #1844) `_. Other Changes ------------- +* Clients will throw ``transaction_too_old`` when attempting to read if ``setVersion`` was called with a version smaller than the smallest read version obtained from the cluster. This is a protection against reading from the wrong cluster in multi-cluster scenarios. `(PR #1413) `_. +* Trace files are now ordered lexicographically. This means that the filename format for trace files did change. `(PR #1828) `_. +* Improved ``TransactionMetrics`` log events by adding a random UID to distinguish multiple open connections, a flag to identify internal vs. client connections, and logging of rates and roughness in addition to total count for several metrics. `(PR #1808) `_. +* FoundationDB can now be built with clang and libc++ on Linux `(PR #1666) `_. +* Added experimental framework to run C and Java clients in simulator `(PR #1678) `_. +* Added new network option for client buggify which will randomly throw expected exceptions in the client. Intended for client testing `(PR #1417) `_. +* Added ``--cache_memory`` parameter for ``fdbserver`` processes to control the amount of memory dedicated to caching pages read from disk. `(PR #1889) `_. + Earlier release notes --------------------- * :doc:`6.1 (API Version 610) ` diff --git a/documentation/sphinx/source/simple-indexes-java.rst b/documentation/sphinx/source/simple-indexes-java.rst index 546b894be5..4bd1281221 100644 --- a/documentation/sphinx/source/simple-indexes-java.rst +++ b/documentation/sphinx/source/simple-indexes-java.rst @@ -87,7 +87,7 @@ In this example, we’re storing user data based on user ID but sometimes need t private static final Subspace index; static { - fdb = FDB.selectAPIVersion(610); + fdb = FDB.selectAPIVersion(620); db = fdb.open(); main = new Subspace(Tuple.from("user")); index = new Subspace(Tuple.from("zipcode_index")); diff --git a/documentation/sphinx/source/tables-java.rst b/documentation/sphinx/source/tables-java.rst index 4b09695ff5..0ca4add535 100644 --- a/documentation/sphinx/source/tables-java.rst +++ b/documentation/sphinx/source/tables-java.rst @@ -62,7 +62,7 @@ Here’s a simple implementation of the basic table pattern: private static final Subspace colIndex; static { - fdb = FDB.selectAPIVersion(610); + fdb = FDB.selectAPIVersion(620); db = fdb.open(); table = new Subspace(Tuple.from("T")); rowIndex = table.subspace(Tuple.from("R")); diff --git a/documentation/sphinx/source/vector-java.rst b/documentation/sphinx/source/vector-java.rst index 6a5f785b35..8b23b16d31 100644 --- a/documentation/sphinx/source/vector-java.rst +++ b/documentation/sphinx/source/vector-java.rst @@ -77,7 +77,7 @@ Here’s the basic pattern: private static final Subspace vector; static { - fdb = FDB.selectAPIVersion(610); + fdb = FDB.selectAPIVersion(620); db = fdb.open(); vector = new Subspace(Tuple.from("V")); } diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 5d62be9a8b..931f5e9472 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -117,37 +117,40 @@ enum { OPT_DEST_CLUSTER, OPT_CLEANUP, - OPT_TRACE_FORMAT + OPT_TRACE_FORMAT, + OPT_USE_OBJECT_SERIALIZER }; CSimpleOpt::SOption g_rgAgentOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_LOCALITY, "--locality_", SO_REQ_SEP }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_LOCALITY, "--locality_", SO_REQ_SEP }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupStartOptions[] = { @@ -173,6 +176,8 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_QUIET, "-q", SO_NONE }, { OPT_QUIET, "--quiet", SO_NONE }, @@ -240,6 +245,8 @@ CSimpleOpt::SOption g_rgBackupStatusOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_VERSION, "--version", SO_NONE }, { OPT_VERSION, "-v", SO_NONE }, @@ -270,6 +277,8 @@ CSimpleOpt::SOption g_rgBackupAbortOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_QUIET, "-q", SO_NONE }, { OPT_QUIET, "--quiet", SO_NONE }, @@ -301,6 +310,8 @@ CSimpleOpt::SOption g_rgBackupDiscontinueOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_QUIET, "-q", SO_NONE }, { OPT_QUIET, "--quiet", SO_NONE }, @@ -332,6 +343,8 @@ CSimpleOpt::SOption g_rgBackupWaitOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_QUIET, "-q", SO_NONE }, { OPT_QUIET, "--quiet", SO_NONE }, @@ -359,6 +372,8 @@ CSimpleOpt::SOption g_rgBackupPauseOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_QUIET, "-q", SO_NONE }, { OPT_QUIET, "--quiet", SO_NONE }, @@ -388,6 +403,8 @@ CSimpleOpt::SOption g_rgBackupExpireOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_QUIET, "-q", SO_NONE }, { OPT_QUIET, "--quiet", SO_NONE }, @@ -425,6 +442,8 @@ CSimpleOpt::SOption g_rgBackupDeleteOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_QUIET, "-q", SO_NONE }, { OPT_QUIET, "--quiet", SO_NONE }, @@ -456,6 +475,8 @@ CSimpleOpt::SOption g_rgBackupDescribeOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_QUIET, "-q", SO_NONE }, { OPT_QUIET, "--quiet", SO_NONE }, @@ -520,6 +541,8 @@ CSimpleOpt::SOption g_rgBackupListOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_QUIET, "-q", SO_NONE }, { OPT_QUIET, "--quiet", SO_NONE }, @@ -562,6 +585,8 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_QUIET, "-q", SO_NONE }, { OPT_QUIET, "--quiet", SO_NONE }, @@ -599,6 +624,8 @@ CSimpleOpt::SOption g_rgDBAgentOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_CRASHONERROR, "--crash", SO_NONE }, { OPT_LOCALITY, "--locality_", SO_REQ_SEP }, @@ -629,6 +656,8 @@ CSimpleOpt::SOption g_rgDBStartOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_QUIET, "-q", SO_NONE }, { OPT_QUIET, "--quiet", SO_NONE }, @@ -662,6 +691,8 @@ CSimpleOpt::SOption g_rgDBStatusOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_VERSION, "--version", SO_NONE }, { OPT_VERSION, "-v", SO_NONE }, @@ -693,6 +724,8 @@ CSimpleOpt::SOption g_rgDBSwitchOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_QUIET, "-q", SO_NONE }, { OPT_QUIET, "--quiet", SO_NONE }, @@ -726,6 +759,8 @@ CSimpleOpt::SOption g_rgDBAbortOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_QUIET, "-q", SO_NONE }, { OPT_QUIET, "--quiet", SO_NONE }, @@ -755,6 +790,8 @@ CSimpleOpt::SOption g_rgDBPauseOptions[] = { { OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP }, + { OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP }, { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, { OPT_QUIET, "-q", SO_NONE }, { OPT_QUIET, "--quiet", SO_NONE }, @@ -829,6 +866,10 @@ static void printAgentUsage(bool devhelp) { printf(" --trace_format FORMAT\n" " Select the format of the trace files. xml (the default) and json are supported.\n" " Has no effect unless --log is specified.\n"); + printf(" -S ON|OFF, --object-serializer ON|OFF\n" + " Use object serializer for sending messages. The object serializer\n" + " is currently a beta feature and it allows fdb processes to talk to\n" + " each other even if they don't have the same version\n"); printf(" -m SIZE, --memory SIZE\n" " Memory limit. The default value is 8GiB. When specified\n" " without a unit, MiB is assumed.\n"); @@ -904,6 +945,17 @@ static void printBackupUsage(bool devhelp) { printf(" -k KEYS List of key ranges to backup.\n" " If not specified, the entire database will be backed up.\n"); printf(" -n, --dryrun For start or restore operations, performs a trial run with no actual changes made.\n"); + printf(" --log Enables trace file logging for the CLI session.\n" + " --logdir PATH Specifes the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); + printf(" --trace_format FORMAT\n" + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); + printf(" -S ON|OFF, --object-serializer ON|OFF\n" + " Use object serializer for sending messages. The object serializer\n" + " is currently a beta feature and it allows fdb processes to talk to\n" + " each other even if they don't have the same version\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif @@ -951,6 +1003,17 @@ static void printRestoreUsage(bool devhelp ) { printf(" --add_prefix PREFIX\n"); printf(" Prefix to add to the restored keys\n"); printf(" -n, --dryrun Perform a trial run with no changes made.\n"); + printf(" --log Enables trace file logging for the CLI session.\n" + " --logdir PATH Specifes the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); + printf(" --trace_format FORMAT\n" + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); + printf(" -S ON|OFF, --object-serializer ON|OFF\n" + " Use object serializer for sending messages. The object serializer\n" + " is currently a beta feature and it allows fdb processes to talk to\n" + " each other even if they don't have the same version\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif @@ -992,6 +1055,10 @@ static void printDBAgentUsage(bool devhelp) { printf(" --trace_format FORMAT\n" " Select the format of the trace files. xml (the default) and json are supported.\n" " Has no effect unless --log is specified.\n"); + printf(" -S ON|OFF, --object-serializer ON|OFF\n" + " Use object serializer for sending messages. The object serializer\n" + " is currently a beta feature and it allows fdb processes to talk to\n" + " each other even if they don't have the same version\n"); printf(" -m SIZE, --memory SIZE\n" " Memory limit. The default value is 8GiB. When specified\n" " without a unit, MiB is assumed.\n"); @@ -1028,6 +1095,17 @@ static void printDBBackupUsage(bool devhelp) { #ifndef TLS_DISABLED printf(TLS_HELP); #endif + printf(" --log Enables trace file logging for the CLI session.\n" + " --logdir PATH Specifes the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); + printf(" --trace_format FORMAT\n" + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); + printf(" -S ON|OFF, --object-serializer ON|OFF\n" + " Use object serializer for sending messages. The object serializer\n" + " is currently a beta feature and it allows fdb processes to talk to\n" + " each other even if they don't have the same version\n"); printf(" -v, --version Print version information and exit.\n"); printf(" -h, --help Display this help and exit.\n"); printf("\n" @@ -2662,6 +2740,7 @@ int main(int argc, char* argv[]) { bool dryRun = false; std::string traceDir = ""; std::string traceFormat = ""; + bool useObjectSerializer = true; std::string traceLogGroup; uint64_t traceRollSize = TRACE_DEFAULT_ROLL_SIZE; uint64_t traceMaxLogsSize = TRACE_DEFAULT_MAX_LOGS_SIZE; @@ -2772,6 +2851,18 @@ int main(int argc, char* argv[]) { } traceFormat = args->OptionArg(); break; + case OPT_USE_OBJECT_SERIALIZER: { + std::string s = args->OptionArg(); + std::transform(s.begin(), s.end(), s.begin(), ::tolower); + if (s == "on" || s == "true" || s == "1") { + useObjectSerializer = true; + } else if (s == "off" || s == "false" || s == "0") { + useObjectSerializer = false; + } else { + fprintf(stderr, "ERROR: Could not parse object serializer option: `%s'\n", s.c_str()); + } + break; + } case OPT_TRACE_LOG_GROUP: traceLogGroup = args->OptionArg(); break; @@ -3118,6 +3209,11 @@ int main(int argc, char* argv[]) { setNetworkOption(FDBNetworkOptions::ENABLE_SLOW_TASK_PROFILING); } setNetworkOption(FDBNetworkOptions::DISABLE_CLIENT_STATISTICS_LOGGING); + // The USE_OBJECT_SERIALIZER network option expects an 8 byte little endian integer which is interpreted as + // zero = false, non-zero = true. + setNetworkOption(FDBNetworkOptions::USE_OBJECT_SERIALIZER, + useObjectSerializer ? LiteralStringRef("\x01\x00\x00\x00\x00\x00\x00\x00") + : LiteralStringRef("\x00\x00\x00\x00\x00\x00\x00\x00")); // deferred TLS options if (tlsCertPath.size()) { @@ -3194,11 +3290,14 @@ int main(int argc, char* argv[]) { } TraceEvent("ProgramStart") + .setMaxEventLength(12000) .detail("SourceVersion", getHGVersion()) .detail("Version", FDB_VT_VERSION ) .detail("PackageName", FDB_VT_PACKAGE_NAME) .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL)) + .setMaxFieldLength(10000) .detail("CommandLine", commandLine) + .setMaxFieldLength(0) .detail("MemoryLimit", memLimit) .trackLatest("ProgramStart"); diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 67611efab1..c20be8a48f 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -479,10 +479,12 @@ void initHelp() { "coordinators auto|
+ [description=new_cluster_description]", "change cluster coordinators or description", "If 'auto' is specified, coordinator addresses will be choosen automatically to support the configured redundancy level. (If the current set of coordinators are healthy and already support the redundancy level, nothing will be changed.)\n\nOtherwise, sets the coordinators to the list of IP:port pairs specified by
+. An fdbserver process must be running on each of the specified addresses.\n\ne.g. coordinators 10.0.0.1:4000 10.0.0.2:4000 10.0.0.3:4000\n\nIf 'description=desc' is specified then the description field in the cluster\nfile is changed to desc, which must match [A-Za-z0-9_]+."); - helpMap["exclude"] = CommandHelp( - "exclude
*", - "exclude servers from the database", - "If no addresses are specified, lists the set of excluded servers.\n\nFor each IP address or IP:port pair in
*, adds the address to the set of excluded servers then waits until all database state has been safely moved away from the specified servers."); + helpMap["exclude"] = + CommandHelp("exclude [no_wait]
*", "exclude servers from the database", + "If no addresses are specified, lists the set of excluded servers.\n\nFor each IP address or " + "IP:port pair in
*, adds the address to the set of excluded servers then waits until all " + "database state has been safely moved away from the specified servers. If 'no_wait' is set, the " + "command returns \nimmediately without checking if the exclusions have completed successfully."); helpMap["include"] = CommandHelp( "include all|
*", "permit previously-excluded servers to rejoin the database", @@ -554,6 +556,10 @@ void initHelp() { "maintenance [on|off] [ZONEID] [SECONDS]", "mark a zone for maintenance", "Calling this command with `on' prevents data distribution from moving data away from the processes with the specified ZONEID. Data distribution will automatically be turned back on for ZONEID after the specified SECONDS have elapsed, or after a storage server with a different ZONEID fails. Only one ZONEID can be marked for maintenance. Calling this command with no arguments will display any ongoing maintenance. Calling this command with `off' will disable maintenance.\n"); + helpMap["consistencycheck"] = CommandHelp( + "consistencycheck [on|off]", + "enables or disables consistencycheck", + "Calling this command with `on' enables consistency check to run and `off' will disable the same. Calling this command with no arguments will display setting for consistency check.\n"); hiddenCommands.insert("expensive_data_check"); hiddenCommands.insert("datadistribution"); @@ -626,6 +632,9 @@ std::string getDateInfoString(StatusObjectReader statusObj, std::string key) { } std::string getProcessAddressByServerID(StatusObjectReader processesMap, std::string serverID) { + if(serverID == "") + return "unknown"; + for (auto proc : processesMap.obj()){ try { StatusArray rolesArray = proc.second.get_obj()["roles"].get_array(); @@ -1981,13 +1990,18 @@ ACTOR Future exclude( Database db, std::vector tokens, Referenc printf("To find out whether it is safe to remove one or more of these\n" "servers from the cluster, type `exclude '.\n" "To return one of these servers to the cluster, type `include '.\n"); + + return false; } else { state std::vector addresses; state std::set exclusions; bool force = false; + state bool waitForAllExcluded = true; for(auto t = tokens.begin()+1; t != tokens.end(); ++t) { if(*t == LiteralStringRef("FORCE")) { force = true; + } else if (*t == LiteralStringRef("no_wait")) { + waitForAllExcluded = false; } else { auto a = AddressExclusion::parse( *t ); if (!a.isValid()) { @@ -2089,13 +2103,16 @@ ACTOR Future exclude( Database db, std::vector tokens, Referenc wait( makeInterruptable(excludeServers(db,addresses)) ); - printf("Waiting for state to be removed from all excluded servers. This may take a while.\n"); - printf("(Interrupting this wait with CTRL+C will not cancel the data movement.)\n"); + if (waitForAllExcluded) { + printf("Waiting for state to be removed from all excluded servers. This may take a while.\n"); + printf("(Interrupting this wait with CTRL+C will not cancel the data movement.)\n"); + } if(warn.isValid()) warn.cancel(); - wait( makeInterruptable(waitForExcludedServers(db,addresses)) ); + state std::set notExcludedServers = + wait(makeInterruptable(checkForExcludingServers(db, addresses, waitForAllExcluded))); std::vector workers = wait( makeInterruptable(getWorkers(db)) ); std::map> workerPorts; for(auto addr : workers) @@ -2122,9 +2139,18 @@ ACTOR Future exclude( Database db, std::vector tokens, Referenc } printf("\n"); - } - else + } else if (notExcludedServers.empty()) { printf("\nIt is now safe to remove these machines or processes from the cluster.\n"); + } else { + printf("\nWARNING: Exclusion in progress. It is not safe to remove the following machines\n" + "or processes from the cluster:\n"); + for (auto addr : notExcludedServers) { + if (addr.port == 0) + printf(" %s\n", addr.ip.toString().c_str()); + else + printf(" %s\n", addr.toString().c_str()); + } + } bool foundCoordinator = false; auto ccs = ClusterConnectionFile( ccf->getFilename() ).getConnectionString(); @@ -2138,12 +2164,20 @@ ACTOR Future exclude( Database db, std::vector tokens, Referenc if (foundCoordinator) printf("Type `help coordinators' for information on how to change the\n" "cluster's coordination servers before removing them.\n"); + + return false; } - return false; } ACTOR Future createSnapshot(Database db, StringRef snapCmd) { - wait(makeInterruptable(mgmtSnapCreate(db, snapCmd))); + try { + UID snapUID = wait(makeInterruptable(mgmtSnapCreate(db, snapCmd))); + printf("Snapshots create succeeded with UID: %s\n", snapUID.toString().c_str()); + } catch (Error& e) { + fprintf(stderr, "Snapshot create failed, %d (%s)." + " Please cleanup any instance level snapshots created.\n", e.code(), e.what()); + return true; + } return false; } @@ -2349,7 +2383,7 @@ struct CLIOptions { bool trace; std::string traceDir; std::string traceFormat; - bool useObjectSerializer = false; + bool useObjectSerializer = true; int exit_timeout; Optional exec; bool initialStatusCheck; @@ -2529,19 +2563,21 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { if (opt.trace) { TraceEvent("CLIProgramStart") + .setMaxEventLength(12000) .detail("SourceVersion", getHGVersion()) .detail("Version", FDB_VT_VERSION) .detail("PackageName", FDB_VT_PACKAGE_NAME) .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL)) .detail("ClusterFile", ccf->getFilename().c_str()) .detail("ConnectionString", ccf->getConnectionString().toString()) + .setMaxFieldLength(10000) .detail("CommandLine", opt.commandLine) .trackLatest("ProgramStart"); } if (!opt.exec.present()) { if(opt.initialStatusCheck) { - wait( makeInterruptable( checkStatus( Void(), ccf )) ); + wait(makeInterruptable(checkStatus(Void(), db->getConnectionFile()))); } else { printf("\n"); @@ -2578,7 +2614,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { linenoise.historyAdd(line); } - warn = checkStatus( timeWarning(5.0, "\nWARNING: Long delay (Ctrl-C to interrupt)\n"), ccf ); + warn = checkStatus(timeWarning(5.0, "\nWARNING: Long delay (Ctrl-C to interrupt)\n"), db->getConnectionFile()); try { state UID randomID = deterministicRandom()->randomUniqueID(); @@ -2706,7 +2742,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { continue; } - StatusObject s = wait( makeInterruptable( StatusClient::statusFetcher( ccf ) ) ); + StatusObject s = wait(makeInterruptable(StatusClient::statusFetcher(db->getConnectionFile()))); if (!opt.exec.present()) printf("\n"); printStatus(s, level); @@ -2715,7 +2751,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { } if (tokencmp(tokens[0], "configure")) { - bool err = wait( configure( db, tokens, ccf, &linenoise, warn ) ); + bool err = wait(configure(db, tokens, db->getConnectionFile(), &linenoise, warn)); if (err) is_error = true; continue; } @@ -2732,7 +2768,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { } if (tokencmp(tokens[0], "coordinators")) { - auto cs = ClusterConnectionFile( ccf->getFilename() ).getConnectionString(); + auto cs = ClusterConnectionFile(db->getConnectionFile()->getFilename()).getConnectionString(); if (tokens.size() < 2) { printf("Cluster description: %s\n", cs.clusterKeyName().toString().c_str()); printf("Cluster coordinators (%zu): %s\n", cs.coordinators().size(), describe(cs.coordinators()).c_str()); @@ -2745,7 +2781,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { } if (tokencmp(tokens[0], "exclude")) { - bool err = wait( exclude(db, tokens, ccf, warn ) ); + bool err = wait(exclude(db, tokens, db->getConnectionFile(), warn)); if (err) is_error = true; continue; } @@ -2919,7 +2955,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { is_error = true; continue; } - wait( makeInterruptable( forceRecovery( ccf, tokens[1] ) ) ); + wait(makeInterruptable(forceRecovery(db->getConnectionFile(), tokens[1]))); continue; } @@ -2947,6 +2983,31 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { continue; } + if (tokencmp(tokens[0], "consistencycheck")) { + getTransaction(db, tr, options, intrans); + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + if (tokens.size() == 1) { + state Future>> ccSuspendSettingFuture = tr->get(fdbShouldConsistencyCheckBeSuspended); + wait( makeInterruptable( success(ccSuspendSettingFuture) ) ); + bool ccSuspendSetting = ccSuspendSettingFuture.get().present() ? BinaryReader::fromStringRef(ccSuspendSettingFuture.get().get(), Unversioned()) : false; + printf("ConsistencyCheck is %s\n", ccSuspendSetting ? "off" : "on"); + } + else if (tokens.size() == 2 && tokencmp(tokens[1], "off")) { + tr->set(fdbShouldConsistencyCheckBeSuspended, BinaryWriter::toValue(true, Unversioned())); + wait( commitTransaction(tr) ); + } + else if (tokens.size() == 2 && tokencmp(tokens[1], "on")) { + tr->set(fdbShouldConsistencyCheckBeSuspended, BinaryWriter::toValue(false, Unversioned())); + wait( commitTransaction(tr) ); + } else { + printUsage(tokens[0]); + is_error = true; + } + continue; + } + if (tokencmp(tokens[0], "profile")) { if (tokens.size() == 1) { printf("ERROR: Usage: profile \n"); diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index a02166262e..00e1241b70 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -843,4 +843,6 @@ public: return updateErrorInfo(cx, e, details); } }; + +#include "flow/unactorcompiler.h" #endif diff --git a/fdbclient/ClientLogEvents.h b/fdbclient/ClientLogEvents.h index 14191bf209..f99f722e84 100644 --- a/fdbclient/ClientLogEvents.h +++ b/fdbclient/ClientLogEvents.h @@ -35,6 +35,14 @@ namespace FdbClientLogEvents { EVENTTYPEEND // End of EventType }; + typedef int TrasactionPriorityType; + enum { + PRIORITY_DEFAULT = 0, + PRIORITY_BATCH = 1, + PRIORITY_IMMEDIATE = 2, + PRIORITY_END + }; + struct Event { Event(EventType t, double ts) : type(t), startTs(ts) { } Event() { } @@ -44,7 +52,7 @@ namespace FdbClientLogEvents { EventType type{ EVENTTYPEEND }; double startTs{ 0 }; - void logEvent(std::string id) const {} + void logEvent(std::string id, int maxFieldLength) const {} }; struct EventGetVersion : public Event { @@ -60,8 +68,43 @@ namespace FdbClientLogEvents { double latency; - void logEvent(std::string id) const { - TraceEvent("TransactionTrace_GetVersion").detail("TransactionID", id).detail("Latency", latency); + void logEvent(std::string id, int maxFieldLength) const { + TraceEvent("TransactionTrace_GetVersion") + .detail("TransactionID", id) + .detail("Latency", latency); + } + }; + + // Version V2 of EventGetVersion starting at 6.2 + struct EventGetVersion_V2 : public Event { + EventGetVersion_V2(double ts, double lat, uint32_t type) : Event(GET_VERSION_LATENCY, ts), latency(lat) { + if(type == GetReadVersionRequest::PRIORITY_DEFAULT) { + priorityType = PRIORITY_DEFAULT; + } else if (type == GetReadVersionRequest::PRIORITY_BATCH) { + priorityType = PRIORITY_BATCH; + } else if (type == GetReadVersionRequest::PRIORITY_SYSTEM_IMMEDIATE){ + priorityType = PRIORITY_IMMEDIATE; + } else { + ASSERT(0); + } + } + EventGetVersion_V2() { } + + template Ar& serialize(Ar &ar) { + if (!ar.isDeserializing) + return serializer(Event::serialize(ar), latency, priorityType); + else + return serializer(ar, latency, priorityType); + } + + double latency; + TrasactionPriorityType priorityType {PRIORITY_END}; + + void logEvent(std::string id, int maxFieldLength) const { + TraceEvent("TransactionTrace_GetVersion") + .detail("TransactionID", id) + .detail("Latency", latency) + .detail("PriorityType", priorityType); } }; @@ -80,8 +123,14 @@ namespace FdbClientLogEvents { int valueSize; Key key; - void logEvent(std::string id) const { - TraceEvent("TransactionTrace_Get").detail("TransactionID", id).detail("Latency", latency).detail("ValueSizeBytes", valueSize).detail("Key", key); + void logEvent(std::string id, int maxFieldLength) const { + TraceEvent("TransactionTrace_Get") + .setMaxEventLength(-1) + .detail("TransactionID", id) + .detail("Latency", latency) + .detail("ValueSizeBytes", valueSize) + .setMaxFieldLength(maxFieldLength) + .detail("Key", key); } }; @@ -101,8 +150,15 @@ namespace FdbClientLogEvents { Key startKey; Key endKey; - void logEvent(std::string id) const { - TraceEvent("TransactionTrace_GetRange").detail("TransactionID", id).detail("Latency", latency).detail("RangeSizeBytes", rangeSize).detail("StartKey", startKey).detail("EndKey", endKey); + void logEvent(std::string id, int maxFieldLength) const { + TraceEvent("TransactionTrace_GetRange") + .setMaxEventLength(-1) + .detail("TransactionID", id) + .detail("Latency", latency) + .detail("RangeSizeBytes", rangeSize) + .setMaxFieldLength(maxFieldLength) + .detail("StartKey", startKey) + .detail("EndKey", endKey); } }; @@ -122,20 +178,38 @@ namespace FdbClientLogEvents { int commitBytes; CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized - void logEvent(std::string id) const { + void logEvent(std::string id, int maxFieldLength) const { for (auto &read_range : req.transaction.read_conflict_ranges) { - TraceEvent("TransactionTrace_Commit_ReadConflictRange").detail("TransactionID", id).detail("Begin", read_range.begin).detail("End", read_range.end); + TraceEvent("TransactionTrace_Commit_ReadConflictRange") + .setMaxEventLength(-1) + .detail("TransactionID", id) + .setMaxFieldLength(maxFieldLength) + .detail("Begin", read_range.begin) + .detail("End", read_range.end); } for (auto &write_range : req.transaction.write_conflict_ranges) { - TraceEvent("TransactionTrace_Commit_WriteConflictRange").detail("TransactionID", id).detail("Begin", write_range.begin).detail("End", write_range.end); + TraceEvent("TransactionTrace_Commit_WriteConflictRange") + .setMaxEventLength(-1) + .detail("TransactionID", id) + .setMaxFieldLength(maxFieldLength) + .detail("Begin", write_range.begin) + .detail("End", write_range.end); } for (auto &mutation : req.transaction.mutations) { - TraceEvent("TransactionTrace_Commit_Mutation").detail("TransactionID", id).detail("Mutation", mutation.toString()); + TraceEvent("TransactionTrace_Commit_Mutation") + .setMaxEventLength(-1) + .detail("TransactionID", id) + .setMaxFieldLength(maxFieldLength) + .detail("Mutation", mutation.toString()); } - TraceEvent("TransactionTrace_Commit").detail("TransactionID", id).detail("Latency", latency).detail("NumMutations", numMutations).detail("CommitSizeBytes", commitBytes); + TraceEvent("TransactionTrace_Commit") + .detail("TransactionID", id) + .detail("Latency", latency) + .detail("NumMutations", numMutations) + .detail("CommitSizeBytes", commitBytes); } }; @@ -153,8 +227,13 @@ namespace FdbClientLogEvents { int errCode; Key key; - void logEvent(std::string id) const { - TraceEvent("TransactionTrace_GetError").detail("TransactionID", id).detail("ErrCode", errCode).detail("Key", key); + void logEvent(std::string id, int maxFieldLength) const { + TraceEvent("TransactionTrace_GetError") + .setMaxEventLength(-1) + .detail("TransactionID", id) + .detail("ErrCode", errCode) + .setMaxFieldLength(maxFieldLength) + .detail("Key", key); } }; @@ -173,8 +252,14 @@ namespace FdbClientLogEvents { Key startKey; Key endKey; - void logEvent(std::string id) const { - TraceEvent("TransactionTrace_GetRangeError").detail("TransactionID", id).detail("ErrCode", errCode).detail("StartKey", startKey).detail("EndKey", endKey); + void logEvent(std::string id, int maxFieldLength) const { + TraceEvent("TransactionTrace_GetRangeError") + .setMaxEventLength(-1) + .detail("TransactionID", id) + .detail("ErrCode", errCode) + .setMaxFieldLength(maxFieldLength) + .detail("StartKey", startKey) + .detail("EndKey", endKey); } }; @@ -192,20 +277,36 @@ namespace FdbClientLogEvents { int errCode; CommitTransactionRequest req; // Only CommitTransactionRef and Arena object within CommitTransactionRequest is serialized - void logEvent(std::string id) const { + void logEvent(std::string id, int maxFieldLength) const { for (auto &read_range : req.transaction.read_conflict_ranges) { - TraceEvent("TransactionTrace_CommitError_ReadConflictRange").detail("TransactionID", id).detail("Begin", read_range.begin).detail("End", read_range.end); + TraceEvent("TransactionTrace_CommitError_ReadConflictRange") + .setMaxEventLength(-1) + .detail("TransactionID", id) + .setMaxFieldLength(maxFieldLength) + .detail("Begin", read_range.begin) + .detail("End", read_range.end); } for (auto &write_range : req.transaction.write_conflict_ranges) { - TraceEvent("TransactionTrace_CommitError_WriteConflictRange").detail("TransactionID", id).detail("Begin", write_range.begin).detail("End", write_range.end); + TraceEvent("TransactionTrace_CommitError_WriteConflictRange") + .setMaxEventLength(-1) + .detail("TransactionID", id) + .setMaxFieldLength(maxFieldLength) + .detail("Begin", write_range.begin) + .detail("End", write_range.end); } for (auto &mutation : req.transaction.mutations) { - TraceEvent("TransactionTrace_CommitError_Mutation").detail("TransactionID", id).detail("Mutation", mutation.toString()); + TraceEvent("TransactionTrace_CommitError_Mutation") + .setMaxEventLength(-1) + .detail("TransactionID", id) + .setMaxFieldLength(maxFieldLength) + .detail("Mutation", mutation.toString()); } - TraceEvent("TransactionTrace_CommitError").detail("TransactionID", id).detail("ErrCode", errCode); + TraceEvent("TransactionTrace_CommitError") + .detail("TransactionID", id) + .detail("ErrCode", errCode); } }; } diff --git a/fdbclient/CommitTransaction.h b/fdbclient/CommitTransaction.h index ec6c4a3723..d33d5e24dd 100644 --- a/fdbclient/CommitTransaction.h +++ b/fdbclient/CommitTransaction.h @@ -44,8 +44,7 @@ static const char* typeString[] = { "SetValue", "ByteMax", "MinV2", "AndV2", - "CompareAndClear", - "Exec" }; + "CompareAndClear"}; struct MutationRef { static const int OVERHEAD_BYTES = 12; //12 is the size of Header in MutationList entries @@ -71,9 +70,6 @@ struct MutationRef { MinV2, AndV2, CompareAndClear, - // ExecOp is always set with FIRST_IN_BATCH option to quickly identify - // the op in a transaction batch while parsing it in TLog - Exec, MAX_ATOMIC_OP }; // This is stored this way for serialization purposes. diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h index 4b752590b3..2fc18d3200 100644 --- a/fdbclient/DatabaseContext.h +++ b/fdbclient/DatabaseContext.h @@ -52,11 +52,14 @@ public: } // For internal (fdbserver) use only - static Database create( Reference> clientInfo, Future clientInfoMonitor, LocalityData clientLocality, bool enableLocalityLoadBalance, TaskPriority taskID=TaskPriority::DefaultEndpoint, bool lockAware=false, int apiVersion=Database::API_VERSION_LATEST ); + static Database create(Reference> clientInfo, Future clientInfoMonitor, + LocalityData clientLocality, bool enableLocalityLoadBalance, + TaskPriority taskID = TaskPriority::DefaultEndpoint, bool lockAware = false, + int apiVersion = Database::API_VERSION_LATEST, bool switchable = false); ~DatabaseContext(); - Database clone() const { return Database(new DatabaseContext( connFile, clientInfo, clientInfoMonitor, taskID, clientLocality, enableLocalityLoadBalance, lockAware, internal, apiVersion )); } + Database clone() const { return Database(new DatabaseContext( connectionFile, clientInfo, clientInfoMonitor, taskID, clientLocality, enableLocalityLoadBalance, lockAware, internal, apiVersion, switchable )); } std::pair> getCachedLocation( const KeyRef&, bool isBackward = false ); bool getCachedLocations( const KeyRangeRef&, vector>>&, int limit, bool reverse ); @@ -93,15 +96,26 @@ public: Future onConnected(); // Returns after a majority of coordination servers are available and have reported a leader. The cluster file therefore is valid, but the database might be unavailable. Reference getConnectionFile(); + // Switch the database to use the new connection file, and recreate all pending watches for committed transactions. + // + // Meant to be used as part of a 'hot standby' solution to switch to the standby. A correct switch will involve + // advancing the version on the new cluster sufficiently far that any transaction begun with a read version from the + // old cluster will fail to commit. Assuming the above version-advancing is done properly, a call to + // switchConnectionFile guarantees that any read with a version from the old cluster will not be attempted on the + // new cluster. + Future switchConnectionFile(Reference standby); + Future connectionFileChanged(); + bool switchable = false; + //private: - explicit DatabaseContext( Reference connFile, Reference> clientDBInfo, + explicit DatabaseContext( Reference>> connectionFile, Reference> clientDBInfo, Future clientInfoMonitor, TaskPriority taskID, LocalityData const& clientLocality, - bool enableLocalityLoadBalance, bool lockAware, bool internal = true, int apiVersion = Database::API_VERSION_LATEST ); + bool enableLocalityLoadBalance, bool lockAware, bool internal = true, int apiVersion = Database::API_VERSION_LATEST, bool switchable = false ); explicit DatabaseContext( const Error &err ); // Key DB-specific information - Reference connFile; + Reference>> connectionFile; AsyncTrigger masterProxiesChangeTrigger; Future monitorMasterProxiesInfoChange; Reference masterProxies; @@ -118,6 +132,14 @@ public: }; std::map versionBatcher; + AsyncTrigger connectionFileChangedTrigger; + + // Disallow any reads at a read version lower than minAcceptableReadVersion. This way the client does not have to + // trust that the read version (possibly set manually by the application) is actually from the correct cluster. + // Updated everytime we get a GRV response + Version minAcceptableReadVersion = std::numeric_limits::max(); + void validateVersion(Version); + // Client status updater struct ClientStatusUpdater { std::vector< std::pair > inStatusQ; diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index e78a5f1813..06b574ec89 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -43,6 +43,7 @@ enum { tagLocalityUpgraded = -4, tagLocalitySatellite = -5, tagLocalityLogRouterMapped = -6, + tagLocalityTxs = -7, tagLocalityInvalid = -99 }; //The TLog and LogRouter require these number to be as compact as possible @@ -81,8 +82,8 @@ struct struct_like_traits : std::true_type { using Member = Tag; using types = pack; - template - static const index_t& get(const Member& m) { + template + static const index_t& get(const Member& m, Context&) { if constexpr (i == 0) { return m.id; } else { @@ -91,8 +92,8 @@ struct struct_like_traits : std::true_type { } } - template - static const void assign(Member& m, const Type& t) { + template + static const void assign(Member& m, const Type& t, Context&) { if constexpr (i == 0) { m.id = t; } else { @@ -321,6 +322,43 @@ struct KeyValueRef { }; }; +template<> +struct string_serialized_traits : std::true_type { + int32_t getSize(const KeyValueRef& item) const { + return 2*sizeof(uint32_t) + item.key.size() + item.value.size(); + } + + uint32_t save(uint8_t* out, const KeyValueRef& item) const { + auto begin = out; + uint32_t sz = item.key.size(); + memcpy(out, &sz, sizeof(sz)); + out += sizeof(sz); + memcpy(out, item.key.begin(), sz); + out += sz; + sz = item.value.size(); + memcpy(out, &sz, sizeof(sz)); + out += sizeof(sz); + memcpy(out, item.value.begin(), sz); + out += sz; + return out - begin; + } + + template + uint32_t load(const uint8_t* data, KeyValueRef& t, Context& context) { + auto begin = data; + uint32_t sz; + memcpy(&sz, data, sizeof(sz)); + data += sizeof(sz); + t.key = StringRef(context.tryReadZeroCopy(data, sz), sz); + data += sz; + memcpy(&sz, data, sizeof(sz)); + data += sizeof(sz); + t.value = StringRef(context.tryReadZeroCopy(data, sz), sz); + data += sz; + return data - begin; + } +}; + template<> struct Traceable : std::true_type { static std::string toString(const KeyValueRef& value) { diff --git a/fdbclient/HTTP.actor.cpp b/fdbclient/HTTP.actor.cpp index 5893588406..a4299176a6 100644 --- a/fdbclient/HTTP.actor.cpp +++ b/fdbclient/HTTP.actor.cpp @@ -334,7 +334,7 @@ namespace HTTP { } // Write headers to a packet buffer chain - PacketBuffer *pFirst = new PacketBuffer(); + PacketBuffer* pFirst = PacketBuffer::create(); PacketBuffer *pLast = writeRequestHeader(verb, resource, headers, pFirst); // Prepend headers to content packer buffer chain pContent->prependWriteBuffer(pFirst, pLast); diff --git a/fdbclient/IClientApi.h b/fdbclient/IClientApi.h index e1d04e5a9f..b3e6217054 100644 --- a/fdbclient/IClientApi.h +++ b/fdbclient/IClientApi.h @@ -61,6 +61,7 @@ public: virtual ThreadFuture commit() = 0; virtual Version getCommittedVersion() = 0; + virtual ThreadFuture getApproximateSize() = 0; virtual void setOption(FDBTransactionOptions::Option option, Optional value=Optional()) = 0; diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index afc64d62c2..49467b6ba6 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -1403,11 +1403,12 @@ ACTOR Future setDDMode( Database cx, int mode ) { rd >> oldMode; } } - if (!mode) { - BinaryWriter wrMyOwner(Unversioned()); - wrMyOwner << dataDistributionModeLock; - tr.set( moveKeysLockOwnerKey, wrMyOwner.toValue() ); - } + BinaryWriter wrMyOwner(Unversioned()); + wrMyOwner << dataDistributionModeLock; + tr.set( moveKeysLockOwnerKey, wrMyOwner.toValue() ); + BinaryWriter wrLastWrite(Unversioned()); + wrLastWrite << deterministicRandom()->randomUniqueID(); + tr.set( moveKeysLockWriteKey, wrLastWrite.toValue() ); tr.set( dataDistributionModeKey, wr.toValue() ); @@ -1420,13 +1421,16 @@ ACTOR Future setDDMode( Database cx, int mode ) { } } -ACTOR Future waitForExcludedServers( Database cx, vector excl ) { +ACTOR Future> checkForExcludingServers(Database cx, vector excl, + bool waitForAllExcluded) { state std::set exclusions( excl.begin(), excl.end() ); + state std::set inProgressExclusion; - if (!excl.size()) return Void(); + if (!excl.size()) return inProgressExclusion; loop { state Transaction tr(cx); + try { tr.setOption( FDBTransactionOptions::READ_SYSTEM_KEYS ); tr.setOption( FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE ); // necessary? @@ -1439,11 +1443,12 @@ ACTOR Future waitForExcludedServers( Database cx, vector ASSERT( !serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY ); state bool ok = true; + inProgressExclusion.clear(); for(auto& s : serverList) { auto addr = decodeServerListValue( s.value ).address(); if ( addressExcluded(exclusions, addr) ) { ok = false; - break; + inProgressExclusion.insert(addr); } } @@ -1454,47 +1459,46 @@ ACTOR Future waitForExcludedServers( Database cx, vector for( auto const& log : logs.first ) { if (log.second == NetworkAddress() || addressExcluded(exclusions, log.second)) { ok = false; - break; + inProgressExclusion.insert(log.second); } } for( auto const& log : logs.second ) { if (log.second == NetworkAddress() || addressExcluded(exclusions, log.second)) { ok = false; - break; + inProgressExclusion.insert(log.second); } } } - if (ok) return Void(); + if (ok) return inProgressExclusion; + if (!waitForAllExcluded) break; wait( delayJittered( 1.0 ) ); // SOMEDAY: watches! } catch (Error& e) { wait( tr.onError(e) ); } } + + return inProgressExclusion; } -ACTOR Future mgmtSnapCreate(Database cx, StringRef snapCmd) { +ACTOR Future mgmtSnapCreate(Database cx, StringRef snapCmd) { state int retryCount = 0; loop { state UID snapUID = deterministicRandom()->randomUniqueID(); try { wait(snapCreate(cx, snapCmd, snapUID)); - printf("Snapshots tagged with UID: %s, check logs for status\n", snapUID.toString().c_str()); TraceEvent("SnapCreateSucceeded").detail("snapUID", snapUID); - break; + return snapUID; } catch (Error& e) { ++retryCount; TraceEvent(retryCount > 3 ? SevWarn : SevInfo, "SnapCreateFailed").error(e); if (retryCount > 3) { - fprintf(stderr, "Snapshot create failed, %d (%s)." - " Please cleanup any instance level snapshots created.\n", e.code(), e.what()); throw; } } } - return Void(); } ACTOR Future waitForFullReplication( Database cx ) { diff --git a/fdbclient/ManagementAPI.actor.h b/fdbclient/ManagementAPI.actor.h index 2b1db48e3a..91640fb4f9 100644 --- a/fdbclient/ManagementAPI.actor.h +++ b/fdbclient/ManagementAPI.actor.h @@ -154,9 +154,11 @@ ACTOR Future setClass( Database cx, AddressExclusion server, ProcessClas // Get the current list of excluded servers ACTOR Future> getExcludedServers( Database cx ); -// Wait for the given, previously excluded servers to be evacuated (no longer used for state). Once this returns it is safe to shut down all such -// machines without impacting fault tolerance, until and unless any of them are explicitly included with includeServers() -ACTOR Future waitForExcludedServers( Database cx, vector servers ); +// Check for the given, previously excluded servers to be evacuated (no longer used for state). If waitForExclusion is +// true, this actor returns once it is safe to shut down all such machines without impacting fault tolerance, until and +// unless any of them are explicitly included with includeServers() +ACTOR Future> checkForExcludingServers(Database cx, vector servers, + bool waitForAllExcluded); // Gets a list of all workers in the cluster (excluding testers) ACTOR Future> getWorkers( Database cx ); @@ -193,7 +195,7 @@ bool schemaMatch( json_spirit::mValue const& schema, json_spirit::mValue const& // execute payload in 'snapCmd' on all the coordinators, TLogs and // storage nodes -ACTOR Future mgmtSnapCreate(Database cx, StringRef snapCmd); +ACTOR Future mgmtSnapCreate(Database cx, StringRef snapCmd); #include "flow/unactorcompiler.h" #endif diff --git a/fdbclient/MasterProxyInterface.h b/fdbclient/MasterProxyInterface.h index 165ded3bde..1c090c4afb 100644 --- a/fdbclient/MasterProxyInterface.h +++ b/fdbclient/MasterProxyInterface.h @@ -49,9 +49,8 @@ struct MasterProxyInterface { RequestStream< struct GetRawCommittedVersionRequest > getRawCommittedVersion; RequestStream< struct TxnStateRequest > txnState; - RequestStream execReq; - RequestStream< struct GetHealthMetricsRequest > getHealthMetrics; + RequestStream< struct ProxySnapRequest > proxySnapReq; UID id() const { return commit.getEndpoint().token; } std::string toString() const { return id().shortString(); } @@ -63,7 +62,7 @@ struct MasterProxyInterface { void serialize(Archive& ar) { serializer(ar, locality, provisional, commit, getConsistentReadVersion, getKeyServersLocations, waitFailure, getStorageServerRejoinInfo, getRawCommittedVersion, - txnState, getHealthMetrics, execReq); + txnState, getHealthMetrics, proxySnapReq); } void initEndpoints() { @@ -166,6 +165,8 @@ struct GetReadVersionReply : public ProxyForwardReply { bool locked; Optional metadataVersion; + GetReadVersionReply() : version(invalidVersion), locked(false) {} + template void serialize(Ar& ar) { serializer(ar, *(ProxyForwardReply*)this, version, locked, metadataVersion); @@ -332,20 +333,21 @@ struct GetHealthMetricsRequest } }; -struct ExecRequest +struct ProxySnapRequest { - constexpr static FileIdentifier file_identifier = 22403900; + constexpr static FileIdentifier file_identifier = 22204900; Arena arena; - StringRef execPayload; + StringRef snapPayload; + UID snapUID; ReplyPromise reply; Optional debugID; - explicit ExecRequest(Optional const& debugID = Optional()) : debugID(debugID) {} - explicit ExecRequest(StringRef exec, Optional debugID = Optional()) : execPayload(exec), debugID(debugID) {} + explicit ProxySnapRequest(Optional const& debugID = Optional()) : debugID(debugID) {} + explicit ProxySnapRequest(StringRef snap, UID snapUID, Optional debugID = Optional()) : snapPayload(snap), snapUID(snapUID), debugID(debugID) {} template void serialize(Ar& ar) { - serializer(ar, execPayload, reply, arena, debugID); + serializer(ar, snapPayload, snapUID, reply, arena, debugID); } }; diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp index afebb505b2..393e820d63 100644 --- a/fdbclient/MonitorLeader.actor.cpp +++ b/fdbclient/MonitorLeader.actor.cpp @@ -246,10 +246,10 @@ TEST_CASE("/flow/FlatBuffers/LeaderInfo") { } in.serializedInfo = rndString; } - ObjectWriter writer; + ObjectWriter writer(IncludeVersion()); writer.serialize(in); Standalone copy = writer.toStringRef(); - ArenaObjectReader reader(copy.arena(), copy); + ArenaObjectReader reader(copy.arena(), copy, IncludeVersion()); reader.deserialize(out); ASSERT(in.forward == out.forward); ASSERT(in.changeID == out.changeID); @@ -268,10 +268,10 @@ TEST_CASE("/flow/FlatBuffers/LeaderInfo") { ErrorOr>> objIn(leaderInfo); ErrorOr>> objOut; Standalone copy; - ObjectWriter writer; + ObjectWriter writer(IncludeVersion()); writer.serialize(objIn); copy = writer.toStringRef(); - ArenaObjectReader reader(copy.arena(), copy); + ArenaObjectReader reader(copy.arena(), copy, IncludeVersion()); reader.deserialize(objOut); ASSERT(!objOut.isError()); @@ -725,10 +725,17 @@ ACTOR Future monitorProxiesOneGeneration( Reference monitorProxies( Reference connFile, Reference> clientInfo, Standalone> supportedVersions, Key traceLogGroup ) { - state MonitorLeaderInfo info(connFile); +ACTOR Future monitorProxies( Reference>> connFile, Reference> clientInfo, Standalone> supportedVersions, Key traceLogGroup ) { + state MonitorLeaderInfo info(connFile->get()); loop { - MonitorLeaderInfo _info = wait( monitorProxiesOneGeneration( connFile, clientInfo, info, supportedVersions, traceLogGroup ) ); - info = _info; + choose { + when(MonitorLeaderInfo _info = wait( monitorProxiesOneGeneration( connFile->get(), clientInfo, info, supportedVersions, traceLogGroup ) )) { + info = _info; + } + when(wait(connFile->onChange())) { + info.hasConnected = false; + info.intermediateConnFile = connFile->get(); + } + } } } diff --git a/fdbclient/MonitorLeader.h b/fdbclient/MonitorLeader.h index 2db24134c0..ab0b1c8787 100644 --- a/fdbclient/MonitorLeader.h +++ b/fdbclient/MonitorLeader.h @@ -57,7 +57,7 @@ Future monitorLeader( Reference const& connFile, Re Future monitorLeaderForProxies( Value const& key, vector const& coordinators, ClientData* const& clientData ); -Future monitorProxies( Reference const& connFile, Reference> const& clientInfo, Standalone> const& supportedVersions, Key const& traceLogGroup ); +Future monitorProxies( Reference>> const& connFile, Reference> const& clientInfo, Standalone> const& supportedVersions, Key const& traceLogGroup ); #pragma region Implementation diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index eb973b2659..ce7e6a02d7 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -47,7 +47,7 @@ ThreadFuture DLTransaction::getReadVersion() { return toThreadFuture(api, f, [](FdbCApi::FDBFuture *f, FdbCApi *api) { int64_t version; - FdbCApi::fdb_error_t error = api->futureGetVersion(f, &version); + FdbCApi::fdb_error_t error = api->futureGetInt64(f, &version); ASSERT(!error); return version; }); @@ -195,6 +195,20 @@ Version DLTransaction::getCommittedVersion() { return version; } +ThreadFuture DLTransaction::getApproximateSize() { + if(!api->transactionGetApproximateSize) { + return unsupported_operation(); + } + + FdbCApi::FDBFuture *f = api->transactionGetApproximateSize(tr); + return toThreadFuture(api, f, [](FdbCApi::FDBFuture *f, FdbCApi *api) { + int64_t size = 0; + FdbCApi::fdb_error_t error = api->futureGetInt64(f, &size); + ASSERT(!error); + return size; + }); +} + void DLTransaction::setOption(FDBTransactionOptions::Option option, Optional value) { throwIfError(api->transactionSetOption(tr, option, value.present() ? value.get().begin() : NULL, value.present() ? value.get().size() : 0)); } @@ -287,6 +301,7 @@ void DLApi::init() { loadClientFunction(&api->transactionAtomicOp, lib, fdbCPath, "fdb_transaction_atomic_op"); loadClientFunction(&api->transactionCommit, lib, fdbCPath, "fdb_transaction_commit"); loadClientFunction(&api->transactionGetCommittedVersion, lib, fdbCPath, "fdb_transaction_get_committed_version"); + loadClientFunction(&api->transactionGetApproximateSize, lib, fdbCPath, "fdb_transaction_get_approximate_size", headerVersion >= 620); loadClientFunction(&api->transactionWatch, lib, fdbCPath, "fdb_transaction_watch"); loadClientFunction(&api->transactionOnError, lib, fdbCPath, "fdb_transaction_on_error"); loadClientFunction(&api->transactionReset, lib, fdbCPath, "fdb_transaction_reset"); @@ -294,7 +309,7 @@ void DLApi::init() { loadClientFunction(&api->transactionAddConflictRange, lib, fdbCPath, "fdb_transaction_add_conflict_range"); loadClientFunction(&api->futureGetDatabase, lib, fdbCPath, "fdb_future_get_database"); - loadClientFunction(&api->futureGetVersion, lib, fdbCPath, "fdb_future_get_version"); + loadClientFunction(&api->futureGetInt64, lib, fdbCPath, headerVersion >= 620 ? "fdb_future_get_int64" : "fdb_future_get_version"); loadClientFunction(&api->futureGetError, lib, fdbCPath, "fdb_future_get_error"); loadClientFunction(&api->futureGetKey, lib, fdbCPath, "fdb_future_get_key"); loadClientFunction(&api->futureGetValue, lib, fdbCPath, "fdb_future_get_value"); @@ -595,6 +610,12 @@ Version MultiVersionTransaction::getCommittedVersion() { return invalidVersion; } +ThreadFuture MultiVersionTransaction::getApproximateSize() { + auto tr = getTransaction(); + auto f = tr.transaction ? tr.transaction->getApproximateSize() : ThreadFuture(Never()); + return abortableFuture(f, tr.onChange); +} + void MultiVersionTransaction::setOption(FDBTransactionOptions::Option option, Optional value) { auto itr = FDBTransactionOptions::optionInfo.find(option); if(itr == FDBTransactionOptions::optionInfo.end()) { diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h index b1a1c3372a..e71f6e895f 100644 --- a/fdbclient/MultiVersionTransaction.h +++ b/fdbclient/MultiVersionTransaction.h @@ -84,6 +84,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted { FDBFuture* (*transactionCommit)(FDBTransaction *tr); fdb_error_t (*transactionGetCommittedVersion)(FDBTransaction *tr, int64_t *outVersion); + FDBFuture* (*transactionGetApproximateSize)(FDBTransaction *tr); FDBFuture* (*transactionWatch)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength); FDBFuture* (*transactionOnError)(FDBTransaction *tr, fdb_error_t error); void (*transactionReset)(FDBTransaction *tr); @@ -94,7 +95,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted { //Future fdb_error_t (*futureGetDatabase)(FDBFuture *f, FDBDatabase **outDb); - fdb_error_t (*futureGetVersion)(FDBFuture *f, int64_t *outVersion); + fdb_error_t (*futureGetInt64)(FDBFuture *f, int64_t *outValue); fdb_error_t (*futureGetError)(FDBFuture *f); fdb_error_t (*futureGetKey)(FDBFuture *f, uint8_t const **outKey, int *outKeyLength); fdb_error_t (*futureGetValue)(FDBFuture *f, fdb_bool_t *outPresent, uint8_t const **outValue, int *outValueLength); @@ -116,41 +117,42 @@ public: DLTransaction(Reference api, FdbCApi::FDBTransaction *tr) : api(api), tr(tr) {} ~DLTransaction() { api->transactionDestroy(tr); } - void cancel(); - void setVersion(Version v); - ThreadFuture getReadVersion(); + void cancel() override; + void setVersion(Version v) override; + ThreadFuture getReadVersion() override; - ThreadFuture> get(const KeyRef& key, bool snapshot=false); - ThreadFuture getKey(const KeySelectorRef& key, bool snapshot=false); - ThreadFuture> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false); - ThreadFuture> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false); - ThreadFuture> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false); - ThreadFuture> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false); - ThreadFuture>> getAddressesForKey(const KeyRef& key); - ThreadFuture> getVersionstamp(); + ThreadFuture> get(const KeyRef& key, bool snapshot=false) override; + ThreadFuture getKey(const KeySelectorRef& key, bool snapshot=false) override; + ThreadFuture> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false) override; + ThreadFuture> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override; + ThreadFuture> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false) override; + ThreadFuture> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override; + ThreadFuture>> getAddressesForKey(const KeyRef& key) override; + ThreadFuture> getVersionstamp() override; - void addReadConflictRange(const KeyRangeRef& keys); + void addReadConflictRange(const KeyRangeRef& keys) override; - void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType); - void set(const KeyRef& key, const ValueRef& value); - void clear(const KeyRef& begin, const KeyRef& end); - void clear(const KeyRangeRef& range); - void clear(const KeyRef& key); + void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) override; + void set(const KeyRef& key, const ValueRef& value) override; + void clear(const KeyRef& begin, const KeyRef& end) override; + void clear(const KeyRangeRef& range) override; + void clear(const KeyRef& key) override; - ThreadFuture watch(const KeyRef& key); + ThreadFuture watch(const KeyRef& key) override; - void addWriteConflictRange(const KeyRangeRef& keys); + void addWriteConflictRange(const KeyRangeRef& keys) override; - ThreadFuture commit(); - Version getCommittedVersion(); + ThreadFuture commit() override; + Version getCommittedVersion() override; + ThreadFuture getApproximateSize() override; - void setOption(FDBTransactionOptions::Option option, Optional value=Optional()); + void setOption(FDBTransactionOptions::Option option, Optional value=Optional()) override; - ThreadFuture onError(Error const& e); - void reset(); + ThreadFuture onError(Error const& e) override; + void reset() override; - void addref() { ThreadSafeReferenceCounted::addref(); } - void delref() { ThreadSafeReferenceCounted::delref(); } + void addref() override { ThreadSafeReferenceCounted::addref(); } + void delref() override { ThreadSafeReferenceCounted::delref(); } private: const Reference api; @@ -165,11 +167,11 @@ public: ThreadFuture onReady(); - Reference createTransaction(); - void setOption(FDBDatabaseOptions::Option option, Optional value = Optional()); + Reference createTransaction() override; + void setOption(FDBDatabaseOptions::Option option, Optional value = Optional()) override; - void addref() { ThreadSafeReferenceCounted::addref(); } - void delref() { ThreadSafeReferenceCounted::delref(); } + void addref() override { ThreadSafeReferenceCounted::addref(); } + void delref() override { ThreadSafeReferenceCounted::delref(); } private: const Reference api; @@ -181,18 +183,18 @@ class DLApi : public IClientApi { public: DLApi(std::string fdbCPath); - void selectApiVersion(int apiVersion); - const char* getClientVersion(); + void selectApiVersion(int apiVersion) override; + const char* getClientVersion() override; - void setNetworkOption(FDBNetworkOptions::Option option, Optional value = Optional()); - void setupNetwork(); - void runNetwork(); - void stopNetwork(); + void setNetworkOption(FDBNetworkOptions::Option option, Optional value = Optional()) override; + void setupNetwork() override; + void runNetwork() override; + void stopNetwork() override; - Reference createDatabase(const char *clusterFilePath); + Reference createDatabase(const char *clusterFilePath) override; Reference createDatabase609(const char *clusterFilePath); // legacy database creation - void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter); + void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter) override; private: const std::string fdbCPath; @@ -212,41 +214,42 @@ class MultiVersionTransaction : public ITransaction, ThreadSafeReferenceCounted< public: MultiVersionTransaction(Reference db, UniqueOrderedOptionList defaultOptions); - void cancel(); - void setVersion(Version v); - ThreadFuture getReadVersion(); + void cancel() override; + void setVersion(Version v) override; + ThreadFuture getReadVersion() override; - ThreadFuture> get(const KeyRef& key, bool snapshot=false); - ThreadFuture getKey(const KeySelectorRef& key, bool snapshot=false); - ThreadFuture> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false); - ThreadFuture> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false); - ThreadFuture> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false); - ThreadFuture> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false); - ThreadFuture>> getAddressesForKey(const KeyRef& key); - ThreadFuture> getVersionstamp(); + ThreadFuture> get(const KeyRef& key, bool snapshot=false) override; + ThreadFuture getKey(const KeySelectorRef& key, bool snapshot=false) override; + ThreadFuture> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot=false, bool reverse=false) override; + ThreadFuture> getRange(const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override; + ThreadFuture> getRange(const KeyRangeRef& keys, int limit, bool snapshot=false, bool reverse=false) override; + ThreadFuture> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override; + ThreadFuture>> getAddressesForKey(const KeyRef& key) override; + ThreadFuture> getVersionstamp() override; - void addReadConflictRange(const KeyRangeRef& keys); + void addReadConflictRange(const KeyRangeRef& keys) override; - void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType); - void set(const KeyRef& key, const ValueRef& value); - void clear(const KeyRef& begin, const KeyRef& end); - void clear(const KeyRangeRef& range); - void clear(const KeyRef& key); + void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) override; + void set(const KeyRef& key, const ValueRef& value) override; + void clear(const KeyRef& begin, const KeyRef& end) override; + void clear(const KeyRangeRef& range) override; + void clear(const KeyRef& key) override; - ThreadFuture watch(const KeyRef& key); + ThreadFuture watch(const KeyRef& key) override; - void addWriteConflictRange(const KeyRangeRef& keys); + void addWriteConflictRange(const KeyRangeRef& keys) override; - ThreadFuture commit(); - Version getCommittedVersion(); + ThreadFuture commit() override; + Version getCommittedVersion() override; + ThreadFuture getApproximateSize() override; - void setOption(FDBTransactionOptions::Option option, Optional value=Optional()); + void setOption(FDBTransactionOptions::Option option, Optional value=Optional()) override; - ThreadFuture onError(Error const& e); - void reset(); + ThreadFuture onError(Error const& e) override; + void reset() override; - void addref() { ThreadSafeReferenceCounted::addref(); } - void delref() { ThreadSafeReferenceCounted::delref(); } + void addref() override { ThreadSafeReferenceCounted::addref(); } + void delref() override { ThreadSafeReferenceCounted::delref(); } private: const Reference db; @@ -289,11 +292,11 @@ public: MultiVersionDatabase(MultiVersionApi *api, std::string clusterFilePath, Reference db, bool openConnectors=true); ~MultiVersionDatabase(); - Reference createTransaction(); - void setOption(FDBDatabaseOptions::Option option, Optional value = Optional()); + Reference createTransaction() override; + void setOption(FDBDatabaseOptions::Option option, Optional value = Optional()) override; - void addref() { ThreadSafeReferenceCounted::addref(); } - void delref() { ThreadSafeReferenceCounted::delref(); } + void addref() override { ThreadSafeReferenceCounted::addref(); } + void delref() override { ThreadSafeReferenceCounted::delref(); } static Reference debugCreateFromExistingDatabase(Reference db); @@ -354,16 +357,16 @@ private: class MultiVersionApi : public IClientApi { public: - void selectApiVersion(int apiVersion); - const char* getClientVersion(); + void selectApiVersion(int apiVersion) override; + const char* getClientVersion() override; - void setNetworkOption(FDBNetworkOptions::Option option, Optional value = Optional()); - void setupNetwork(); - void runNetwork(); - void stopNetwork(); - void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter); + void setNetworkOption(FDBNetworkOptions::Option option, Optional value = Optional()) override; + void setupNetwork() override; + void runNetwork() override; + void stopNetwork() override; + void addNetworkThreadCompletionHook(void (*hook)(void*), void *hookParameter) override; - Reference createDatabase(const char *clusterFilePath); + Reference createDatabase(const char *clusterFilePath) override; static MultiVersionApi* api; Reference getLocalClient(); diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 2b82c4967b..e6ef1c4b24 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -29,6 +29,7 @@ #include "fdbclient/FailureMonitorClient.h" #include "fdbclient/KeyRangeMap.h" #include "fdbclient/Knobs.h" +#include "fdbclient/ManagementAPI.actor.h" #include "fdbclient/MasterProxyInterface.h" #include "fdbclient/MonitorLeader.h" #include "fdbclient/MutationList.h" @@ -175,13 +176,17 @@ std::string unprintable( std::string const& val ) { return s; } -void validateVersion(Version version) { - // Version could be 0 if the INITIALIZE_NEW_DATABASE option is set. In that case, it is illegal to perform any reads. - // We throw client_invalid_operation because the caller didn't directly set the version, so the version_invalid error - // might be confusing. - if(version == 0) { +void DatabaseContext::validateVersion(Version version) { + // Version could be 0 if the INITIALIZE_NEW_DATABASE option is set. In that case, it is illegal to perform any + // reads. We throw client_invalid_operation because the caller didn't directly set the version, so the + // version_invalid error might be confusing. + if (version == 0) { throw client_invalid_operation(); } + if (switchable && version < minAcceptableReadVersion) { + TEST(true); // Attempted to read a version lower than any this client has seen from the current cluster + throw transaction_too_old(); + } ASSERT(version > 0 || version == latestVersion); } @@ -505,10 +510,11 @@ ACTOR static Future getHealthMetricsActor(DatabaseContext *cx, bo Future DatabaseContext::getHealthMetrics(bool detailed = false) { return getHealthMetricsActor(this, detailed); } -DatabaseContext::DatabaseContext( Reference connFile, Reference> clientInfo, Future clientInfoMonitor, - TaskPriority taskID, LocalityData const& clientLocality, bool enableLocalityLoadBalance, bool lockAware, bool internal, int apiVersion ) - : connFile(connFile), clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), taskID(taskID), clientLocality(clientLocality), enableLocalityLoadBalance(enableLocalityLoadBalance), - lockAware(lockAware), apiVersion(apiVersion), provisional(false), cc("TransactionMetrics"), +DatabaseContext::DatabaseContext( + Reference>> connectionFile, Reference> clientInfo, Future clientInfoMonitor, + TaskPriority taskID, LocalityData const& clientLocality, bool enableLocalityLoadBalance, bool lockAware, bool internal, int apiVersion, bool switchable ) + : connectionFile(connectionFile),clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), taskID(taskID), clientLocality(clientLocality), enableLocalityLoadBalance(enableLocalityLoadBalance), + lockAware(lockAware), apiVersion(apiVersion), switchable(switchable), provisional(false), cc("TransactionMetrics"), transactionReadVersions("ReadVersions", cc), transactionLogicalReads("LogicalUncachedReads", cc), transactionPhysicalReads("PhysicalReadRequests", cc), transactionCommittedMutations("CommittedMutations", cc), transactionCommittedMutationBytes("CommittedMutationBytes", cc), transactionsCommitStarted("CommitStarted", cc), transactionsCommitCompleted("CommitCompleted", cc), transactionsTooOld("TooOld", cc), transactionsFutureVersions("FutureVersions", cc), @@ -546,8 +552,9 @@ DatabaseContext::DatabaseContext( const Error &err ) : deferredError(err), cc("T GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), internal(false) {} -Database DatabaseContext::create( Reference> clientInfo, Future clientInfoMonitor, LocalityData clientLocality, bool enableLocalityLoadBalance, TaskPriority taskID, bool lockAware, int apiVersion) { - return Database( new DatabaseContext( Reference(), clientInfo, clientInfoMonitor, taskID, clientLocality, enableLocalityLoadBalance, lockAware, true, apiVersion ) ); + +Database DatabaseContext::create(Reference> clientInfo, Future clientInfoMonitor, LocalityData clientLocality, bool enableLocalityLoadBalance, TaskPriority taskID, bool lockAware, int apiVersion, bool switchable) { + return Database( new DatabaseContext( Reference>>(), clientInfo, clientInfoMonitor, taskID, clientLocality, enableLocalityLoadBalance, lockAware, true, apiVersion, switchable ) ); } DatabaseContext::~DatabaseContext() { @@ -714,8 +721,50 @@ Future DatabaseContext::onConnected() { return connected; } +ACTOR static Future switchConnectionFileImpl(Reference connFile, DatabaseContext* self) { + TEST(true); // Switch connection file + TraceEvent("SwitchConnectionFile") + .detail("ConnectionFile", connFile->canGetFilename() ? connFile->getFilename() : "") + .detail("ConnectionString", connFile->getConnectionString().toString()); + + // Reset state from former cluster. + self->masterProxies.clear(); + self->masterProxiesChangeTrigger.trigger(); + self->minAcceptableReadVersion = std::numeric_limits::max(); + self->invalidateCache(allKeys); + + self->connectionFile->set(connFile); + state Database db(Reference::addRef(self)); + state Transaction tr(db); + loop { + tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE); + try { + TraceEvent("SwitchConnectionFileAttemptingGRV"); + Version v = wait(tr.getReadVersion()); + TraceEvent("SwitchConnectionFileGotRV") + .detail("ReadVersion", v) + .detail("MinAcceptableReadVersion", self->minAcceptableReadVersion); + ASSERT(self->minAcceptableReadVersion != std::numeric_limits::max()); + self->connectionFileChangedTrigger.trigger(); + return Void(); + } catch (Error& e) { + TraceEvent("SwitchConnectionFileError").detail("Error", e.what()); + wait(tr.onError(e)); + } + } +} + Reference DatabaseContext::getConnectionFile() { - return connFile; + return connectionFile->get(); +} + +Future DatabaseContext::switchConnectionFile(Reference standby) { + ASSERT(switchable); + return switchConnectionFileImpl(standby, this); +} + +Future DatabaseContext::connectionFileChanged() { + return connectionFileChangedTrigger.onTrigger(); } extern IPAddress determinePublicIPAutomatically(ClusterConnectionString const& ccs); @@ -752,16 +801,16 @@ Database Database::createDatabase( Reference connFile, in } } - Reference> clientInfo(new AsyncVar()); - Future clientInfoMonitor = monitorProxies(connFile, clientInfo, networkOptions.supportedVersions, StringRef(networkOptions.traceLogGroup)); + Reference>> connectionFile(new AsyncVar>()); + Future clientInfoMonitor = monitorProxies(connectionFile, clientInfo, networkOptions.supportedVersions, StringRef(networkOptions.traceLogGroup)); DatabaseContext *db; if(preallocatedDb) { - db = new (preallocatedDb) DatabaseContext(connFile, clientInfo, clientInfoMonitor, TaskPriority::DefaultEndpoint, clientLocality, true, false, internal, apiVersion); + db = new (preallocatedDb) DatabaseContext(connectionFile, clientInfo, clientInfoMonitor, TaskPriority::DefaultEndpoint, clientLocality, true, false, internal, apiVersion, /*switchable*/ true); } else { - db = new DatabaseContext(connFile, clientInfo, clientInfoMonitor, TaskPriority::DefaultEndpoint, clientLocality, true, false, internal, apiVersion); + db = new DatabaseContext(connectionFile, clientInfo, clientInfoMonitor, TaskPriority::DefaultEndpoint, clientLocality, true, false, internal, apiVersion, /*switchable*/ true); } return Database(db); @@ -1252,7 +1301,7 @@ Future Transaction::warmRange(Database cx, KeyRange keys) { ACTOR Future> getValue( Future version, Key key, Database cx, TransactionInfo info, Reference trLogInfo ) { state Version ver = wait( version ); - validateVersion(ver); + cx->validateVersion(ver); loop { state pair> ssi = wait( getKeyLocation(cx, key, &StorageServerInterface::getValue, info) ); @@ -1277,13 +1326,22 @@ ACTOR Future> getValue( Future version, Key key, Databa startTime = timer_int(); startTimeD = now(); ++cx->transactionPhysicalReads; + if (CLIENT_BUGGIFY) { throw deterministicRandom()->randomChoice( std::vector{ transaction_too_old(), future_version() }); } - state GetValueReply reply = wait( - loadBalance(ssi.second, &StorageServerInterface::getValue, GetValueRequest(key, ver, getValueID), - TaskPriority::DefaultPromiseEndpoint, false, cx->enableLocalityLoadBalance ? &cx->queueModel : NULL)); + state GetValueReply reply; + choose { + when(wait(cx->connectionFileChanged())) { throw transaction_too_old(); } + when(GetValueReply _reply = + wait(loadBalance(ssi.second, &StorageServerInterface::getValue, + GetValueRequest(key, ver, getValueID), TaskPriority::DefaultPromiseEndpoint, false, + cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { + reply = _reply; + } + } + double latency = now() - startTimeD; cx->readLatencies.addSample(latency); if (trLogInfo) { @@ -1346,7 +1404,16 @@ ACTOR Future getKey( Database cx, KeySelector k, Future version, T if( info.debugID.present() ) g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKey.Before"); //.detail("StartKey", k.getKey()).detail("Offset",k.offset).detail("OrEqual",k.orEqual); ++cx->transactionPhysicalReads; - GetKeyReply reply = wait( loadBalance( ssi.second, &StorageServerInterface::getKey, GetKeyRequest(k, version.get()), TaskPriority::DefaultPromiseEndpoint, false, cx->enableLocalityLoadBalance ? &cx->queueModel : NULL ) ); + state GetKeyReply reply; + choose { + when(wait(cx->connectionFileChanged())) { throw transaction_too_old(); } + when(GetKeyReply _reply = + wait(loadBalance(ssi.second, &StorageServerInterface::getKey, GetKeyRequest(k, version.get()), + TaskPriority::DefaultPromiseEndpoint, false, + cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { + reply = _reply; + } + } if( info.debugID.present() ) g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKey.After"); //.detail("NextKey",reply.sel.key).detail("Offset", reply.sel.offset).detail("OrEqual", k.orEqual); k = reply.sel; @@ -1375,6 +1442,7 @@ ACTOR Future waitForCommittedVersion( Database cx, Version version ) { choose { when ( wait( cx->onMasterProxiesChanged() ) ) {} when ( GetReadVersionReply v = wait( loadBalance( cx->getMasterProxies(false), &MasterProxyInterface::getConsistentReadVersion, GetReadVersionRequest( 0, GetReadVersionRequest::PRIORITY_SYSTEM_IMMEDIATE ), cx->taskID ) ) ) { + cx->minAcceptableReadVersion = std::min(cx->minAcceptableReadVersion, v.version); if(v.newClientInfo.present()) { cx->clientInfo->set(v.newClientInfo.get()); continue; @@ -1397,10 +1465,10 @@ ACTOR Future readVersionBatcher( DatabaseContext* cx, FutureStream, Optional>> versionStream, uint32_t flags); -ACTOR Future< Void > watchValue( Future version, Key key, Optional value, Database cx, int readVersionFlags, TransactionInfo info ) -{ +ACTOR Future watchValue(Future version, Key key, Optional value, Database cx, + TransactionInfo info) { state Version ver = wait( version ); - validateVersion(ver); + cx->validateVersion(ver); ASSERT(ver != latestVersion); loop { @@ -1414,7 +1482,15 @@ ACTOR Future< Void > watchValue( Future version, Key key, OptionalgetCurrentTask()); } - state Version resp = wait( loadBalance( ssi.second, &StorageServerInterface::watchValue, WatchValueRequest(key, value, ver, watchValueID), TaskPriority::DefaultPromiseEndpoint ) ); + state Version resp; + choose { + when(Version r = wait(loadBalance(ssi.second, &StorageServerInterface::watchValue, + WatchValueRequest(key, value, ver, watchValueID), + TaskPriority::DefaultPromiseEndpoint))) { + resp = r; + } + when(wait(cx->connectionFile ? cx->connectionFile->onChange() : Never())) { wait(Never()); } + } if( info.debugID.present() ) { g_traceBatch.addEvent("WatchValueDebug", watchValueID.get().first(), "NativeAPI.watchValue.After"); //.detail("TaskID", g_network->getCurrentTask()); } @@ -1506,7 +1582,16 @@ ACTOR Future> getExactRange( Database cx, Version ver .detail("Servers", locations[shard].second->description());*/ } ++cx->transactionPhysicalReads; - GetKeyValuesReply rep = wait( loadBalance( locations[shard].second, &StorageServerInterface::getKeyValues, req, TaskPriority::DefaultPromiseEndpoint, false, cx->enableLocalityLoadBalance ? &cx->queueModel : NULL ) ); + state GetKeyValuesReply rep; + choose { + when(wait(cx->connectionFileChanged())) { throw transaction_too_old(); } + when(GetKeyValuesReply _rep = + wait(loadBalance(locations[shard].second, &StorageServerInterface::getKeyValues, req, + TaskPriority::DefaultPromiseEndpoint, false, + cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { + rep = _rep; + } + } if( info.debugID.present() ) g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getExactRange.After"); output.arena().dependsOn( rep.arena ); @@ -1711,7 +1796,7 @@ ACTOR Future> getRange( Database cx, ReferencevalidateVersion(version); state double startTime = now(); state Version readVersion = version; // Needed for latestVersion requests; if more, make future requests at the version that the first one completed @@ -1738,6 +1823,7 @@ ACTOR Future> getRange( Database cx, Reference watchFuture) { //FIXME: This seems pretty horrible. Now a Database can't die until all of its watches do... ACTOR Future watch( Reference watch, Database cx, Transaction *self ) { + state TransactionInfo info = self->info; cx->addWatch(); try { self->watches.push_back(watch); @@ -2029,8 +2116,18 @@ ACTOR Future watch( Reference watch, Database cx, Transaction *self // NativeAPI finished commit and updated watchFuture when(wait(watch->onSetWatchTrigger.getFuture())) { - // NativeAPI watchValue future finishes or errors - wait(watch->watchFuture); + loop { + choose { + // NativeAPI watchValue future finishes or errors + when(wait(watch->watchFuture)) { break; } + + when(wait(cx->connectionFileChanged())) { + TEST(true); // Recreated a watch after switch + watch->watchFuture = + watchValue(cx->minAcceptableReadVersion, watch->key, watch->value, cx, info); + } + } + } } } } @@ -2236,45 +2333,6 @@ void Transaction::atomicOp(const KeyRef& key, const ValueRef& operand, MutationR TEST(true); //NativeAPI atomic operation } -ACTOR Future executeCoordinators(DatabaseContext* cx, StringRef execPayload, Optional debugID) { - try { - if (debugID.present()) { - g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.executeCoordinators.Before"); - } - - state ExecRequest req(execPayload, debugID); - if (debugID.present()) { - g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), - "NativeAPI.executeCoordinators.Inside loop"); - } - wait(loadBalance(cx->getMasterProxies(false), &MasterProxyInterface::execReq, req, cx->taskID)); - if (debugID.present()) - g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), - "NativeAPI.executeCoordinators.After"); - return Void(); - } catch (Error& e) { - TraceEvent("NativeAPI.executeCoordinatorsError").error(e); - throw; - } -} - -void Transaction::execute(const KeyRef& cmdType, const ValueRef& cmdPayload) { - TraceEvent("Execute operation").detail("Key", cmdType.toString()).detail("Value", cmdPayload.toString()); - - if (cmdType.size() > CLIENT_KNOBS->KEY_SIZE_LIMIT) throw key_too_large(); - if (cmdPayload.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) throw value_too_large(); - - auto& req = tr; - - // Helps with quickly finding the exec op in a tlog batch - setOption(FDBTransactionOptions::FIRST_IN_BATCH); - - auto& t = req.transaction; - auto r = singleKeyRange(cmdType, req.arena); - auto v = ValueRef(req.arena, cmdPayload); - t.mutations.push_back(req.arena, MutationRef(MutationRef::Exec, r.begin, v)); -} - void Transaction::clear( const KeyRangeRef& range, bool addConflictRange ) { auto &req = tr; auto &t = req.transaction; @@ -2537,7 +2595,7 @@ void Transaction::setupWatches() { Future watchVersion = getCommittedVersion() > 0 ? getCommittedVersion() : getReadVersion(); for(int i = 0; i < watches.size(); ++i) - watches[i]->setWatch(watchValue( watchVersion, watches[i]->key, watches[i]->value, cx, options.getReadVersionFlags, info )); + watches[i]->setWatch(watchValue(watchVersion, watches[i]->key, watches[i]->value, cx, info)); watches.clear(); } @@ -2691,7 +2749,7 @@ Future Transaction::commitMutations() { cx->mutationsPerCommit.addSample(tr.transaction.mutations.size()); cx->bytesPerCommit.addSample(tr.transaction.mutations.expectedSize()); - size_t transactionSize = tr.transaction.mutations.expectedSize() + tr.transaction.read_conflict_ranges.expectedSize() + tr.transaction.write_conflict_ranges.expectedSize(); + size_t transactionSize = getSize(); if (transactionSize > (uint64_t)FLOW_KNOBS->PACKET_WARNING) { TraceEvent(!g_network->isSimulated() ? SevWarnAlways : SevWarn, "LargeTransaction") .suppressFor(1.0) @@ -2861,6 +2919,7 @@ void Transaction::setOption( FDBTransactionOptions::Option option, Optional(new TransactionLogInfo(value.get().printable(), TransactionLogInfo::DONT_LOG)); + trLogInfo->maxFieldLength = options.maxTransactionLoggingFieldLength; } break; @@ -2875,6 +2934,20 @@ void Transaction::setOption( FDBTransactionOptions::Option option, Optional::max()); + if(maxFieldLength == 0) { + throw invalid_option_value(); + } + options.maxTransactionLoggingFieldLength = maxFieldLength; + } + if(trLogInfo) { + trLogInfo->maxFieldLength = options.maxTransactionLoggingFieldLength; + } + break; + case FDBTransactionOptions::MAX_RETRY_DELAY: validateOptionValue(value, true); options.maxBackoff = extractIntOption(value, 0, std::numeric_limits::max()) / 1000.0; @@ -2931,6 +3004,7 @@ ACTOR Future getConsistentReadVersion( DatabaseContext *cx, if( debugID.present() ) g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getConsistentReadVersion.After"); ASSERT( v.version > 0 ); + cx->minAcceptableReadVersion = std::min(cx->minAcceptableReadVersion, v.version); return v; } } @@ -3001,12 +3075,12 @@ ACTOR Future readVersionBatcher( DatabaseContext *cx, FutureStream< std::p } } -ACTOR Future extractReadVersion(DatabaseContext* cx, Reference trLogInfo, Future f, bool lockAware, double startTime, Promise> metadataVersion) { +ACTOR Future extractReadVersion(DatabaseContext* cx, uint32_t flags, Reference trLogInfo, Future f, bool lockAware, double startTime, Promise> metadataVersion) { GetReadVersionReply rep = wait(f); double latency = now() - startTime; cx->GRVLatencies.addSample(latency); if (trLogInfo) - trLogInfo->addLog(FdbClientLogEvents::EventGetVersion(startTime, latency)); + trLogInfo->addLog(FdbClientLogEvents::EventGetVersion_V2(startTime, latency, flags & GetReadVersionRequest::FLAG_PRIORITY_MASK)); if(rep.locked && !lockAware) throw database_locked(); @@ -3031,7 +3105,7 @@ Future Transaction::getReadVersion(uint32_t flags) { Promise p; batcher.stream.send( std::make_pair( p, info.debugID ) ); startTime = now(); - readVersion = extractReadVersion( cx.getPtr(), trLogInfo, p.getFuture(), options.lockAware, startTime, metadataVersion); + readVersion = extractReadVersion( cx.getPtr(), flags, trLogInfo, p.getFuture(), options.lockAware, startTime, metadataVersion); } return readVersion; } @@ -3043,6 +3117,12 @@ Future> Transaction::getVersionstamp() { return versionstampPromise.getFuture(); } +uint32_t Transaction::getSize() { + auto s = tr.transaction.mutations.expectedSize() + tr.transaction.read_conflict_ranges.expectedSize() + + tr.transaction.write_conflict_ranges.expectedSize(); + return s; +} + Future Transaction::onError( Error const& e ) { if (e.code() == error_code_success) { return client_invalid_operation(); @@ -3284,101 +3364,84 @@ void enableClientInfoLogging() { TraceEvent(SevInfo, "ClientInfoLoggingEnabled"); } -ACTOR Future snapCreate(Database inputCx, StringRef snapCmd, UID snapUID) { - state Transaction tr(inputCx); - state DatabaseContext* cx = inputCx.getPtr(); +ACTOR Future snapshotDatabase(Reference cx, StringRef snapPayload, UID snapUID, Optional debugID) { + TraceEvent("NativeAPI.SnapshotDatabaseEnter") + .detail("SnapPayload", snapPayload) + .detail("SnapUID", snapUID); + try { + if (debugID.present()) { + g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.snapshotDatabase.Before"); + } + + ProxySnapRequest req(snapPayload, snapUID, debugID); + wait(loadBalance(cx->getMasterProxies(false), &MasterProxyInterface::proxySnapReq, req, cx->taskID, true /*atmostOnce*/ )); + if (debugID.present()) + g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), + "NativeAPI.SnapshotDatabase.After"); + } catch (Error& e) { + TraceEvent("NativeAPI.SnapshotDatabaseError") + .detail("SnapPayload", snapPayload) + .detail("SnapUID", snapUID) + .error(e, true /* includeCancelled */); + throw; + } + return Void(); +} + +ACTOR Future snapCreateCore(Database cx, StringRef snapCmd, UID snapUID) { // remember the client ID before the snap operation state UID preSnapClientUID = cx->clientInfo->get().id; - TraceEvent("SnapCreateEnter") + TraceEvent("SnapCreateCoreEnter") .detail("SnapCmd", snapCmd.toString()) .detail("UID", snapUID) .detail("PreSnapClientUID", preSnapClientUID); StringRef snapCmdArgs = snapCmd; StringRef snapCmdPart = snapCmdArgs.eat(":"); - state Standalone snapUIDRef(snapUID.toString()); - state Standalone snapPayloadRef = snapCmdPart + Standalone snapUIDRef(snapUID.toString()); + Standalone snapPayloadRef = snapCmdPart .withSuffix(LiteralStringRef(":uid=")) .withSuffix(snapUIDRef) .withSuffix(LiteralStringRef(",")) .withSuffix(snapCmdArgs); - state Standalone - tLogCmdPayloadRef = LiteralStringRef("empty-binary:uid=").withSuffix(snapUIDRef); - // disable popping of TLog - tr.reset(); - loop { - try { - tr.setOption(FDBTransactionOptions::LOCK_AWARE); - tr.execute(execDisableTLogPop, tLogCmdPayloadRef); - wait(timeoutError(tr.commit(), 10)); - break; - } catch (Error& e) { - TraceEvent("DisableTLogPopFailed").error(e); - wait(tr.onError(e)); - } - } - TraceEvent("SnapCreateAfterLockingTLogs").detail("UID", snapUID); - - // snap the storage and Tlogs - // if we retry the below command in failure cases with the same snapUID - // then the snapCreate can end up creating multiple snapshots with - // the same name which needs additional handling, hence we fail in - // failure cases and let the caller retry with different snapUID - tr.reset(); try { - tr.setOption(FDBTransactionOptions::LOCK_AWARE); - tr.execute(execSnap, snapPayloadRef); - wait(tr.commit()); + Future exec = snapshotDatabase(Reference::addRef(cx.getPtr()), snapPayloadRef, snapUID, snapUID); + wait(exec); } catch (Error& e) { - TraceEvent("SnapCreateErroSnapTLogStorage").error(e); + TraceEvent("SnapCreateCoreError") + .detail("SnapCmd", snapCmd.toString()) + .detail("UID", snapUID) + .error(e); throw; } - TraceEvent("SnapCreateAfterSnappingTLogStorage").detail("UID", snapUID); - - if (BUGGIFY) { - int32_t toDelay = deterministicRandom()->randomInt(1, 30); - wait(delay(toDelay)); - } - - // enable popping of the TLog - tr.reset(); - loop { - try { - tr.setOption(FDBTransactionOptions::LOCK_AWARE); - tr.execute(execEnableTLogPop, tLogCmdPayloadRef); - wait(tr.commit()); - break; - } catch (Error& e) { - TraceEvent("EnableTLogPopFailed").error(e); - wait(tr.onError(e)); - } - } - - TraceEvent("SnapCreateAfterUnlockingTLogs").detail("UID", snapUID); - - // snap the coordinators - try { - Future exec = executeCoordinators(cx, snapPayloadRef, snapUID); - wait(timeoutError(exec, 5.0)); - } catch (Error& e) { - TraceEvent("SnapCreateErrorSnapCoords").error(e); - throw; - } - - TraceEvent("SnapCreateAfterSnappingCoords").detail("UID", snapUID); - - // if the client IDs did not change then we have a clean snapshot UID postSnapClientUID = cx->clientInfo->get().id; if (preSnapClientUID != postSnapClientUID) { - TraceEvent("UID mismatch") + // if the client IDs changed then we fail the snapshot + TraceEvent("SnapCreateCoreUIDMismatch") .detail("SnapPreSnapClientUID", preSnapClientUID) .detail("SnapPostSnapClientUID", postSnapClientUID); throw coordinators_changed(); } - TraceEvent("SnapCreateComplete").detail("UID", snapUID); + TraceEvent("SnapCreateCoreExit") + .detail("SnapCmd", snapCmd.toString()) + .detail("UID", snapUID) + .detail("PreSnapClientUID", preSnapClientUID); + return Void(); +} + +ACTOR Future snapCreate(Database cx, StringRef snapCmd, UID snapUID) { + state int oldMode = wait( setDDMode( cx, 0 ) ); + try { + wait(snapCreateCore(cx, snapCmd, snapUID)); + } catch (Error& e) { + state Error err = e; + wait(success( setDDMode( cx, oldMode ) )); + throw err; + } + wait(success( setDDMode( cx, oldMode ) )); return Void(); } diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h index 50f1508e57..5358e496b7 100644 --- a/fdbclient/NativeAPI.actor.h +++ b/fdbclient/NativeAPI.actor.h @@ -67,7 +67,7 @@ struct NetworkOptions { NetworkOptions() : localAddress(""), clusterFile(""), traceDirectory(Optional()), traceRollSize(TRACE_DEFAULT_ROLL_SIZE), traceMaxLogsSize(TRACE_DEFAULT_MAX_LOGS_SIZE), traceLogGroup("default"), - traceFormat("xml"), slowTaskProfilingEnabled(false), useObjectSerializer(false) {} + traceFormat("xml"), slowTaskProfilingEnabled(false), useObjectSerializer(true) {} }; class Database { @@ -124,6 +124,7 @@ struct TransactionOptions { double maxBackoff; uint32_t getReadVersionFlags; uint32_t sizeLimit; + int maxTransactionLoggingFieldLength; bool checkWritesEnabled : 1; bool causalWriteRisky : 1; bool commitOnFirstProxy : 1; @@ -149,17 +150,18 @@ struct TransactionInfo { struct TransactionLogInfo : public ReferenceCounted, NonCopyable { enum LoggingLocation { DONT_LOG = 0, TRACE_LOG = 1, DATABASE = 2 }; - TransactionLogInfo() : logLocation(DONT_LOG) {} - TransactionLogInfo(LoggingLocation location) : logLocation(location) {} - TransactionLogInfo(std::string id, LoggingLocation location) : logLocation(location), identifier(id) {} + TransactionLogInfo() : logLocation(DONT_LOG), maxFieldLength(0) {} + TransactionLogInfo(LoggingLocation location) : logLocation(location), maxFieldLength(0) {} + TransactionLogInfo(std::string id, LoggingLocation location) : logLocation(location), identifier(id), maxFieldLength(0) {} void setIdentifier(std::string id) { identifier = id; } void logTo(LoggingLocation loc) { logLocation = logLocation | loc; } + template void addLog(const T& event) { if(logLocation & TRACE_LOG) { ASSERT(!identifier.empty()) - event.logEvent(identifier); + event.logEvent(identifier, maxFieldLength); } if (flushed) { @@ -177,6 +179,7 @@ struct TransactionLogInfo : public ReferenceCounted, NonCopy bool logsAdded{ false }; bool flushed{ false }; int logLocation; + int maxFieldLength; std::string identifier; }; @@ -240,14 +243,6 @@ public: // If checkWriteConflictRanges is true, existing write conflict ranges will be searched for this key void set( const KeyRef& key, const ValueRef& value, bool addConflictRange = true ); void atomicOp( const KeyRef& key, const ValueRef& value, MutationRef::Type operationType, bool addConflictRange = true ); - // execute operation is similar to set, but the command will reach - // one of the proxies, all the TLogs and all the storage nodes. - // instead of setting a key and value on the DB, it executes the command - // that is passed in the value field. - // - cmdType can be used for logging purposes - // - cmdPayload contains the details of the command to be executed: - // format of the cmdPayload : :,... - void execute(const KeyRef& cmdType, const ValueRef& cmdPayload); void clear( const KeyRangeRef& range, bool addConflictRange = true ); void clear( const KeyRef& key, bool addConflictRange = true ); Future commit(); // Throws not_committed or commit_unknown_result errors in normal operation @@ -259,6 +254,7 @@ public: Promise> versionstampPromise; + uint32_t getSize(); Future onError( Error const& e ); void flushTrLogsIfEnabled(); diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp index 2f862cb338..c41739d907 100644 --- a/fdbclient/ReadYourWrites.actor.cpp +++ b/fdbclient/ReadYourWrites.actor.cpp @@ -990,9 +990,10 @@ public: if(!ryw->options.readYourWritesDisabled) { ryw->watchMap[key].push_back(watch); val = readWithConflictRange( ryw, GetValueReq(key), false ); - } - else + } else { + ryw->approximateSize += 2 * key.expectedSize() + 1; val = ryw->tr.get(key); + } try { wait(ryw->resetPromise.getFuture() || success(val) || watch->onChangeTrigger.getFuture()); @@ -1045,7 +1046,7 @@ public: return Void(); } - + ryw->writeRangeToNativeTransaction(KeyRangeRef(StringRef(), allKeys.end)); auto conflictRanges = ryw->readConflicts.ranges(); @@ -1123,8 +1124,11 @@ public: } }; -ReadYourWritesTransaction::ReadYourWritesTransaction( Database const& cx ) : cache(&arena), writes(&arena), tr(cx), retries(0), creationTime(now()), commitStarted(false), options(tr), deferredError(cx->deferredError) { - std::copy(cx.getTransactionDefaults().begin(), cx.getTransactionDefaults().end(), std::back_inserter(persistentOptions)); +ReadYourWritesTransaction::ReadYourWritesTransaction(Database const& cx) + : cache(&arena), writes(&arena), tr(cx), retries(0), approximateSize(0), creationTime(now()), commitStarted(false), + options(tr), deferredError(cx->deferredError) { + std::copy(cx.getTransactionDefaults().begin(), cx.getTransactionDefaults().end(), + std::back_inserter(persistentOptions)); applyPersistentOptions(); } @@ -1367,6 +1371,7 @@ void ReadYourWritesTransaction::addReadConflictRange( KeyRangeRef const& keys ) } if(options.readYourWritesDisabled) { + approximateSize += r.expectedSize() + sizeof(KeyRangeRef); tr.addReadConflictRange(r); return; } @@ -1381,6 +1386,7 @@ void ReadYourWritesTransaction::updateConflictMap( KeyRef const& key, WriteMap:: //it.skip( key ); //ASSERT( it.beginKey() <= key && key < it.endKey() ); if( it.is_unmodified_range() || ( it.is_operation() && !it.is_independent() ) ) { + approximateSize += 2 * key.expectedSize() + 1 + sizeof(KeyRangeRef); readConflicts.insert( singleKeyRange( key, arena ), true ); } } @@ -1391,13 +1397,15 @@ void ReadYourWritesTransaction::updateConflictMap( KeyRangeRef const& keys, Writ for(; it.beginKey() < keys.end; ++it ) { if( it.is_unmodified_range() || ( it.is_operation() && !it.is_independent() ) ) { KeyRangeRef insert_range = KeyRangeRef( std::max( keys.begin, it.beginKey().toArenaOrRef( arena ) ), std::min( keys.end, it.endKey().toArenaOrRef( arena ) ) ); - if( !insert_range.empty() ) + if (!insert_range.empty()) { + approximateSize += keys.expectedSize() + sizeof(KeyRangeRef); readConflicts.insert( insert_range, true ); + } } } } -void ReadYourWritesTransaction::writeRangeToNativeTransaction( KeyRangeRef const& keys ) { +void ReadYourWritesTransaction::writeRangeToNativeTransaction(KeyRangeRef const& keys) { WriteMap::iterator it( &writes ); it.skip(keys.begin); @@ -1410,12 +1418,12 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction( KeyRangeRef const clearBegin = std::max(ExtStringRef(keys.begin), it.beginKey()); inClearRange = true; } else if( !it.is_cleared_range() && inClearRange ) { - tr.clear( KeyRangeRef( clearBegin.toArenaOrRef(arena), it.beginKey().toArenaOrRef(arena) ), false ); + tr.clear(KeyRangeRef(clearBegin.toArenaOrRef(arena), it.beginKey().toArenaOrRef(arena)), false); inClearRange = false; } } - if( inClearRange ) { + if (inClearRange) { tr.clear(KeyRangeRef(clearBegin.toArenaOrRef(arena), keys.end), false); } @@ -1429,7 +1437,7 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction( KeyRangeRef const conflictBegin = std::max(ExtStringRef(keys.begin), it.beginKey()); inConflictRange = true; } else if( !it.is_conflict_range() && inConflictRange ) { - tr.addWriteConflictRange( KeyRangeRef( conflictBegin.toArenaOrRef(arena), it.beginKey().toArenaOrRef(arena) ) ); + tr.addWriteConflictRange(KeyRangeRef(conflictBegin.toArenaOrRef(arena), it.beginKey().toArenaOrRef(arena))); inConflictRange = false; } @@ -1440,9 +1448,9 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction( KeyRangeRef const switch(op[i].type) { case MutationRef::SetValue: if (op[i].value.present()) { - tr.set( it.beginKey().assertRef(), op[i].value.get(), false ); + tr.set(it.beginKey().assertRef(), op[i].value.get(), false); } else { - tr.clear( it.beginKey().assertRef(), false ); + tr.clear(it.beginKey().assertRef(), false); } break; case MutationRef::AddValue: @@ -1469,7 +1477,7 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction( KeyRangeRef const } if( inConflictRange ) { - tr.addWriteConflictRange( KeyRangeRef( conflictBegin.toArenaOrRef(arena), keys.end ) ); + tr.addWriteConflictRange(KeyRangeRef(conflictBegin.toArenaOrRef(arena), keys.end)); } } @@ -1574,7 +1582,9 @@ void ReadYourWritesTransaction::atomicOp( const KeyRef& key, const ValueRef& ope throw client_invalid_operation(); } - if(options.readYourWritesDisabled) { + approximateSize += k.expectedSize() + v.expectedSize() + sizeof(MutationRef) + + (addWriteConflict ? sizeof(KeyRangeRef) + 2 * key.expectedSize() + 1 : 0); + if (options.readYourWritesDisabled) { return tr.atomicOp(k, v, (MutationRef::Type) operationType, addWriteConflict); } @@ -1604,7 +1614,9 @@ void ReadYourWritesTransaction::set( const KeyRef& key, const ValueRef& value ) if(key >= getMaxWriteKey()) throw key_outside_legal_range(); - if(options.readYourWritesDisabled ) { + approximateSize += key.expectedSize() + value.expectedSize() + sizeof(MutationRef) + + (addWriteConflict ? sizeof(KeyRangeRef) + 2 * key.expectedSize() + 1 : 0); + if (options.readYourWritesDisabled) { return tr.set(key, value, addWriteConflict); } @@ -1632,10 +1644,12 @@ void ReadYourWritesTransaction::clear( const KeyRangeRef& range ) { if(range.begin > maxKey || range.end > maxKey) throw key_outside_legal_range(); - if( options.readYourWritesDisabled ) { + approximateSize += range.expectedSize() + sizeof(MutationRef) + + (addWriteConflict ? sizeof(KeyRangeRef) + range.expectedSize() : 0); + if (options.readYourWritesDisabled) { return tr.clear(range, addWriteConflict); } - + //There aren't any keys in the database with size larger than KEY_SIZE_LIMIT, so if range contains large keys //we can translate it to an equivalent one with smaller keys KeyRef begin = range.begin; @@ -1676,6 +1690,8 @@ void ReadYourWritesTransaction::clear( const KeyRef& key ) { } KeyRangeRef r = singleKeyRange( key, arena ); + approximateSize += r.expectedSize() + sizeof(KeyRangeRef) + + (addWriteConflict ? sizeof(KeyRangeRef) + r.expectedSize() : 0); //SOMEDAY: add an optimized single key clear to write map writes.clear(r, addWriteConflict); @@ -1703,7 +1719,7 @@ Future ReadYourWritesTransaction::watch(const Key& key) { return RYWImpl::watch(this, key); } -void ReadYourWritesTransaction::addWriteConflictRange( KeyRangeRef const& keys ) { +void ReadYourWritesTransaction::addWriteConflictRange(KeyRangeRef const& keys) { if(checkUsedDuringCommit()) { throw used_during_commit(); } @@ -1730,6 +1746,7 @@ void ReadYourWritesTransaction::addWriteConflictRange( KeyRangeRef const& keys ) return; } + approximateSize += r.expectedSize() + sizeof(KeyRangeRef); if(options.readYourWritesDisabled) { tr.addWriteConflictRange(r); return; @@ -1854,6 +1871,7 @@ void ReadYourWritesTransaction::operator=(ReadYourWritesTransaction&& r) BOOST_N r.resetPromise = Promise(); deferredError = std::move( r.deferredError ); retries = r.retries; + approximateSize = r.approximateSize; timeoutActor = r.timeoutActor; creationTime = r.creationTime; commitStarted = r.commitStarted; @@ -1870,6 +1888,7 @@ ReadYourWritesTransaction::ReadYourWritesTransaction(ReadYourWritesTransaction&& arena( std::move(r.arena) ), reading( std::move(r.reading) ), retries( r.retries ), + approximateSize(r.approximateSize), creationTime( r.creationTime ), deferredError( std::move(r.deferredError) ), timeoutActor( std::move(r.timeoutActor) ), @@ -1921,6 +1940,7 @@ void ReadYourWritesTransaction::resetRyow() { readConflicts = CoalescedKeyRefRangeMap(); watchMap.clear(); reading = AndFuture(); + approximateSize = 0; commitStarted = false; deferredError = Error(); @@ -1941,6 +1961,7 @@ void ReadYourWritesTransaction::cancel() { void ReadYourWritesTransaction::reset() { retries = 0; + approximateSize = 0; creationTime = now(); timeoutActor.cancel(); persistentOptions.clear(); diff --git a/fdbclient/ReadYourWrites.h b/fdbclient/ReadYourWrites.h index f4b93eabcc..c26c7cbbe7 100644 --- a/fdbclient/ReadYourWrites.h +++ b/fdbclient/ReadYourWrites.h @@ -98,6 +98,7 @@ public: Future commit(); Version getCommittedVersion() { return tr.getCommittedVersion(); } + int64_t getApproximateSize() { return approximateSize; } Future> getVersionstamp(); void setOption( FDBTransactionOptions::Option option, Optional value = Optional() ); @@ -140,6 +141,7 @@ private: Promise resetPromise; AndFuture reading; int retries; + int64_t approximateSize; Future timeoutActor; double creationTime; bool commitStarted; @@ -149,7 +151,7 @@ private: void resetTimeout(); void updateConflictMap( KeyRef const& key, WriteMap::iterator& it ); // pre: it.segmentContains(key) void updateConflictMap( KeyRangeRef const& keys, WriteMap::iterator& it ); // pre: it.segmentContains(keys.begin), keys are already inside this->arena - void writeRangeToNativeTransaction( KeyRangeRef const& keys ); + void writeRangeToNativeTransaction(KeyRangeRef const& keys); void resetRyow(); // doesn't reset the encapsulated transaction, or creation time/retry state KeyRef getMaxReadKey(); diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index 35867be20b..f1fca48546 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -56,6 +56,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "roles":[ { "query_queue_max":0, + "local_rate":0, "input_bytes":{ "hz":0.0, "counter":0, @@ -250,7 +251,9 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "storage_server_min_free_space", "storage_server_min_free_space_ratio", "log_server_min_free_space", - "log_server_min_free_space_ratio" + "log_server_min_free_space_ratio", + "storage_server_durability_lag", + "storage_server_list_fetch_failed" ] }, "description":"The database is not being saturated by the workload." @@ -269,7 +272,9 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "storage_server_min_free_space", "storage_server_min_free_space_ratio", "log_server_min_free_space", - "log_server_min_free_space_ratio" + "log_server_min_free_space_ratio", + "storage_server_durability_lag", + "storage_server_list_fetch_failed" ] }, "description":"The database is not being saturated by the workload." @@ -358,7 +363,9 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "storage_servers_error", "status_incomplete", "layer_status_incomplete", - "database_availability_timeout" + "database_availability_timeout", + "consistencycheck_suspendkey_fetch_timeout", + "consistencycheck_disabled" ] }, "issues":[ diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 721654a462..634c4b68eb 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -159,7 +159,7 @@ struct WatchValueRequest { struct GetKeyValuesReply : public LoadBalancedReply { constexpr static FileIdentifier file_identifier = 1783066; Arena arena; - VectorRef data; + VectorRef data; Version version; // useful when latestVersion was requested bool more; @@ -177,14 +177,15 @@ struct GetKeyValuesRequest : TimedRequest { KeySelectorRef begin, end; Version version; // or latestVersion int limit, limitBytes; + bool isFetchKeys; Optional debugID; ReplyPromise reply; - GetKeyValuesRequest() {} + GetKeyValuesRequest() : isFetchKeys(false) {} // GetKeyValuesRequest(const KeySelectorRef& begin, const KeySelectorRef& end, Version version, int limit, int limitBytes, Optional debugID) : begin(begin), end(end), version(version), limit(limit), limitBytes(limitBytes) {} template void serialize( Ar& ar ) { - serializer(ar, begin, end, version, limit, limitBytes, debugID, reply, arena); + serializer(ar, begin, end, version, limit, limitBytes, isFetchKeys, debugID, reply, arena); } }; diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index c1146e3b25..a3a49f1965 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -442,6 +442,9 @@ const KeyRangeRef fdbClientInfoPrefixRange(LiteralStringRef("\xff\x02/fdbClientI const KeyRef fdbClientInfoTxnSampleRate = LiteralStringRef("\xff\x02/fdbClientInfo/client_txn_sample_rate/"); const KeyRef fdbClientInfoTxnSizeLimit = LiteralStringRef("\xff\x02/fdbClientInfo/client_txn_size_limit/"); +// ConsistencyCheck settings +const KeyRef fdbShouldConsistencyCheckBeSuspended = LiteralStringRef("\xff\x02/ConsistencyCheck/Suspend"); + // Request latency measurement key const KeyRef latencyBandConfigKey = LiteralStringRef("\xff\x02/latencyBandConfig"); diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index b29805208c..6a897fb222 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -216,6 +216,9 @@ extern const KeyRangeRef fdbClientInfoPrefixRange; extern const KeyRef fdbClientInfoTxnSampleRate; extern const KeyRef fdbClientInfoTxnSizeLimit; +// Consistency Check settings +extern const KeyRef fdbShouldConsistencyCheckBeSuspended; + // Request latency measurement key extern const KeyRef latencyBandConfigKey; diff --git a/fdbclient/ThreadSafeTransaction.actor.cpp b/fdbclient/ThreadSafeTransaction.actor.cpp index c26170c8c8..7772aae862 100644 --- a/fdbclient/ThreadSafeTransaction.actor.cpp +++ b/fdbclient/ThreadSafeTransaction.actor.cpp @@ -71,7 +71,7 @@ void ThreadSafeDatabase::setOption( FDBDatabaseOptions::Option option, Optional< } ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion) { - Reference connFile = Reference(new ClusterConnectionFile(ClusterConnectionFile::lookupClusterFileName(connFilename).first)); + ClusterConnectionFile *connFile = new ClusterConnectionFile(ClusterConnectionFile::lookupClusterFileName(connFilename).first); // Allocate memory for the Database from this thread (so the pointer is known for subsequent method calls) // but run its constructor on the main thread @@ -79,7 +79,7 @@ ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion) onMainThreadVoid([db, connFile, apiVersion](){ try { - Database::createDatabase(connFile, apiVersion, false, LocalityData(), db).extractPtr(); + Database::createDatabase(Reference(connFile), apiVersion, false, LocalityData(), db).extractPtr(); } catch(Error &e) { new (db) DatabaseContext(e); @@ -271,8 +271,12 @@ ThreadFuture< Void > ThreadSafeTransaction::commit() { Version ThreadSafeTransaction::getCommittedVersion() { // This should be thread safe when called legally, but it is fragile - Version v = tr->getCommittedVersion(); - return v; + return tr->getCommittedVersion(); +} + +ThreadFuture ThreadSafeTransaction::getApproximateSize() { + ReadYourWritesTransaction *tr = this->tr; + return onMainThread([tr]() -> Future { return tr->getApproximateSize(); }); } ThreadFuture> ThreadSafeTransaction::getVersionstamp() { diff --git a/fdbclient/ThreadSafeTransaction.h b/fdbclient/ThreadSafeTransaction.h index 21c8de199a..c5832cec45 100644 --- a/fdbclient/ThreadSafeTransaction.h +++ b/fdbclient/ThreadSafeTransaction.h @@ -55,54 +55,54 @@ public: explicit ThreadSafeTransaction(DatabaseContext* cx); ~ThreadSafeTransaction(); - void cancel(); - void setVersion( Version v ); - ThreadFuture getReadVersion(); + void cancel() override; + void setVersion( Version v ) override; + ThreadFuture getReadVersion() override; - ThreadFuture< Optional > get( const KeyRef& key, bool snapshot = false ); - ThreadFuture< Key > getKey( const KeySelectorRef& key, bool snapshot = false ); - ThreadFuture< Standalone > getRange( const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot = false, bool reverse = false ); - ThreadFuture< Standalone > getRange( const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot = false, bool reverse = false ); - ThreadFuture< Standalone > getRange( const KeyRangeRef& keys, int limit, bool snapshot = false, bool reverse = false ) { + ThreadFuture< Optional > get( const KeyRef& key, bool snapshot = false ) override; + ThreadFuture< Key > getKey( const KeySelectorRef& key, bool snapshot = false ) override; + ThreadFuture< Standalone > getRange( const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot = false, bool reverse = false ) override; + ThreadFuture< Standalone > getRange( const KeySelectorRef& begin, const KeySelectorRef& end, GetRangeLimits limits, bool snapshot = false, bool reverse = false ) override; + ThreadFuture< Standalone > getRange( const KeyRangeRef& keys, int limit, bool snapshot = false, bool reverse = false ) override { return getRange( firstGreaterOrEqual(keys.begin), firstGreaterOrEqual(keys.end), limit, snapshot, reverse ); } - ThreadFuture< Standalone > getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot = false, bool reverse = false ) { + ThreadFuture< Standalone > getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot = false, bool reverse = false ) override { return getRange( firstGreaterOrEqual(keys.begin), firstGreaterOrEqual(keys.end), limits, snapshot, reverse ); } + ThreadFuture>> getAddressesForKey(const KeyRef& key) override; + ThreadFuture> getVersionstamp() override; - ThreadFuture>> getAddressesForKey(const KeyRef& key); - - void addReadConflictRange( const KeyRangeRef& keys ); + void addReadConflictRange( const KeyRangeRef& keys ) override; void makeSelfConflicting(); - void atomicOp( const KeyRef& key, const ValueRef& value, uint32_t operationType ); - void set( const KeyRef& key, const ValueRef& value ); - void clear( const KeyRef& begin, const KeyRef& end); - void clear( const KeyRangeRef& range ); - void clear( const KeyRef& key ); + void atomicOp( const KeyRef& key, const ValueRef& value, uint32_t operationType ) override; + void set( const KeyRef& key, const ValueRef& value ) override; + void clear( const KeyRef& begin, const KeyRef& end) override; + void clear( const KeyRangeRef& range ) override; + void clear( const KeyRef& key ) override; - ThreadFuture< Void > watch( const KeyRef& key ); + ThreadFuture< Void > watch( const KeyRef& key ) override; - void addWriteConflictRange( const KeyRangeRef& keys ); + void addWriteConflictRange( const KeyRangeRef& keys ) override; - ThreadFuture commit(); - Version getCommittedVersion(); - ThreadFuture> getVersionstamp(); + ThreadFuture commit() override; + Version getCommittedVersion() override; + ThreadFuture getApproximateSize() override; - void setOption( FDBTransactionOptions::Option option, Optional value = Optional() ); + void setOption( FDBTransactionOptions::Option option, Optional value = Optional() ) override; ThreadFuture checkDeferredError(); - ThreadFuture onError( Error const& e ); + ThreadFuture onError( Error const& e ) override; // These are to permit use as state variables in actors: ThreadSafeTransaction() : tr(NULL) {} void operator=(ThreadSafeTransaction&& r) BOOST_NOEXCEPT; ThreadSafeTransaction(ThreadSafeTransaction&& r) BOOST_NOEXCEPT; - void reset(); + void reset() override; - void addref() { ThreadSafeReferenceCounted::addref(); } - void delref() { ThreadSafeReferenceCounted::delref(); } + void addref() override { ThreadSafeReferenceCounted::addref(); } + void delref() override { ThreadSafeReferenceCounted::delref(); } private: ReadYourWritesTransaction *tr; diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options index 775e443137..6044403dc6 100644 --- a/fdbclient/vexillographer/fdb.options +++ b/fdbclient/vexillographer/fdb.options @@ -146,6 +146,15 @@ description is not currently required but encouraged.