From cd19a9b61af72acc69e93f3e39af3f4bf21961ff Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Mon, 1 Mar 2021 14:50:27 -0800 Subject: [PATCH 01/26] Fix bug where regions reported extra satellites --- fdbcli/fdbcli.actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index f34534f7b3..896c3cfd12 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -1025,9 +1025,9 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, outputString += "\n Regions: "; regions = statusObjConfig["regions"].get_array(); bool isPrimary = false; - std::vector regionSatelliteDCs; std::string regionDC; for (StatusObjectReader region : regions) { + std::vector regionSatelliteDCs; for (StatusObjectReader dc : region["datacenters"].get_array()) { if (!dc.has("satellite")) { regionDC = dc["id"].get_str(); From 1025a0da37f3517da99e540fea43817a8fd0ad06 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 3 Mar 2021 21:45:33 -0800 Subject: [PATCH 02/26] File-based backup now uses async rename to prevent blocking the network thread in the event of slow filesystem metadata operations. Backup now opens its write-only files without a block cache so that writes to disk will be larger (1MB by default instead of 4k). --- fdbclient/BackupContainer.actor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index a886fd2444..6695dad269 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -1314,7 +1314,7 @@ public: wait(f->m_file->sync()); std::string name = f->m_file->getFilename(); f->m_file.clear(); - renameFile(name, f->m_finalFullPath); + wait(IAsyncFileSystem::filesystem()->renameFile(name, f->m_finalFullPath)); return Void(); } @@ -1337,7 +1337,7 @@ public: }; Future> writeFile(std::string path) { - int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE; + int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE; std::string fullPath = joinPath(m_path, path); platform::createDirectory(parentDirectory(fullPath)); std::string temp = fullPath + "." + deterministicRandom()->randomUniqueID().toString() + ".temp"; From b2f313774f5b5780d8c15060d24ad14c15b81e30 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 3 Mar 2021 22:24:00 -0800 Subject: [PATCH 03/26] Added release notes. --- .../sphinx/source/release-notes/release-notes-620.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/documentation/sphinx/source/release-notes/release-notes-620.rst b/documentation/sphinx/source/release-notes/release-notes-620.rst index 35094d3497..50098b9ec5 100644 --- a/documentation/sphinx/source/release-notes/release-notes-620.rst +++ b/documentation/sphinx/source/release-notes/release-notes-620.rst @@ -4,6 +4,11 @@ Release Notes ############# +6.2.33 +====== +* Fix backup agent stall when writing to local filesystem with slow metadata operations. `(PR #4428) `_ +* Backup agent writes to local filesystems without 4k caching layer, resulting in larger write operations. `(PR #4428) `_ + 6.2.32 ====== * Fix an issue where symbolic links in cmake-built RPMs are broken if you unpack the RPM to a custom directory. `(PR #4380) `_ From cd5341ceff4a86af410cc47abc8145f8ecc149c3 Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Wed, 3 Mar 2021 22:26:43 -0800 Subject: [PATCH 04/26] Make release note more clear. --- documentation/sphinx/source/release-notes/release-notes-620.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/sphinx/source/release-notes/release-notes-620.rst b/documentation/sphinx/source/release-notes/release-notes-620.rst index 50098b9ec5..f8df76b7f8 100644 --- a/documentation/sphinx/source/release-notes/release-notes-620.rst +++ b/documentation/sphinx/source/release-notes/release-notes-620.rst @@ -7,7 +7,7 @@ Release Notes 6.2.33 ====== * Fix backup agent stall when writing to local filesystem with slow metadata operations. `(PR #4428) `_ -* Backup agent writes to local filesystems without 4k caching layer, resulting in larger write operations. `(PR #4428) `_ +* Backup agent no longer uses 4k block caching layer on local output files so that write operations are larger. `(PR #4428) `_ 6.2.32 ====== From 90a324fda2c526aaaef2b1744b9ea79b81ede58d Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Thu, 4 Mar 2021 10:31:03 -0800 Subject: [PATCH 05/26] Updated release-6.3 release note proposal --- .../release-notes/release-notes-630.rst | 26 ++++++------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/documentation/sphinx/source/release-notes/release-notes-630.rst b/documentation/sphinx/source/release-notes/release-notes-630.rst index 7ca1a2e721..5f249e2329 100644 --- a/documentation/sphinx/source/release-notes/release-notes-630.rst +++ b/documentation/sphinx/source/release-notes/release-notes-630.rst @@ -6,28 +6,18 @@ Release Notes 6.3.11 ====== -* Added a hint field in the trace event when all replicas of some data are lost. `(PR #4209) `_ -* Rewrote SQLite injected fault handling. `(PR #4212) `_ -* Add a SevWarnAlways trace line to help debug a rare failure. `(PR #4214) `_ -* Use VFSAsyncFile::checkInjectedError to detect injected faults. `(PR #4253) `_ -* Build on Windows using VS 2019 + LLVM/Clang. `(PR #4258) `_ -* RateControl support in AFCCached to enable write op throttling. The feature is disabled by default. `(PR #4229) `_ -* Add knobs for prefix bloom filters and larger block cache for RocksDB. `(PR #4201) `_ -* Adding debug tools to FDB runtime image. `(PR #4247) `_ -* Fix bug in simulated coordinator selection. `(PR #4285) `_ -* Add option to prevent synchronous file deletes on reads for RocksDB. `(PR #4270) `_ -* Report warning when TLS verification fails. `(PR #4299) `_ -* Support multiple worker threads for each version of client that is loaded so that each cluster will be serviced by a client thread. `(PR #4269) `_ -* Reboot simulated process on io_timeout error. `(PR #4345) `_ -* Fix Snapshot backup test failure. `(PR #4372) `_ + +* Support multiple worker threads for each client version that is loaded. `(PR #4269) `_ * fdbcli: Output errors and warnings to stderr. `(PR #4332) `_ -* Do not generate machine id in locality field if it is set by the user. `(PR #4022) `_ -* Make the RocksDB init method idempotent. `(PR #4400) `_ -* Fix bugs turned up by _GLIBCXX_DEBUG. `(PR #4301) `_ -* Add New Unit and Integration Tests, and associated infrastructure. `(PR #4366) `_ +* Do not rely on shared memory to generate a machine id if it is set explicitly. `(Issue #4022) `_ +* Added ``workload.transactions.rejected_for_queued_too_long`` to status to report the number of transaction commits that failed because they were queued too long and could no longer be checked for conflicts. `(PR #4353) `_ +* Add knobs for prefix bloom filters and larger block cache for RocksDB. `(PR #4201) `_ +* Add option to prevent synchronous file deletes on reads for RocksDB. `(PR #4270) `_ +* Build on Windows using VS 2019 + LLVM/Clang. `(PR #4258) `_ 6.3.10 ====== + * Make fault tolerance metric calculation in HA clusters consistent with 6.2 branch. `(PR #4175) `_ * Bug fix, stack overflow in redwood storage engine. `(PR #4161) `_ * Bug fix, getting certain special keys fail. `(PR #4128) `_ From 75c2e1e2ea78e386ea88984ab1d3309a341df872 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Fri, 5 Mar 2021 10:07:09 -0800 Subject: [PATCH 06/26] fix: if the time associated with newest fetchkeys is equal to now(), longestTime will throw an unknown error --- fdbserver/storageserver.actor.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index b75d4a768e..dcdb0ed617 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -338,15 +338,18 @@ public: const double currentTime = now(); double longest = 0; - UID UIDofLongest; + Optional UIDofLongest; for (const auto& kv: startTimeMap) { const double currentRunningTime = currentTime - kv.second; - if (longest < currentRunningTime) { + if (longest <= currentRunningTime) { longest = currentRunningTime; UIDofLongest = kv.first; } } - return {longest, keyRangeMap.at(UIDofLongest)}; + if(UIDofLongest.present()) { + return {longest, keyRangeMap[UIDofLongest.get()]}; + } + return {-1, emptyKeyRange}; } int numRunning() const { return startTimeMap.size(); } From 5988ac5487d263270b1f13a8d25713fd0400a01e Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Fri, 5 Mar 2021 10:12:24 -0800 Subject: [PATCH 07/26] updated documentation for 6.2.33 --- documentation/sphinx/source/downloads.rst | 24 +++++++++---------- .../release-notes/release-notes-620.rst | 4 ++++ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/documentation/sphinx/source/downloads.rst b/documentation/sphinx/source/downloads.rst index 5729a4ba1c..8ca21b8007 100644 --- a/documentation/sphinx/source/downloads.rst +++ b/documentation/sphinx/source/downloads.rst @@ -10,38 +10,38 @@ macOS The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server. -* `FoundationDB-6.2.30.pkg `_ +* `FoundationDB-6.2.33.pkg `_ Ubuntu ------ The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x. -* `foundationdb-clients-6.2.30-1_amd64.deb `_ -* `foundationdb-server-6.2.30-1_amd64.deb `_ (depends on the clients package) +* `foundationdb-clients-6.2.33-1_amd64.deb `_ +* `foundationdb-server-6.2.33-1_amd64.deb `_ (depends on the clients package) RHEL/CentOS EL6 --------------- The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x. -* `foundationdb-clients-6.2.30-1.el6.x86_64.rpm `_ -* `foundationdb-server-6.2.30-1.el6.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.2.33-1.el6.x86_64.rpm `_ +* `foundationdb-server-6.2.33-1.el6.x86_64.rpm `_ (depends on the clients package) RHEL/CentOS EL7 --------------- The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x. -* `foundationdb-clients-6.2.30-1.el7.x86_64.rpm `_ -* `foundationdb-server-6.2.30-1.el7.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.2.33-1.el7.x86_64.rpm `_ +* `foundationdb-server-6.2.33-1.el7.x86_64.rpm `_ (depends on the clients package) Windows ------- The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server. -* `foundationdb-6.2.30-x64.msi `_ +* `foundationdb-6.2.33-x64.msi `_ API Language Bindings ===================== @@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part If you need to use the FoundationDB Python API from other Python installations or paths, download the Python package: -* `foundationdb-6.2.30.tar.gz `_ +* `foundationdb-6.2.33.tar.gz `_ Ruby 1.9.3/2.0.0+ ----------------- -* `fdb-6.2.30.gem `_ +* `fdb-6.2.33.gem `_ Java 8+ ------- -* `fdb-java-6.2.30.jar `_ -* `fdb-java-6.2.30-javadoc.jar `_ +* `fdb-java-6.2.33.jar `_ +* `fdb-java-6.2.33-javadoc.jar `_ Go 1.11+ -------- diff --git a/documentation/sphinx/source/release-notes/release-notes-620.rst b/documentation/sphinx/source/release-notes/release-notes-620.rst index 35094d3497..1eb4dd8d91 100644 --- a/documentation/sphinx/source/release-notes/release-notes-620.rst +++ b/documentation/sphinx/source/release-notes/release-notes-620.rst @@ -4,6 +4,10 @@ Release Notes ############# +6.2.33 +====== +* Fixed a issue where storage servers could shutdown with ``unknown_error``. `(PR #4380) `_ + 6.2.32 ====== * Fix an issue where symbolic links in cmake-built RPMs are broken if you unpack the RPM to a custom directory. `(PR #4380) `_ From 2f625fd767092e5ca633be3b826d8b6bdce4dd27 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Fri, 5 Mar 2021 10:18:08 -0800 Subject: [PATCH 08/26] added more protection --- fdbserver/storageserver.actor.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index dcdb0ed617..eb2535daf7 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -346,7 +346,10 @@ public: UIDofLongest = kv.first; } } - if(UIDofLongest.present()) { + if(BUGGIFY) { + UIDofLongest = Optional(); + } + if(UIDofLongest.present() && keyRangeMap.count(UIDofLongest.get())) { return {longest, keyRangeMap[UIDofLongest.get()]}; } return {-1, emptyKeyRange}; From 980755b9cf2a0d520bb858b16cd8d95069659197 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Fri, 5 Mar 2021 10:22:16 -0800 Subject: [PATCH 09/26] fixed compile error --- fdbserver/storageserver.actor.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index eb2535daf7..dd7f510710 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -338,7 +338,7 @@ public: const double currentTime = now(); double longest = 0; - Optional UIDofLongest; + UID UIDofLongest; for (const auto& kv: startTimeMap) { const double currentRunningTime = currentTime - kv.second; if (longest <= currentRunningTime) { @@ -347,10 +347,10 @@ public: } } if(BUGGIFY) { - UIDofLongest = Optional(); + UIDofLongest = deterministicRandom()->randomUniqueID(); } - if(UIDofLongest.present() && keyRangeMap.count(UIDofLongest.get())) { - return {longest, keyRangeMap[UIDofLongest.get()]}; + if(keyRangeMap.count(UIDofLongest)) { + return {longest, keyRangeMap[UIDofLongest]}; } return {-1, emptyKeyRange}; } From 1d8977f3c24566ef6ee80b1dd47d8a3cf755d8e6 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Fri, 5 Mar 2021 10:25:44 -0800 Subject: [PATCH 10/26] another attempt to fix compile error --- fdbserver/storageserver.actor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index dd7f510710..059ce81fe5 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -350,7 +350,8 @@ public: UIDofLongest = deterministicRandom()->randomUniqueID(); } if(keyRangeMap.count(UIDofLongest)) { - return {longest, keyRangeMap[UIDofLongest]}; + const KeyRange& range = keyRangeMap[UIDofLongest]; + return {longest, range}; } return {-1, emptyKeyRange}; } From 36c904b6f77b0ef328b52b498eeed5b8740e4d7d Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Fri, 5 Mar 2021 10:32:24 -0800 Subject: [PATCH 11/26] go back to using .at() because we are in a const function --- fdbserver/storageserver.actor.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 059ce81fe5..534dca4fec 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -350,8 +350,7 @@ public: UIDofLongest = deterministicRandom()->randomUniqueID(); } if(keyRangeMap.count(UIDofLongest)) { - const KeyRange& range = keyRangeMap[UIDofLongest]; - return {longest, range}; + return {longest, keyRangeMap.at(UIDofLongest)}; } return {-1, emptyKeyRange}; } From 8d88afec8a186db1f5ff5e0f267953f0c3f16c02 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Fri, 5 Mar 2021 10:35:17 -0800 Subject: [PATCH 12/26] change to using find --- fdbserver/storageserver.actor.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 534dca4fec..ff886bd75c 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -349,8 +349,9 @@ public: if(BUGGIFY) { UIDofLongest = deterministicRandom()->randomUniqueID(); } - if(keyRangeMap.count(UIDofLongest)) { - return {longest, keyRangeMap.at(UIDofLongest)}; + auto it = keyRangeMap.find(UIDofLongest); + if(it != keyRangeMap.end()) { + return {longest, *it}; } return {-1, emptyKeyRange}; } From 940ea42c6e1d12b65af58db550b3eb663de2b488 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Fri, 5 Mar 2021 10:37:49 -0800 Subject: [PATCH 13/26] wrong use of the iterator --- fdbserver/storageserver.actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index ff886bd75c..32d8f91ac0 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -351,7 +351,7 @@ public: } auto it = keyRangeMap.find(UIDofLongest); if(it != keyRangeMap.end()) { - return {longest, *it}; + return {longest, it->second}; } return {-1, emptyKeyRange}; } From 0c214d71da83f8c496774521cc7b68c31f19959c Mon Sep 17 00:00:00 2001 From: FDB Formatster Date: Fri, 5 Mar 2021 16:31:35 -0600 Subject: [PATCH 14/26] apply clang-format to *.c, *.cpp, *.h, *.hpp files --- .clang-format | 8 +- FDBLibTLS/FDBLibTLSPlugin.cpp | 7 +- FDBLibTLS/FDBLibTLSPlugin.h | 2 +- FDBLibTLS/FDBLibTLSPolicy.cpp | 71 +- FDBLibTLS/FDBLibTLSPolicy.h | 10 +- FDBLibTLS/FDBLibTLSSession.cpp | 106 +- FDBLibTLS/FDBLibTLSSession.h | 15 +- FDBLibTLS/FDBLibTLSVerify.cpp | 43 +- FDBLibTLS/FDBLibTLSVerify.h | 20 +- FDBLibTLS/plugin-test.cpp | 1138 +- FDBLibTLS/verify-test.cpp | 140 +- bindings/c/ThreadCleanup.cpp | 8 +- bindings/c/fdb_c.cpp | 748 +- bindings/c/foundationdb/ClientWorkload.h | 3 +- bindings/c/test/mako/mako.c | 3101 ++-- bindings/c/test/mako/mako.h | 76 +- bindings/c/test/mako/utils.c | 86 +- bindings/c/test/mako/utils.h | 17 +- bindings/c/test/mako/zipf.c | 91 +- bindings/c/test/performance_test.c | 336 +- bindings/c/test/ryw_benchmark.c | 156 +- bindings/c/test/test.h | 109 +- bindings/c/test/txn_size_test.c | 16 +- bindings/c/test/workloads/SimpleWorkload.cpp | 22 +- bindings/c/test/workloads/workloads.cpp | 20 +- bindings/c/test/workloads/workloads.h | 10 +- bindings/flow/DirectoryLayer.actor.cpp | 901 +- bindings/flow/DirectoryLayer.h | 154 +- bindings/flow/DirectoryPartition.h | 50 +- bindings/flow/DirectorySubspace.cpp | 165 +- bindings/flow/DirectorySubspace.h | 64 +- bindings/flow/FDBLoanerTypes.h | 576 +- .../flow/HighContentionAllocator.actor.cpp | 152 +- bindings/flow/HighContentionAllocator.h | 21 +- bindings/flow/IDirectory.h | 54 +- bindings/flow/Node.actor.cpp | 73 +- bindings/flow/Subspace.cpp | 117 +- bindings/flow/Subspace.h | 98 +- bindings/flow/Tuple.cpp | 1147 +- bindings/flow/Tuple.h | 143 +- bindings/flow/fdb_flow.actor.cpp | 670 +- bindings/flow/fdb_flow.h | 204 +- .../flow/tester/DirectoryTester.actor.cpp | 201 +- bindings/flow/tester/Tester.actor.cpp | 484 +- bindings/flow/tester/Tester.actor.h | 98 +- bindings/java/JavaWorkload.cpp | 36 +- bindings/java/fdbJNI.cpp | 1082 +- fdbbackup/backup.actor.cpp | 2941 ++-- fdbcli/FlowLineNoise.actor.cpp | 236 +- fdbcli/FlowLineNoise.h | 52 +- fdbcli/fdbcli.actor.cpp | 2033 +-- fdbcli/linenoise/linenoise.c | 1509 +- fdbcli/linenoise/linenoise.h | 18 +- fdbclient/AsyncFileBlobStore.actor.cpp | 30 +- fdbclient/AsyncFileBlobStore.actor.h | 135 +- fdbclient/Atomic.h | 152 +- fdbclient/AutoPublicAddress.cpp | 3 +- fdbclient/BackupAgent.actor.h | 462 +- fdbclient/BackupAgentBase.actor.cpp | 502 +- fdbclient/BackupContainer.actor.cpp | 1035 +- fdbclient/BackupContainer.h | 64 +- fdbclient/BlobStore.actor.cpp | 670 +- fdbclient/BlobStore.h | 171 +- fdbclient/ClientLogEvents.h | 539 +- fdbclient/ClientWorkerInterface.h | 28 +- fdbclient/ClusterInterface.h | 103 +- fdbclient/CommitTransaction.h | 64 +- fdbclient/CoordinationInterface.h | 61 +- fdbclient/DatabaseBackupAgent.actor.cpp | 3916 +++-- fdbclient/DatabaseConfiguration.cpp | 408 +- fdbclient/DatabaseConfiguration.h | 145 +- fdbclient/DatabaseContext.h | 91 +- fdbclient/EventTypes.actor.h | 13 +- fdbclient/FDBOptions.h | 43 +- fdbclient/FDBTypes.h | 545 +- fdbclient/FailureMonitorClient.actor.cpp | 123 +- fdbclient/FailureMonitorClient.h | 3 +- fdbclient/FileBackupAgent.actor.cpp | 7007 +++++---- fdbclient/HTTP.actor.cpp | 875 +- fdbclient/HTTP.h | 56 +- fdbclient/IClientApi.h | 41 +- fdbclient/JSONDoc.h | 140 +- fdbclient/JsonBuilder.cpp | 65 +- fdbclient/JsonBuilder.h | 225 +- fdbclient/KeyBackedTypes.h | 239 +- fdbclient/KeyRangeMap.actor.cpp | 191 +- fdbclient/KeyRangeMap.h | 306 +- fdbclient/Knobs.cpp | 351 +- fdbclient/Knobs.h | 13 +- fdbclient/ManagementAPI.actor.cpp | 1347 +- fdbclient/ManagementAPI.actor.h | 116 +- fdbclient/MasterProxyInterface.h | 148 +- fdbclient/MetricLogger.actor.cpp | 271 +- fdbclient/MetricLogger.h | 2 +- fdbclient/MonitorLeader.actor.cpp | 416 +- fdbclient/MonitorLeader.h | 39 +- fdbclient/MultiVersionAssignmentVars.h | 157 +- fdbclient/MultiVersionTransaction.actor.cpp | 953 +- fdbclient/MultiVersionTransaction.h | 243 +- fdbclient/MutationList.h | 110 +- fdbclient/NativeAPI.actor.cpp | 3068 ++-- fdbclient/NativeAPI.actor.h | 162 +- fdbclient/Notified.h | 62 +- fdbclient/RYWIterator.cpp | 395 +- fdbclient/RYWIterator.h | 31 +- fdbclient/ReadYourWrites.actor.cpp | 1634 +- fdbclient/ReadYourWrites.h | 105 +- fdbclient/RunTransaction.actor.h | 47 +- fdbclient/Schemas.cpp | 3 +- fdbclient/SnapshotCache.h | 266 +- fdbclient/Status.h | 33 +- fdbclient/StatusClient.actor.cpp | 480 +- fdbclient/StorageServerInterface.h | 188 +- fdbclient/Subspace.cpp | 9 +- fdbclient/Subspace.h | 4 +- fdbclient/SystemData.cpp | 386 +- fdbclient/SystemData.h | 119 +- fdbclient/TaskBucket.actor.cpp | 620 +- fdbclient/TaskBucket.h | 233 +- fdbclient/ThreadSafeTransaction.actor.cpp | 298 +- fdbclient/ThreadSafeTransaction.h | 73 +- fdbclient/Tuple.cpp | 105 +- fdbclient/Tuple.h | 2 +- fdbclient/VersionedMap.actor.h | 23 +- fdbclient/VersionedMap.h | 1034 +- fdbclient/WriteMap.h | 678 +- .../json_spirit/json_spirit_error_position.h | 56 +- .../json_spirit/json_spirit_reader_template.h | 1122 +- fdbclient/json_spirit/json_spirit_value.h | 955 +- .../json_spirit/json_spirit_writer_options.h | 36 +- .../json_spirit/json_spirit_writer_template.h | 590 +- fdbclient/libb64/cdecode.c | 51 +- fdbclient/libb64/cdecode.h | 8 +- fdbclient/libb64/cencode.c | 44 +- fdbclient/libb64/cencode.h | 8 +- fdbclient/libb64/decode.h | 84 +- fdbclient/libb64/encode.h | 106 +- fdbclient/md5/md5.c | 62 +- fdbclient/md5/md5.h | 8 +- fdbclient/rapidjson/allocators.h | 332 +- fdbclient/rapidjson/document.h | 4133 ++--- fdbclient/rapidjson/encodedstream.h | 432 +- fdbclient/rapidjson/encodings.h | 1016 +- fdbclient/rapidjson/error/en.h | 79 +- fdbclient/rapidjson/error/error.h | 89 +- fdbclient/rapidjson/filereadstream.h | 116 +- fdbclient/rapidjson/filewritestream.h | 122 +- fdbclient/rapidjson/fwd.h | 68 +- fdbclient/rapidjson/internal/biginteger.h | 449 +- fdbclient/rapidjson/internal/diyfp.h | 346 +- fdbclient/rapidjson/internal/dtoa.h | 388 +- fdbclient/rapidjson/internal/ieee754.h | 86 +- fdbclient/rapidjson/internal/itoa.h | 514 +- fdbclient/rapidjson/internal/meta.h | 201 +- fdbclient/rapidjson/internal/pow10.h | 57 +- fdbclient/rapidjson/internal/regex.h | 1052 +- fdbclient/rapidjson/internal/stack.h | 312 +- fdbclient/rapidjson/internal/strfunc.h | 47 +- fdbclient/rapidjson/internal/strtod.h | 384 +- fdbclient/rapidjson/internal/swap.h | 8 +- fdbclient/rapidjson/istreamwrapper.h | 103 +- fdbclient/rapidjson/memorybuffer.h | 43 +- fdbclient/rapidjson/memorystream.h | 57 +- fdbclient/rapidjson/msinttypes/inttypes.h | 403 +- fdbclient/rapidjson/msinttypes/stdint.h | 260 +- fdbclient/rapidjson/ostreamwrapper.h | 55 +- fdbclient/rapidjson/pointer.h | 2043 +-- fdbclient/rapidjson/prettywriter.h | 376 +- fdbclient/rapidjson/rapidjson.h | 185 +- fdbclient/rapidjson/reader.h | 2923 ++-- fdbclient/rapidjson/schema.h | 3086 ++-- fdbclient/rapidjson/stream.h | 121 +- fdbclient/rapidjson/stringbuffer.h | 94 +- fdbclient/rapidjson/writer.h | 943 +- fdbclient/rapidxml/rapidxml.hpp | 5449 ++++--- fdbclient/rapidxml/rapidxml_iterators.hpp | 225 +- fdbclient/rapidxml/rapidxml_print.hpp | 743 +- fdbclient/rapidxml/rapidxml_utils.hpp | 164 +- fdbclient/sha1/SHA1.cpp | 390 +- fdbclient/sha1/SHA1.h | 37 +- fdbmonitor/ConvertUTF.h | 109 +- fdbmonitor/SimpleIni.h | 4721 +++--- fdbmonitor/fdbmonitor.cpp | 869 +- fdbrpc/ActorFuzz.actor.cpp | 960 +- fdbrpc/ActorFuzz.h | 14 +- fdbrpc/AsyncFileCached.actor.cpp | 162 +- fdbrpc/AsyncFileCached.actor.h | 268 +- fdbrpc/AsyncFileEIO.actor.h | 408 +- fdbrpc/AsyncFileKAIO.actor.h | 458 +- fdbrpc/AsyncFileNonDurable.actor.cpp | 11 +- fdbrpc/AsyncFileNonDurable.actor.h | 591 +- fdbrpc/AsyncFileReadAhead.actor.h | 95 +- fdbrpc/AsyncFileWinASIO.actor.h | 124 +- fdbrpc/AsyncFileWriteChecker.h | 69 +- fdbrpc/ContinuousSample.h | 44 +- fdbrpc/FailureMonitor.actor.cpp | 124 +- fdbrpc/FailureMonitor.h | 58 +- fdbrpc/FlowTests.actor.cpp | 346 +- fdbrpc/FlowTransport.actor.cpp | 773 +- fdbrpc/FlowTransport.h | 61 +- fdbrpc/IAsyncFile.actor.cpp | 76 +- fdbrpc/IAsyncFile.h | 75 +- fdbrpc/IRateControl.h | 20 +- fdbrpc/LoadBalance.actor.h | 300 +- fdbrpc/LoadPlugin.h | 10 +- fdbrpc/Locality.cpp | 33 +- fdbrpc/Locality.h | 300 +- fdbrpc/MultiInterface.h | 114 +- fdbrpc/Net2FileSystem.cpp | 47 +- fdbrpc/Net2FileSystem.h | 15 +- fdbrpc/PerfMetric.h | 26 +- fdbrpc/Platform.cpp | 29 +- fdbrpc/QueueModel.cpp | 53 +- fdbrpc/QueueModel.h | 44 +- fdbrpc/RangeMap.h | 180 +- fdbrpc/Replication.h | 436 +- fdbrpc/ReplicationPolicy.cpp | 312 +- fdbrpc/ReplicationPolicy.h | 24 +- fdbrpc/ReplicationTypes.cpp | 2 +- fdbrpc/ReplicationTypes.h | 132 +- fdbrpc/ReplicationUtils.cpp | 748 +- fdbrpc/ReplicationUtils.h | 65 +- fdbrpc/Smoother.h | 42 +- fdbrpc/Stats.actor.cpp | 23 +- fdbrpc/Stats.h | 105 +- fdbrpc/TraceFileIO.cpp | 140 +- fdbrpc/TraceFileIO.h | 11 +- fdbrpc/batcher.actor.h | 64 +- fdbrpc/crc32c.cpp | 426 +- fdbrpc/crc32c.h | 6 +- fdbrpc/dsltest.actor.cpp | 741 +- fdbrpc/fdbrpc.h | 194 +- fdbrpc/generated-constants.cpp | 729 +- fdbrpc/genericactors.actor.cpp | 8 +- fdbrpc/genericactors.actor.h | 138 +- fdbrpc/libcoroutine/386-ucontext.h | 72 +- fdbrpc/libcoroutine/Common.c | 214 +- fdbrpc/libcoroutine/Common.h | 107 +- fdbrpc/libcoroutine/Coro.c | 493 +- fdbrpc/libcoroutine/Coro.h | 74 +- fdbrpc/libcoroutine/amd64-ucontext.h | 99 +- fdbrpc/libcoroutine/context.c | 53 +- fdbrpc/libcoroutine/power-ucontext.h | 43 +- fdbrpc/libcoroutine/taskimpl.h | 124 +- fdbrpc/libeio/ecb.h | 533 +- fdbrpc/libeio/eio.c | 3680 +++-- fdbrpc/libeio/eio.h | 453 +- fdbrpc/libeio/xthread.h | 149 +- fdbrpc/linux_kaio.h | 21 +- fdbrpc/networksender.actor.h | 12 +- fdbrpc/sim2.actor.cpp | 1762 ++- fdbrpc/sim_validation.cpp | 74 +- fdbrpc/simulator.h | 255 +- fdbrpc/zlib/adler32.c | 280 +- fdbrpc/zlib/crc32.c | 527 +- fdbrpc/zlib/crc32.h | 693 +- fdbrpc/zlib/deflate.c | 2769 ++-- fdbrpc/zlib/deflate.h | 413 +- fdbrpc/zlib/gzclose.c | 15 +- fdbrpc/zlib/gzguts.h | 220 +- fdbrpc/zlib/gzlib.c | 845 +- fdbrpc/zlib/gzread.c | 891 +- fdbrpc/zlib/gzwrite.c | 947 +- fdbrpc/zlib/infback.c | 983 +- fdbrpc/zlib/inffast.c | 485 +- fdbrpc/zlib/inffixed.h | 181 +- fdbrpc/zlib/inflate.c | 2205 +-- fdbrpc/zlib/inflate.h | 146 +- fdbrpc/zlib/inftrees.c | 496 +- fdbrpc/zlib/inftrees.h | 23 +- fdbrpc/zlib/trees.c | 1545 +- fdbrpc/zlib/trees.h | 211 +- fdbrpc/zlib/zconf.h | 638 +- fdbrpc/zlib/zlib.h | 400 +- fdbrpc/zlib/zutil.c | 392 +- fdbrpc/zlib/zutil.h | 272 +- fdbserver/ApplyMetadataMutation.cpp | 413 +- fdbserver/ApplyMetadataMutation.h | 35 +- fdbserver/ClusterController.actor.cpp | 2309 +-- fdbserver/ClusterRecruitmentInterface.h | 152 +- fdbserver/CompactMap.cpp | 422 +- fdbserver/ConflictSet.h | 22 +- fdbserver/CoordinatedState.actor.cpp | 332 +- fdbserver/CoordinatedState.h | 16 +- fdbserver/Coordination.actor.cpp | 410 +- fdbserver/CoordinationInterface.h | 62 +- fdbserver/CoroFlow.actor.cpp | 154 +- fdbserver/CoroFlow.h | 18 +- fdbserver/DBCoreState.h | 96 +- fdbserver/DataDistribution.actor.cpp | 2050 +-- fdbserver/DataDistribution.actor.h | 118 +- fdbserver/DataDistributionQueue.actor.cpp | 1308 +- fdbserver/DataDistributionTracker.actor.cpp | 660 +- fdbserver/DataDistributorInterface.h | 18 +- fdbserver/DeltaTree.h | 290 +- fdbserver/DiskQueue.actor.cpp | 1052 +- fdbserver/FDBExecHelper.actor.cpp | 41 +- fdbserver/FDBExecHelper.actor.h | 13 +- fdbserver/IDiskQueue.h | 79 +- fdbserver/IKeyValueStore.h | 94 +- fdbserver/IPager.h | 40 +- fdbserver/IVersionedStore.h | 31 +- .../KeyValueStoreCompressTestData.actor.cpp | 96 +- fdbserver/KeyValueStoreMemory.actor.cpp | 550 +- fdbserver/KeyValueStoreSQLite.actor.cpp | 1311 +- fdbserver/Knobs.cpp | 9 +- fdbserver/Knobs.h | 58 +- fdbserver/LatencyBandConfig.cpp | 44 +- fdbserver/LeaderElection.actor.cpp | 184 +- fdbserver/LeaderElection.h | 40 +- fdbserver/LogProtocolMessage.h | 11 +- fdbserver/LogRouter.actor.cpp | 371 +- fdbserver/LogSystem.h | 555 +- fdbserver/LogSystemConfig.h | 221 +- fdbserver/LogSystemDiskQueueAdapter.actor.cpp | 146 +- fdbserver/LogSystemDiskQueueAdapter.h | 72 +- fdbserver/LogSystemPeekCursor.actor.cpp | 684 +- fdbserver/MasterInterface.h | 57 +- fdbserver/MasterProxyServer.actor.cpp | 1261 +- fdbserver/MoveKeys.actor.cpp | 807 +- fdbserver/MoveKeys.actor.h | 26 +- fdbserver/NetworkTest.h | 14 +- fdbserver/OldTLogServer_4_6.actor.cpp | 2757 ++-- fdbserver/OldTLogServer_6_0.actor.cpp | 1936 ++- fdbserver/Orderer.actor.h | 48 +- fdbserver/QuietDatabase.actor.cpp | 443 +- fdbserver/QuietDatabase.h | 27 +- fdbserver/Ratekeeper.actor.cpp | 506 +- fdbserver/RatekeeperInterface.h | 20 +- fdbserver/RecoveryState.h | 64 +- fdbserver/Resolver.actor.cpp | 260 +- fdbserver/ResolverInterface.h | 42 +- fdbserver/Restore.actor.cpp | 36 +- fdbserver/RestoreInterface.h | 14 +- fdbserver/ServerDBInfo.h | 45 +- fdbserver/SimulatedCluster.actor.cpp | 1127 +- fdbserver/SimulatedCluster.h | 6 +- fdbserver/SkipList.cpp | 1235 +- fdbserver/Status.actor.cpp | 1374 +- fdbserver/Status.h | 16 +- fdbserver/StorageMetrics.actor.h | 337 +- fdbserver/StorageMetrics.h | 4 +- fdbserver/TLogInterface.h | 130 +- fdbserver/TLogServer.actor.cpp | 2115 +-- fdbserver/TagPartitionedLogSystem.actor.cpp | 2382 +-- fdbserver/TesterInterface.actor.h | 53 +- fdbserver/VFSAsync.cpp | 898 +- fdbserver/VFSAsync.h | 86 +- fdbserver/VersionedBTree.actor.cpp | 3405 +++-- fdbserver/WaitFailure.actor.cpp | 52 +- fdbserver/WaitFailure.h | 19 +- fdbserver/WorkerInterface.actor.h | 272 +- fdbserver/fdbserver.actor.cpp | 1855 +-- fdbserver/masterserver.actor.cpp | 1250 +- fdbserver/networktest.actor.cpp | 69 +- fdbserver/pubsub.actor.cpp | 181 +- fdbserver/pubsub.h | 3 +- fdbserver/sqlite/btree.c | 12579 ++++++++-------- fdbserver/sqlite/btree.h | 189 +- fdbserver/sqlite/hash.h | 35 +- fdbserver/sqlite/shell.c | 4699 +++--- fdbserver/sqlite/sqlite3.h | 1722 +-- fdbserver/sqlite/sqlite3ext.h | 755 +- fdbserver/sqlite/sqliteInt.h | 1182 +- fdbserver/sqlite/sqliteLimit.h | 69 +- fdbserver/storageserver.actor.cpp | 2928 ++-- fdbserver/template_fdb.h | 3180 ++-- fdbserver/tester.actor.cpp | 941 +- fdbserver/worker.actor.cpp | 1080 +- fdbserver/workloads/ApiCorrectness.actor.cpp | 484 +- fdbserver/workloads/ApiWorkload.actor.cpp | 200 +- fdbserver/workloads/ApiWorkload.h | 313 +- fdbserver/workloads/AsyncFile.actor.h | 82 +- fdbserver/workloads/AsyncFile.cpp | 78 +- .../workloads/AsyncFileCorrectness.actor.cpp | 395 +- fdbserver/workloads/AsyncFileRead.actor.cpp | 220 +- fdbserver/workloads/AsyncFileWrite.actor.cpp | 113 +- fdbserver/workloads/AtomicOps.actor.cpp | 149 +- .../AtomicOpsApiCorrectness.actor.cpp | 367 +- fdbserver/workloads/AtomicRestore.actor.cpp | 56 +- .../workloads/AtomicSwitchover.actor.cpp | 102 +- .../workloads/BackgroundSelectors.actor.cpp | 157 +- .../workloads/BackupCorrectness.actor.cpp | 517 +- fdbserver/workloads/BackupToDBAbort.actor.cpp | 33 +- .../workloads/BackupToDBCorrectness.actor.cpp | 444 +- .../workloads/BackupToDBUpgrade.actor.cpp | 264 +- fdbserver/workloads/BulkLoad.actor.cpp | 78 +- fdbserver/workloads/BulkSetup.actor.h | 258 +- fdbserver/workloads/ChangeConfig.actor.cpp | 71 +- ...entTransactionProfileCorrectness.actor.cpp | 265 +- fdbserver/workloads/CommitBugCheck.actor.cpp | 91 +- .../workloads/ConfigureDatabase.actor.cpp | 251 +- fdbserver/workloads/ConflictRange.actor.cpp | 334 +- .../workloads/ConsistencyCheck.actor.cpp | 1208 +- fdbserver/workloads/CpuProfiler.actor.cpp | 106 +- fdbserver/workloads/Cycle.actor.cpp | 167 +- fdbserver/workloads/DDBalance.actor.cpp | 200 +- fdbserver/workloads/DDMetrics.actor.cpp | 39 +- .../workloads/DDMetricsExclude.actor.cpp | 46 +- .../DifferentClustersSameRV.actor.cpp | 6 +- fdbserver/workloads/DiskDurability.actor.cpp | 96 +- .../workloads/DiskDurabilityTest.actor.cpp | 123 +- fdbserver/workloads/DummyWorkload.actor.cpp | 18 +- .../workloads/ExternalWorkload.actor.cpp | 8 +- .../workloads/FastTriggeredWatches.actor.cpp | 117 +- fdbserver/workloads/FileSystem.actor.cpp | 285 +- fdbserver/workloads/Fuzz.cpp | 16 +- .../workloads/FuzzApiCorrectness.actor.cpp | 813 +- fdbserver/workloads/Increment.actor.cpp | 127 +- fdbserver/workloads/IndexScan.actor.cpp | 88 +- fdbserver/workloads/Inventory.actor.cpp | 178 +- fdbserver/workloads/KVStoreTest.actor.cpp | 259 +- fdbserver/workloads/KillRegion.actor.cpp | 75 +- fdbserver/workloads/LocalRatekeeper.actor.cpp | 29 +- fdbserver/workloads/LockDatabase.actor.cpp | 84 +- .../LockDatabaseFrequently.actor.cpp | 4 +- fdbserver/workloads/LogMetrics.actor.cpp | 55 +- fdbserver/workloads/LowLatency.actor.cpp | 57 +- .../workloads/MachineAttrition.actor.cpp | 176 +- fdbserver/workloads/Mako.actor.cpp | 236 +- fdbserver/workloads/MemoryKeyValueStore.cpp | 72 +- fdbserver/workloads/MemoryKeyValueStore.h | 24 +- fdbserver/workloads/MemoryLifetime.actor.cpp | 129 +- fdbserver/workloads/MetricLogging.actor.cpp | 59 +- fdbserver/workloads/Performance.actor.cpp | 173 +- fdbserver/workloads/Ping.actor.cpp | 211 +- fdbserver/workloads/PubSubMultiples.actor.cpp | 87 +- fdbserver/workloads/QueuePush.actor.cpp | 92 +- fdbserver/workloads/RYWDisable.actor.cpp | 86 +- fdbserver/workloads/RYWPerformance.actor.cpp | 293 +- fdbserver/workloads/RandomClogging.actor.cpp | 82 +- fdbserver/workloads/RandomMoveKeys.actor.cpp | 148 +- fdbserver/workloads/RandomSelector.actor.cpp | 469 +- fdbserver/workloads/ReadWrite.actor.cpp | 527 +- .../workloads/RemoveServersSafely.actor.cpp | 428 +- fdbserver/workloads/Rollback.actor.cpp | 78 +- fdbserver/workloads/RyowCorrectness.actor.cpp | 308 +- fdbserver/workloads/SaveAndKill.actor.cpp | 85 +- .../workloads/SelectorCorrectness.actor.cpp | 202 +- fdbserver/workloads/Serializability.actor.cpp | 404 +- fdbserver/workloads/Sideband.actor.cpp | 123 +- .../workloads/SlowTaskWorkload.actor.cpp | 36 +- fdbserver/workloads/SnapTest.actor.cpp | 24 +- fdbserver/workloads/StatusWorkload.actor.cpp | 99 +- fdbserver/workloads/Storefront.actor.cpp | 224 +- fdbserver/workloads/StreamingRead.actor.cpp | 116 +- .../workloads/SuspendProcesses.actor.cpp | 6 +- fdbserver/workloads/TargetedKill.actor.cpp | 86 +- .../workloads/TaskBucketCorrectness.actor.cpp | 185 +- fdbserver/workloads/ThreadSafety.actor.cpp | 142 +- fdbserver/workloads/Throttling.actor.cpp | 58 +- fdbserver/workloads/Throughput.actor.cpp | 324 +- .../workloads/TimeKeeperCorrectness.actor.cpp | 56 +- fdbserver/workloads/TriggerRecovery.actor.cpp | 14 +- fdbserver/workloads/UnitPerf.actor.cpp | 20 +- fdbserver/workloads/UnitTests.actor.cpp | 21 +- fdbserver/workloads/Unreadable.actor.cpp | 158 +- fdbserver/workloads/VersionStamp.actor.cpp | 139 +- fdbserver/workloads/WatchAndWait.actor.cpp | 91 +- fdbserver/workloads/Watches.actor.cpp | 196 +- fdbserver/workloads/WorkerErrors.actor.cpp | 39 +- fdbserver/workloads/WriteBandwidth.actor.cpp | 86 +- fdbserver/workloads/WriteDuringRead.actor.cpp | 1445 +- fdbserver/workloads/workloads.actor.h | 139 +- fdbservice/FDBService.cpp | 645 +- fdbservice/ServiceBase.cpp | 478 +- fdbservice/ServiceBase.h | 143 +- fdbservice/ThreadPool.h | 80 +- flow/ActorCollection.actor.cpp | 27 +- flow/ActorCollection.h | 38 +- flow/Arena.cpp | 12 +- flow/Arena.h | 443 +- flow/AsioReactor.h | 36 +- flow/CompressedInt.actor.cpp | 85 +- flow/CompressedInt.h | 92 +- flow/Deque.cpp | 4 +- flow/Deque.h | 79 +- flow/DeterministicRandom.cpp | 21 +- flow/Error.cpp | 32 +- flow/Error.h | 72 +- flow/EventTypes.actor.h | 15 +- flow/FastAlloc.cpp | 231 +- flow/FastAlloc.h | 117 +- flow/FastRef.h | 70 +- flow/FaultInjection.cpp | 2 +- flow/FaultInjection.h | 17 +- flow/FileIdentifier.h | 3 +- flow/FileTraceLogWriter.cpp | 61 +- flow/FileTraceLogWriter.h | 8 +- flow/Hash3.c | 1716 ++- flow/Hash3.h | 12 +- flow/Histogram.cpp | 8 +- flow/IDispatched.h | 29 +- flow/IRandom.h | 62 +- flow/IThreadPool.cpp | 61 +- flow/IThreadPool.h | 39 +- flow/IndexedSet.actor.h | 27 +- flow/IndexedSet.cpp | 309 +- flow/IndexedSet.h | 668 +- flow/Knobs.cpp | 371 +- flow/Knobs.h | 49 +- flow/MetricSample.h | 40 +- flow/Net2.actor.cpp | 814 +- flow/Net2Packet.cpp | 60 +- flow/Net2Packet.h | 30 +- flow/ObjectSerializer.h | 33 +- flow/Platform.cpp | 1535 +- flow/Profiler.actor.cpp | 134 +- flow/Profiler.h | 2 +- flow/ProtocolVersion.h | 2 +- flow/SignalSafeUnwind.cpp | 30 +- flow/SignalSafeUnwind.h | 1 - flow/SimpleOpt.h | 1327 +- flow/SystemMonitor.cpp | 136 +- flow/SystemMonitor.h | 28 +- flow/TDMetric.actor.h | 739 +- flow/TDMetric.cpp | 169 +- flow/TLSConfig.actor.cpp | 211 +- flow/TLSConfig.actor.h | 152 +- flow/ThreadHelper.actor.h | 316 +- flow/ThreadHelper.cpp | 2 +- flow/ThreadPrimitives.cpp | 12 +- flow/ThreadPrimitives.h | 37 +- flow/ThreadSafeQueue.h | 68 +- flow/Trace.cpp | 591 +- flow/Trace.h | 343 +- flow/UnitTest.cpp | 3 +- flow/UnitTest.h | 37 +- flow/Util.h | 4 +- flow/XmlTraceLogFormatter.cpp | 37 +- flow/XmlTraceLogFormatter.h | 5 +- flow/actorcompiler.h | 20 +- flow/error_definitions.h | 352 +- flow/flat_buffers.cpp | 15 +- flow/flat_buffers.h | 48 +- flow/flow.cpp | 85 +- flow/flow.h | 423 +- flow/genericactors.actor.cpp | 74 +- flow/genericactors.actor.h | 1179 +- flow/network.cpp | 42 +- flow/network.h | 174 +- flow/serialize.cpp | 8 +- flow/serialize.h | 402 +- flow/stacktrace.amalgamation.cpp | 2011 ++- flow/stacktrace.h | 33 +- monitoring/actor_flamegraph.cpp | 7 +- 546 files changed, 121628 insertions(+), 106307 deletions(-) mode change 100755 => 100644 bindings/c/test/mako/mako.c mode change 100755 => 100644 bindings/c/test/mako/mako.h mode change 100755 => 100644 bindings/c/test/mako/utils.c mode change 100755 => 100644 bindings/c/test/mako/utils.h mode change 100755 => 100644 bindings/flow/DirectoryLayer.h mode change 100755 => 100644 bindings/flow/DirectorySubspace.cpp mode change 100755 => 100644 bindings/flow/DirectorySubspace.h mode change 100755 => 100644 bindings/flow/FDBLoanerTypes.h mode change 100755 => 100644 bindings/flow/IDirectory.h mode change 100755 => 100644 bindings/flow/Node.actor.cpp mode change 100755 => 100644 bindings/flow/Subspace.cpp mode change 100755 => 100644 bindings/flow/Subspace.h mode change 100755 => 100644 bindings/flow/Tuple.cpp mode change 100755 => 100644 bindings/flow/Tuple.h mode change 100755 => 100644 fdbclient/JsonBuilder.cpp mode change 100755 => 100644 fdbclient/MetricLogger.h mode change 100755 => 100644 fdbclient/json_spirit/json_spirit_error_position.h mode change 100755 => 100644 fdbclient/json_spirit/json_spirit_reader_template.h mode change 100755 => 100644 fdbclient/json_spirit/json_spirit_value.h mode change 100755 => 100644 fdbclient/json_spirit/json_spirit_writer_options.h mode change 100755 => 100644 fdbclient/json_spirit/json_spirit_writer_template.h mode change 100755 => 100644 fdbclient/libb64/cdecode.c mode change 100755 => 100644 fdbclient/libb64/cdecode.h mode change 100755 => 100644 fdbclient/libb64/cencode.h mode change 100755 => 100644 fdbclient/libb64/decode.h mode change 100755 => 100644 fdbclient/libb64/encode.h mode change 100755 => 100644 fdbclient/rapidjson/allocators.h mode change 100755 => 100644 fdbclient/rapidjson/document.h mode change 100755 => 100644 fdbclient/rapidjson/encodedstream.h mode change 100755 => 100644 fdbclient/rapidjson/encodings.h mode change 100755 => 100644 fdbclient/rapidjson/error/en.h mode change 100755 => 100644 fdbclient/rapidjson/error/error.h mode change 100755 => 100644 fdbclient/rapidjson/filereadstream.h mode change 100755 => 100644 fdbclient/rapidjson/filewritestream.h mode change 100755 => 100644 fdbclient/rapidjson/fwd.h mode change 100755 => 100644 fdbclient/rapidjson/internal/biginteger.h mode change 100755 => 100644 fdbclient/rapidjson/internal/diyfp.h mode change 100755 => 100644 fdbclient/rapidjson/internal/dtoa.h mode change 100755 => 100644 fdbclient/rapidjson/internal/ieee754.h mode change 100755 => 100644 fdbclient/rapidjson/internal/itoa.h mode change 100755 => 100644 fdbclient/rapidjson/internal/meta.h mode change 100755 => 100644 fdbclient/rapidjson/internal/pow10.h mode change 100755 => 100644 fdbclient/rapidjson/internal/regex.h mode change 100755 => 100644 fdbclient/rapidjson/internal/stack.h mode change 100755 => 100644 fdbclient/rapidjson/internal/strfunc.h mode change 100755 => 100644 fdbclient/rapidjson/internal/strtod.h mode change 100755 => 100644 fdbclient/rapidjson/internal/swap.h mode change 100755 => 100644 fdbclient/rapidjson/istreamwrapper.h mode change 100755 => 100644 fdbclient/rapidjson/memorybuffer.h mode change 100755 => 100644 fdbclient/rapidjson/memorystream.h mode change 100755 => 100644 fdbclient/rapidjson/msinttypes/inttypes.h mode change 100755 => 100644 fdbclient/rapidjson/msinttypes/stdint.h mode change 100755 => 100644 fdbclient/rapidjson/ostreamwrapper.h mode change 100755 => 100644 fdbclient/rapidjson/pointer.h mode change 100755 => 100644 fdbclient/rapidjson/prettywriter.h mode change 100755 => 100644 fdbclient/rapidjson/rapidjson.h mode change 100755 => 100644 fdbclient/rapidjson/reader.h mode change 100755 => 100644 fdbclient/rapidjson/schema.h mode change 100755 => 100644 fdbclient/rapidjson/stream.h mode change 100755 => 100644 fdbclient/rapidjson/stringbuffer.h mode change 100755 => 100644 fdbclient/rapidjson/writer.h mode change 100755 => 100644 fdbclient/rapidxml/rapidxml.hpp mode change 100755 => 100644 fdbclient/rapidxml/rapidxml_iterators.hpp mode change 100755 => 100644 fdbclient/rapidxml/rapidxml_print.hpp mode change 100755 => 100644 fdbclient/rapidxml/rapidxml_utils.hpp mode change 100755 => 100644 fdbserver/sqlite/sqlite3.h mode change 100755 => 100644 fdbserver/workloads/DiskDurability.actor.cpp mode change 100755 => 100644 fdbserver/workloads/DiskDurabilityTest.actor.cpp mode change 100755 => 100644 fdbserver/workloads/KVStoreTest.actor.cpp mode change 100755 => 100644 flow/TDMetric.actor.h mode change 100755 => 100644 flow/error_definitions.h diff --git a/.clang-format b/.clang-format index c7faa9e0e6..60b24f6172 100644 --- a/.clang-format +++ b/.clang-format @@ -11,14 +11,14 @@ AllowAllParametersOfDeclarationOnNextLine: false AllowShortBlocksOnASingleLine: false AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: Inline -AllowShortIfStatementsOnASingleLine: true -AllowShortLoopsOnASingleLine: true +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false AlwaysBreakAfterDefinitionReturnType: None AlwaysBreakAfterReturnType: None AlwaysBreakBeforeMultilineStrings: false AlwaysBreakTemplateDeclarations: true -BinPackArguments: true -BinPackParameters: true +BinPackArguments: false +BinPackParameters: false BreakBeforeBinaryOperators: None BreakBeforeBraces: Attach ColumnLimit: 120 diff --git a/FDBLibTLS/FDBLibTLSPlugin.cpp b/FDBLibTLS/FDBLibTLSPlugin.cpp index 6bcdb48600..1be61305d1 100644 --- a/FDBLibTLS/FDBLibTLSPlugin.cpp +++ b/FDBLibTLS/FDBLibTLSPlugin.cpp @@ -31,10 +31,9 @@ FDBLibTLSPlugin::FDBLibTLSPlugin() { rc = tls_init(); } -FDBLibTLSPlugin::~FDBLibTLSPlugin() { -} +FDBLibTLSPlugin::~FDBLibTLSPlugin() {} -ITLSPolicy *FDBLibTLSPlugin::create_policy() { +ITLSPolicy* FDBLibTLSPlugin::create_policy() { if (rc < 0) { // Log the failure from tls_init during our constructor. TraceEvent(SevError, "FDBLibTLSInitError").detail("LibTLSErrorMessage", "failed to initialize libtls"); @@ -43,7 +42,7 @@ ITLSPolicy *FDBLibTLSPlugin::create_policy() { return new FDBLibTLSPolicy(Reference::addRef(this)); } -extern "C" BOOST_SYMBOL_EXPORT void *get_tls_plugin(const char *plugin_type_name_and_version) { +extern "C" BOOST_SYMBOL_EXPORT void* get_tls_plugin(const char* plugin_type_name_and_version) { if (strcmp(plugin_type_name_and_version, FDBLibTLSPlugin::get_plugin_type_name_and_version()) == 0) { return new FDBLibTLSPlugin; } diff --git a/FDBLibTLS/FDBLibTLSPlugin.h b/FDBLibTLS/FDBLibTLSPlugin.h index a6ffa03e55..98f11d63e1 100644 --- a/FDBLibTLS/FDBLibTLSPlugin.h +++ b/FDBLibTLS/FDBLibTLSPlugin.h @@ -35,7 +35,7 @@ struct FDBLibTLSPlugin : ITLSPlugin, ReferenceCounted { virtual void addref() { ReferenceCounted::addref(); } virtual void delref() { ReferenceCounted::delref(); } - virtual ITLSPolicy *create_policy(); + virtual ITLSPolicy* create_policy(); int rc; }; diff --git a/FDBLibTLS/FDBLibTLSPolicy.cpp b/FDBLibTLS/FDBLibTLSPolicy.cpp index d22f7d8f67..d1d7792581 100644 --- a/FDBLibTLS/FDBLibTLSPolicy.cpp +++ b/FDBLibTLS/FDBLibTLSPolicy.cpp @@ -37,9 +37,9 @@ #include #include -FDBLibTLSPolicy::FDBLibTLSPolicy(Reference plugin): - plugin(plugin), tls_cfg(NULL), roots(NULL), session_created(false), ca_data_set(false), - cert_data_set(false), key_data_set(false), verify_peers_set(false) { +FDBLibTLSPolicy::FDBLibTLSPolicy(Reference plugin) + : plugin(plugin), tls_cfg(NULL), roots(NULL), session_created(false), ca_data_set(false), cert_data_set(false), + key_data_set(false), verify_peers_set(false) { if ((tls_cfg = tls_config_new()) == NULL) { TraceEvent(SevError, "FDBLibTLSConfigError"); @@ -55,7 +55,13 @@ FDBLibTLSPolicy::~FDBLibTLSPolicy() { tls_config_free(tls_cfg); } -ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid) { +ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, + const char* servername, + TLSSendCallbackFunc send_func, + void* send_ctx, + TLSRecvCallbackFunc recv_func, + void* recv_ctx, + void* uid) { if (is_client) { // If verify peers has been set then there is no point specifying a // servername, since this will be ignored - the servername should be @@ -75,14 +81,21 @@ ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servern session_created = true; try { - return new FDBLibTLSSession(Reference::addRef(this), is_client, servername, send_func, send_ctx, recv_func, recv_ctx, uid); - } catch ( ... ) { + return new FDBLibTLSSession(Reference::addRef(this), + is_client, + servername, + send_func, + send_ctx, + recv_func, + recv_ctx, + uid); + } catch (...) { return NULL; } } -static int password_cb(char *buf, int size, int rwflag, void *u) { - const char *password = (const char *)u; +static int password_cb(char* buf, int size, int rwflag, void* u) { + const char* password = (const char*)u; int plen; if (size < 0) @@ -102,14 +115,14 @@ static int password_cb(char *buf, int size, int rwflag, void *u) { } struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, size_t cert_pem_len) { - struct stack_st_X509 *certs = NULL; - X509 *cert = NULL; - BIO *bio = NULL; + struct stack_st_X509* certs = NULL; + X509* cert = NULL; + BIO* bio = NULL; int errnum; if (cert_pem_len > INT_MAX) goto err; - if ((bio = BIO_new_mem_buf((void *)cert_pem, cert_pem_len)) == NULL) { + if ((bio = BIO_new_mem_buf((void*)cert_pem, cert_pem_len)) == NULL) { TraceEvent(SevError, "FDBLibTLSOutOfMemory"); goto err; } @@ -145,7 +158,7 @@ struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, s return certs; - err: +err: sk_X509_pop_free(certs, X509_free); X509_free(cert); BIO_free(bio); @@ -200,8 +213,8 @@ bool FDBLibTLSPolicy::set_cert_data(const uint8_t* cert_data, int cert_len) { } bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const char* password) { - EVP_PKEY *key = NULL; - BIO *bio = NULL; + EVP_PKEY* key = NULL; + BIO* bio = NULL; bool rc = false; if (key_data_set) { @@ -214,20 +227,20 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c } if (password != NULL) { - char *data; + char* data; long len; - if ((bio = BIO_new_mem_buf((void *)key_data, key_len)) == NULL) { + if ((bio = BIO_new_mem_buf((void*)key_data, key_len)) == NULL) { TraceEvent(SevError, "FDBLibTLSOutOfMemory"); goto err; } ERR_clear_error(); - if ((key = PEM_read_bio_PrivateKey(bio, NULL, password_cb, (void *)password)) == NULL) { + if ((key = PEM_read_bio_PrivateKey(bio, NULL, password_cb, (void*)password)) == NULL) { int errnum = ERR_peek_error(); char errbuf[256]; if ((ERR_GET_LIB(errnum) == ERR_LIB_PEM && ERR_GET_REASON(errnum) == PEM_R_BAD_DECRYPT) || - (ERR_GET_LIB(errnum) == ERR_LIB_EVP && ERR_GET_REASON(errnum) == EVP_R_BAD_DECRYPT)) { + (ERR_GET_LIB(errnum) == ERR_LIB_EVP && ERR_GET_REASON(errnum) == EVP_R_BAD_DECRYPT)) { TraceEvent(SevError, "FDBLibTLSIncorrectPassword"); } else { ERR_error_string_n(errnum, errbuf, sizeof(errbuf)); @@ -248,7 +261,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c TraceEvent(SevError, "FDBLibTLSOutOfMemory"); goto err; } - if (tls_config_set_key_mem(tls_cfg, (const uint8_t *)data, len) == -1) { + if (tls_config_set_key_mem(tls_cfg, (const uint8_t*)data, len) == -1) { TraceEvent(SevError, "FDBLibTLSKeyError").detail("LibTLSErrorMessage", tls_config_error(tls_cfg)); goto err; } @@ -262,7 +275,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c key_data_set = true; rc = true; - err: +err: BIO_free(bio); EVP_PKEY_free(key); return rc; @@ -287,20 +300,22 @@ bool FDBLibTLSPolicy::set_verify_peers(int count, const uint8_t* verify_peers[], try { std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]); int start = 0; - while(start < verifyString.size()) { + while (start < verifyString.size()) { int split = verifyString.find('|', start); - if(split == std::string::npos) { + if (split == std::string::npos) { break; } - if(split == start || verifyString[split-1] != '\\') { - Reference verify = Reference(new FDBLibTLSVerify(verifyString.substr(start,split-start))); + if (split == start || verifyString[split - 1] != '\\') { + Reference verify = + Reference(new FDBLibTLSVerify(verifyString.substr(start, split - start))); verify_rules.push_back(verify); - start = split+1; + start = split + 1; } } - Reference verify = Reference(new FDBLibTLSVerify(verifyString.substr(start))); + Reference verify = + Reference(new FDBLibTLSVerify(verifyString.substr(start))); verify_rules.push_back(verify); - } catch ( const std::runtime_error& e ) { + } catch (const std::runtime_error& e) { verify_rules.clear(); std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]); TraceEvent(SevError, "FDBLibTLSVerifyPeersParseError").detail("Config", verifyString); diff --git a/FDBLibTLS/FDBLibTLSPolicy.h b/FDBLibTLS/FDBLibTLSPolicy.h index 268a5f4182..ad874ddd3e 100644 --- a/FDBLibTLS/FDBLibTLSPolicy.h +++ b/FDBLibTLS/FDBLibTLSPolicy.h @@ -32,7 +32,7 @@ #include #include -struct FDBLibTLSPolicy: ITLSPolicy, ReferenceCounted { +struct FDBLibTLSPolicy : ITLSPolicy, ReferenceCounted { FDBLibTLSPolicy(Reference plugin); virtual ~FDBLibTLSPolicy(); @@ -41,7 +41,13 @@ struct FDBLibTLSPolicy: ITLSPolicy, ReferenceCounted { Reference plugin; - virtual ITLSSession* create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid); + virtual ITLSSession* create_session(bool is_client, + const char* servername, + TLSSendCallbackFunc send_func, + void* send_ctx, + TLSRecvCallbackFunc recv_func, + void* recv_ctx, + void* uid); struct stack_st_X509* parse_cert_pem(const uint8_t* cert_pem, size_t cert_pem_len); void parse_verify(std::string input); diff --git a/FDBLibTLS/FDBLibTLSSession.cpp b/FDBLibTLS/FDBLibTLSSession.cpp index d81dc4e509..dac0301e04 100644 --- a/FDBLibTLS/FDBLibTLSSession.cpp +++ b/FDBLibTLS/FDBLibTLSSession.cpp @@ -36,11 +36,10 @@ #include #include -static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb_arg) -{ - FDBLibTLSSession *session = (FDBLibTLSSession *)cb_arg; +static ssize_t tls_read_func(struct tls* ctx, void* buf, size_t buflen, void* cb_arg) { + FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg; - int rv = session->recv_func(session->recv_ctx, (uint8_t *)buf, buflen); + int rv = session->recv_func(session->recv_ctx, (uint8_t*)buf, buflen); if (rv < 0) return 0; if (rv == 0) @@ -48,11 +47,10 @@ static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb return (ssize_t)rv; } -static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, void *cb_arg) -{ - FDBLibTLSSession *session = (FDBLibTLSSession *)cb_arg; +static ssize_t tls_write_func(struct tls* ctx, const void* buf, size_t buflen, void* cb_arg) { + FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg; - int rv = session->send_func(session->send_ctx, (const uint8_t *)buf, buflen); + int rv = session->send_func(session->send_ctx, (const uint8_t*)buf, buflen); if (rv < 0) return 0; if (rv == 0) @@ -60,11 +58,18 @@ static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, v return (ssize_t)rv; } -FDBLibTLSSession::FDBLibTLSSession(Reference policy, bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uidptr) : - tls_ctx(NULL), tls_sctx(NULL), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx), - recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) { +FDBLibTLSSession::FDBLibTLSSession(Reference policy, + bool is_client, + const char* servername, + TLSSendCallbackFunc send_func, + void* send_ctx, + TLSRecvCallbackFunc recv_func, + void* recv_ctx, + void* uidptr) + : tls_ctx(NULL), tls_sctx(NULL), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx), + recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) { if (uidptr) - uid = * (UID*) uidptr; + uid = *(UID*)uidptr; if (is_client) { if ((tls_ctx = tls_client()) == NULL) { @@ -123,12 +128,10 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match if ((entry_utf8_len = ASN1_STRING_to_UTF8(&entry_utf8, entry)) < 1) goto err; if (mt == MatchType::EXACT) { - if (criteria_utf8_len == entry_utf8_len && - memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0) + if (criteria_utf8_len == entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0) rc = true; } else if (mt == MatchType::PREFIX) { - if (criteria_utf8_len <= entry_utf8_len && - memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0) + if (criteria_utf8_len <= entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0) rc = true; } else if (mt == MatchType::SUFFIX) { if (criteria_utf8_len <= entry_utf8_len && @@ -136,15 +139,15 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match rc = true; } - err: +err: ASN1_STRING_free(asn_criteria); free(criteria_utf8); free(entry_utf8); return rc; } -bool match_name_criteria(X509_NAME *name, NID nid, const std::string& criteria, MatchType mt) { - X509_NAME_ENTRY *name_entry; +bool match_name_criteria(X509_NAME* name, NID nid, const std::string& criteria, MatchType mt) { + X509_NAME_ENTRY* name_entry; int idx; // If name does not exist, or has multiple of this RDN, refuse to proceed. @@ -158,7 +161,7 @@ bool match_name_criteria(X509_NAME *name, NID nid, const std::string& criteria, return match_criteria_entry(criteria, name_entry->value, mt); } -bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, MatchType mt) { +bool match_extension_criteria(X509* cert, NID nid, const std::string& value, MatchType mt) { if (nid != NID_subject_alt_name && nid != NID_issuer_alt_name) { // I have no idea how other extensions work. return false; @@ -168,28 +171,26 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat return false; } std::string value_gen = value.substr(0, pos); - std::string value_val = value.substr(pos+1, value.npos); + std::string value_val = value.substr(pos + 1, value.npos); STACK_OF(GENERAL_NAME)* sans = reinterpret_cast(X509_get_ext_d2i(cert, nid, NULL, NULL)); if (sans == NULL) { return false; } - int num_sans = sk_GENERAL_NAME_num( sans ); + int num_sans = sk_GENERAL_NAME_num(sans); bool rc = false; - for( int i = 0; i < num_sans && !rc; ++i ) { - GENERAL_NAME* altname = sk_GENERAL_NAME_value( sans, i ); + for (int i = 0; i < num_sans && !rc; ++i) { + GENERAL_NAME* altname = sk_GENERAL_NAME_value(sans, i); std::string matchable; switch (altname->type) { case GEN_OTHERNAME: break; case GEN_EMAIL: - if (value_gen == "EMAIL" && - match_criteria_entry( value_val, altname->d.rfc822Name, mt)) { + if (value_gen == "EMAIL" && match_criteria_entry(value_val, altname->d.rfc822Name, mt)) { rc = true; break; } case GEN_DNS: - if (value_gen == "DNS" && - match_criteria_entry( value_val, altname->d.dNSName, mt )) { + if (value_gen == "DNS" && match_criteria_entry(value_val, altname->d.dNSName, mt)) { rc = true; break; } @@ -198,14 +199,12 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat case GEN_EDIPARTY: break; case GEN_URI: - if (value_gen == "URI" && - match_criteria_entry( value_val, altname->d.uniformResourceIdentifier, mt )) { + if (value_gen == "URI" && match_criteria_entry(value_val, altname->d.uniformResourceIdentifier, mt)) { rc = true; break; } case GEN_IPADD: - if (value_gen == "IP" && - match_criteria_entry( value_val, altname->d.iPAddress, mt )) { + if (value_gen == "IP" && match_criteria_entry(value_val, altname->d.iPAddress, mt)) { rc = true; break; } @@ -217,8 +216,13 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat return rc; } -bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string& criteria, MatchType mt, X509Location loc) { - switch(loc) { +bool match_criteria(X509* cert, + X509_NAME* subject, + NID nid, + const std::string& criteria, + MatchType mt, + X509Location loc) { + switch (loc) { case X509Location::NAME: { return match_name_criteria(subject, nid, criteria, mt); } @@ -230,8 +234,9 @@ bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string& return false; } -std::tuple FDBLibTLSSession::check_verify(Reference verify, struct stack_st_X509 *certs) { - X509_STORE_CTX *store_ctx = NULL; +std::tuple FDBLibTLSSession::check_verify(Reference verify, + struct stack_st_X509* certs) { + X509_STORE_CTX* store_ctx = NULL; X509_NAME *subject, *issuer; bool rc = false; X509* cert = NULL; @@ -257,7 +262,7 @@ std::tuple FDBLibTLSSession::check_verify(Referenceverify_time) X509_VERIFY_PARAM_set_flags(X509_STORE_CTX_get0_param(store_ctx), X509_V_FLAG_NO_CHECK_TIME); if (X509_verify_cert(store_ctx) <= 0) { - const char *errstr = X509_verify_cert_error_string(X509_STORE_CTX_get_error(store_ctx)); + const char* errstr = X509_verify_cert_error_string(X509_STORE_CTX_get_error(store_ctx)); reason = "Verify cert error: " + std::string(errstr); goto err; } @@ -268,8 +273,9 @@ std::tuple FDBLibTLSSession::check_verify(Referencesubject_criteria) { - if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { + for (auto& pair : verify->subject_criteria) { + if (!match_criteria( + cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { reason = "Cert subject match failure"; goto err; } @@ -280,8 +286,9 @@ std::tuple FDBLibTLSSession::check_verify(Referenceissuer_criteria) { - if (!match_criteria(cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { + for (auto& pair : verify->issuer_criteria) { + if (!match_criteria( + cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { reason = "Cert issuer match failure"; goto err; } @@ -293,8 +300,9 @@ std::tuple FDBLibTLSSession::check_verify(Referenceroot_criteria) { - if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { + for (auto& pair : verify->root_criteria) { + if (!match_criteria( + cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { reason = "Root subject match failure"; goto err; } @@ -303,15 +311,15 @@ std::tuple FDBLibTLSSession::check_verify(Reference verify_failure_reasons; @@ -331,7 +339,7 @@ bool FDBLibTLSSession::verify_peer() { goto err; // Any matching rule is sufficient. - for (auto &verify_rule: policy->verify_rules) { + for (auto& verify_rule : policy->verify_rules) { std::tie(verify_success, verify_failure_reason) = check_verify(verify_rule, certs); if (verify_success) { rc = true; @@ -344,7 +352,7 @@ bool FDBLibTLSSession::verify_peer() { if (!rc) { // log the various failure reasons - if(now() - lastVerifyFailureLogged > 1.0) { + if (now() - lastVerifyFailureLogged > 1.0) { for (std::string reason : verify_failure_reasons) { lastVerifyFailureLogged = now(); TraceEvent("FDBLibTLSVerifyFailure", uid).suppressFor(1.0).detail("Reason", reason); @@ -352,7 +360,7 @@ bool FDBLibTLSSession::verify_peer() { } } - err: +err: sk_X509_pop_free(certs, X509_free); return rc; diff --git a/FDBLibTLS/FDBLibTLSSession.h b/FDBLibTLS/FDBLibTLSSession.h index d648bd80a7..83f0e4d1c9 100644 --- a/FDBLibTLS/FDBLibTLSSession.h +++ b/FDBLibTLS/FDBLibTLSSession.h @@ -33,14 +33,21 @@ #include struct FDBLibTLSSession : ITLSSession, ReferenceCounted { - FDBLibTLSSession(Reference policy, bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid); + FDBLibTLSSession(Reference policy, + bool is_client, + const char* servername, + TLSSendCallbackFunc send_func, + void* send_ctx, + TLSRecvCallbackFunc recv_func, + void* recv_ctx, + void* uid); virtual ~FDBLibTLSSession(); virtual void addref() { ReferenceCounted::addref(); } virtual void delref() { ReferenceCounted::delref(); } bool verify_peer(); - std::tuple check_verify(Reference verify, struct stack_st_X509 *certs); + std::tuple check_verify(Reference verify, struct stack_st_X509* certs); virtual int handshake(); virtual int read(uint8_t* data, int length); @@ -50,8 +57,8 @@ struct FDBLibTLSSession : ITLSSession, ReferenceCounted { bool is_client; - struct tls *tls_ctx; - struct tls *tls_sctx; + struct tls* tls_ctx; + struct tls* tls_sctx; TLSSendCallbackFunc send_func; void* send_ctx; diff --git a/FDBLibTLS/FDBLibTLSVerify.cpp b/FDBLibTLS/FDBLibTLSVerify.cpp index 594389916f..2f9a5b678a 100644 --- a/FDBLibTLS/FDBLibTLSVerify.cpp +++ b/FDBLibTLS/FDBLibTLSVerify.cpp @@ -43,24 +43,24 @@ static int hexValue(char c) { static std::string de4514(std::string const& input, int start, int& out_end) { std::string output; - if(input[start] == '#' || input[start] == ' ') { + if (input[start] == '#' || input[start] == ' ') { out_end = start; return output; } int space_count = 0; - for(int p = start; p < input.size();) { - switch(input[p]) { + for (int p = start; p < input.size();) { + switch (input[p]) { case '\\': // Handle escaped sequence // Backslash escaping nothing! - if(p == input.size() - 1) { + if (p == input.size() - 1) { out_end = p; goto FIN; } - switch(input[p+1]) { + switch (input[p + 1]) { case ' ': case '"': case '#': @@ -72,24 +72,24 @@ static std::string de4514(std::string const& input, int start, int& out_end) { case '>': case '|': case '\\': - output += input[p+1]; + output += input[p + 1]; p += 2; space_count = 0; continue; default: // Backslash escaping pair of hex digits requires two characters - if(p == input.size() - 2) { + if (p == input.size() - 2) { out_end = p; goto FIN; } try { - output += hexValue(input[p+1]) * 16 + hexValue(input[p+2]); + output += hexValue(input[p + 1]) * 16 + hexValue(input[p + 2]); p += 3; space_count = 0; continue; - } catch( ... ) { + } catch (...) { out_end = p; goto FIN; } @@ -109,7 +109,7 @@ static std::string de4514(std::string const& input, int start, int& out_end) { default: // Character is what it is output += input[p]; - if(input[p] == ' ') + if (input[p] == ' ') space_count++; else space_count = 0; @@ -119,7 +119,7 @@ static std::string de4514(std::string const& input, int start, int& out_end) { out_end = input.size(); - FIN: +FIN: out_end -= space_count; output.resize(output.size() - space_count); @@ -128,16 +128,17 @@ static std::string de4514(std::string const& input, int start, int& out_end) { static std::pair splitPair(std::string const& input, char c) { int p = input.find_first_of(c); - if(p == input.npos) { + if (p == input.npos) { throw std::runtime_error("splitPair"); } - return std::make_pair(input.substr(0, p), input.substr(p+1, input.size())); + return std::make_pair(input.substr(0, p), input.substr(p + 1, input.size())); } static NID abbrevToNID(std::string const& sn) { NID nid = NID_undef; - if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" || sn == "subjectAltName") + if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" || + sn == "subjectAltName") nid = OBJ_sn2nid(sn.c_str()); if (nid == NID_undef) throw std::runtime_error("abbrevToNID"); @@ -158,13 +159,11 @@ static X509Location locationForNID(NID nid) { } } -FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config): - verify_cert(true), verify_time(true) { +FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config) : verify_cert(true), verify_time(true) { parse_verify(verify_config); } -FDBLibTLSVerify::~FDBLibTLSVerify() { -} +FDBLibTLSVerify::~FDBLibTLSVerify() {} void FDBLibTLSVerify::parse_verify(std::string input) { int s = 0; @@ -176,8 +175,10 @@ void FDBLibTLSVerify::parse_verify(std::string input) { throw std::runtime_error("parse_verify"); MatchType mt = MatchType::EXACT; - if (input[eq-1] == '>') mt = MatchType::PREFIX; - if (input[eq-1] == '<') mt = MatchType::SUFFIX; + if (input[eq - 1] == '>') + mt = MatchType::PREFIX; + if (input[eq - 1] == '<') + mt = MatchType::SUFFIX; std::string term = input.substr(s, eq - s - (mt == MatchType::EXACT ? 0 : 1)); if (term.find("Check.") == 0) { @@ -206,7 +207,7 @@ void FDBLibTLSVerify::parse_verify(std::string input) { s = eq + 3; } else { - std::map< int, Criteria >* criteria = &subject_criteria; + std::map* criteria = &subject_criteria; if (term.find('.') != term.npos) { auto scoped = splitPair(term, '.'); diff --git a/FDBLibTLS/FDBLibTLSVerify.h b/FDBLibTLS/FDBLibTLSVerify.h index 10fb07913e..8cf3330112 100644 --- a/FDBLibTLS/FDBLibTLSVerify.h +++ b/FDBLibTLS/FDBLibTLSVerify.h @@ -47,14 +47,10 @@ enum class X509Location { }; struct Criteria { - Criteria( const std::string& s ) - : criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {} - Criteria( const std::string& s, MatchType mt ) - : criteria(s), match_type(mt), location(X509Location::NAME) {} - Criteria( const std::string& s, X509Location loc) - : criteria(s), match_type(MatchType::EXACT), location(loc) {} - Criteria( const std::string& s, MatchType mt, X509Location loc) - : criteria(s), match_type(mt), location(loc) {} + Criteria(const std::string& s) : criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {} + Criteria(const std::string& s, MatchType mt) : criteria(s), match_type(mt), location(X509Location::NAME) {} + Criteria(const std::string& s, X509Location loc) : criteria(s), match_type(MatchType::EXACT), location(loc) {} + Criteria(const std::string& s, MatchType mt, X509Location loc) : criteria(s), match_type(mt), location(loc) {} std::string criteria; MatchType match_type; @@ -65,7 +61,7 @@ struct Criteria { } }; -struct FDBLibTLSVerify: ReferenceCounted { +struct FDBLibTLSVerify : ReferenceCounted { FDBLibTLSVerify(std::string verify); virtual ~FDBLibTLSVerify(); @@ -77,9 +73,9 @@ struct FDBLibTLSVerify: ReferenceCounted { bool verify_cert; bool verify_time; - std::map< NID, Criteria > subject_criteria; - std::map< NID, Criteria > issuer_criteria; - std::map< NID, Criteria > root_criteria; + std::map subject_criteria; + std::map issuer_criteria; + std::map root_criteria; }; #endif /* FDB_LIBTLS_VERIFY_H */ diff --git a/FDBLibTLS/plugin-test.cpp b/FDBLibTLS/plugin-test.cpp index e12d4b82b6..3567ec83a8 100644 --- a/FDBLibTLS/plugin-test.cpp +++ b/FDBLibTLS/plugin-test.cpp @@ -36,8 +36,7 @@ #define TESTDATA "./testdata/" -static std::string load_file(std::string path) -{ +static std::string load_file(std::string path) { std::ifstream fs(path); std::stringstream ss; @@ -71,8 +70,8 @@ struct FDBLibTLSPluginTest { boost::circular_buffer client_buffer; boost::circular_buffer server_buffer; - int circular_read(boost::circular_buffer *cb, uint8_t* buf, int len); - int circular_write(boost::circular_buffer *cb, const uint8_t* buf, int len); + int circular_read(boost::circular_buffer* cb, uint8_t* buf, int len); + int circular_write(boost::circular_buffer* cb, const uint8_t* buf, int len); int client_read(uint8_t* buf, int len); int client_write(const uint8_t* buf, int len); int server_read(uint8_t* buf, int len); @@ -85,23 +84,18 @@ struct FDBLibTLSPluginTest { void circular_reset(void); void circular_self_test(void); - int client_server_test(const struct client_server_test *cst); + int client_server_test(const struct client_server_test* cst); int set_cert_data_test(void); }; -FDBLibTLSPluginTest::FDBLibTLSPluginTest(Reference plugin) : - plugin(plugin) -{ +FDBLibTLSPluginTest::FDBLibTLSPluginTest(Reference plugin) : plugin(plugin) { circular_reset(); circular_self_test(); } -FDBLibTLSPluginTest::~FDBLibTLSPluginTest() -{ -} +FDBLibTLSPluginTest::~FDBLibTLSPluginTest() {} -int FDBLibTLSPluginTest::circular_read(boost::circular_buffer *cb, uint8_t* buf, int len) -{ +int FDBLibTLSPluginTest::circular_read(boost::circular_buffer* cb, uint8_t* buf, int len) { int n = 0; for (n = 0; n < len; n++) { @@ -114,8 +108,7 @@ int FDBLibTLSPluginTest::circular_read(boost::circular_buffer *cb, uint return n; } -int FDBLibTLSPluginTest::circular_write(boost::circular_buffer *cb, const uint8_t* buf, int len) -{ +int FDBLibTLSPluginTest::circular_write(boost::circular_buffer* cb, const uint8_t* buf, int len) { int n = 0; for (n = 0; n < len; n++) { @@ -127,39 +120,33 @@ int FDBLibTLSPluginTest::circular_write(boost::circular_buffer *cb, con return n; } -int FDBLibTLSPluginTest::client_read(uint8_t* buf, int len) -{ +int FDBLibTLSPluginTest::client_read(uint8_t* buf, int len) { // Read bytes from the server from the client's buffer. return circular_read(&client_buffer, buf, len); } -int FDBLibTLSPluginTest::client_write(const uint8_t* buf, int len) -{ +int FDBLibTLSPluginTest::client_write(const uint8_t* buf, int len) { // Write bytes from the client into the server's buffer. return circular_write(&server_buffer, buf, len); } -int FDBLibTLSPluginTest::server_read(uint8_t* buf, int len) -{ +int FDBLibTLSPluginTest::server_read(uint8_t* buf, int len) { // Read bytes from the client from the server's buffer. return circular_read(&server_buffer, buf, len); } -int FDBLibTLSPluginTest::server_write(const uint8_t* buf, int len) -{ +int FDBLibTLSPluginTest::server_write(const uint8_t* buf, int len) { // Write bytes from the server into the client's buffer. return circular_write(&client_buffer, buf, len); } -void FDBLibTLSPluginTest::circular_reset() -{ +void FDBLibTLSPluginTest::circular_reset() { client_buffer = boost::circular_buffer(1024); server_buffer = boost::circular_buffer(1024); } -void FDBLibTLSPluginTest::circular_self_test() -{ - uint8_t buf[1024] = {1, 2, 3}; +void FDBLibTLSPluginTest::circular_self_test() { + uint8_t buf[1024] = { 1, 2, 3 }; std::cerr << "INFO: running circular buffer self tests...\n"; @@ -198,73 +185,74 @@ void FDBLibTLSPluginTest::circular_self_test() assert(client_read(buf, 1024) == 0); } -Reference FDBLibTLSPluginTest::create_policy(void) -{ +Reference FDBLibTLSPluginTest::create_policy(void) { return Reference(plugin->create_policy()); } static int client_send_func(void* ctx, const uint8_t* buf, int len) { - FDBLibTLSPluginTest *pt = (FDBLibTLSPluginTest *)ctx; + FDBLibTLSPluginTest* pt = (FDBLibTLSPluginTest*)ctx; try { return pt->client_write(buf, len); - } catch ( const std::runtime_error& e ) { + } catch (const std::runtime_error& e) { return -1; } } static int client_recv_func(void* ctx, uint8_t* buf, int len) { - FDBLibTLSPluginTest *pt = (FDBLibTLSPluginTest *)ctx; + FDBLibTLSPluginTest* pt = (FDBLibTLSPluginTest*)ctx; try { return pt->client_read(buf, len); - } catch ( const std::runtime_error& e ) { + } catch (const std::runtime_error& e) { return -1; } } -Reference FDBLibTLSPluginTest::create_client_session(Reference policy, const char* servername) -{ - return Reference(policy->create_session(true, servername, client_send_func, this, client_recv_func, this, NULL)); +Reference FDBLibTLSPluginTest::create_client_session(Reference policy, + const char* servername) { + return Reference( + policy->create_session(true, servername, client_send_func, this, client_recv_func, this, NULL)); } static int server_send_func(void* ctx, const uint8_t* buf, int len) { - FDBLibTLSPluginTest *pt = (FDBLibTLSPluginTest *)ctx; + FDBLibTLSPluginTest* pt = (FDBLibTLSPluginTest*)ctx; try { return pt->server_write(buf, len); - } catch ( const std::runtime_error& e ) { + } catch (const std::runtime_error& e) { return -1; } } static int server_recv_func(void* ctx, uint8_t* buf, int len) { - FDBLibTLSPluginTest *pt = (FDBLibTLSPluginTest *)ctx; + FDBLibTLSPluginTest* pt = (FDBLibTLSPluginTest*)ctx; try { return pt->server_read(buf, len); - } catch ( const std::runtime_error& e ) { + } catch (const std::runtime_error& e) { return -1; } } -Reference FDBLibTLSPluginTest::create_server_session(Reference policy) -{ - return Reference(policy->create_session(false, NULL, server_send_func, this, server_recv_func, this, NULL)); +Reference FDBLibTLSPluginTest::create_server_session(Reference policy) { + return Reference( + policy->create_session(false, NULL, server_send_func, this, server_recv_func, this, NULL)); } #define MAX_VERIFY_RULES 5 -static void convert_verify_peers(const std::vector *verify_rules, const uint8_t *verify_peers[], int verify_peers_len[]) { +static void convert_verify_peers(const std::vector* verify_rules, + const uint8_t* verify_peers[], + int verify_peers_len[]) { if (verify_rules->size() > MAX_VERIFY_RULES) throw std::runtime_error("verify"); int i = 0; - for (auto &verify_rule: *verify_rules) { - verify_peers[i] = (const uint8_t *)&verify_rule[0]; + for (auto& verify_rule : *verify_rules) { + verify_peers[i] = (const uint8_t*)&verify_rule[0]; verify_peers_len[i] = verify_rule.size(); i++; } } -int FDBLibTLSPluginTest::client_server_test(const struct client_server_test* cst) -{ - const uint8_t *verify_peers[MAX_VERIFY_RULES]; +int FDBLibTLSPluginTest::client_server_test(const struct client_server_test* cst) { + const uint8_t* verify_peers[MAX_VERIFY_RULES]; int verify_peers_len[MAX_VERIFY_RULES]; circular_reset(); @@ -395,7 +383,7 @@ int FDBLibTLSPluginTest::client_server_test(const struct client_server_test* cst i = 0; do { if (!client_done) { - rc = client_session->write((const uint8_t*)&client_msg[cn], client_msg.size()-cn); + rc = client_session->write((const uint8_t*)&client_msg[cn], client_msg.size() - cn); if (rc > 0) { cn += rc; if (cn >= client_msg.size()) @@ -446,7 +434,7 @@ int FDBLibTLSPluginTest::client_server_test(const struct client_server_test* cst i = 0; do { if (!server_done) { - rc = server_session->write((const uint8_t*)&server_msg[cn], server_msg.size()-cn); + rc = server_session->write((const uint8_t*)&server_msg[cn], server_msg.size() - cn); if (rc > 0) { cn += rc; if (cn >= server_msg.size()) @@ -490,7 +478,7 @@ int FDBLibTLSPluginTest::client_server_test(const struct client_server_test* cst static void logf(const char* event, void* uid, bool is_error, ...) { va_list args; - std::string log_type ("INFO"); + std::string log_type("INFO"); if (is_error) log_type = "ERROR"; @@ -498,10 +486,10 @@ static void logf(const char* event, void* uid, bool is_error, ...) { va_start(args, is_error); - const char *s = va_arg(args, const char *); + const char* s = va_arg(args, const char*); while (s != NULL) { std::cerr << " " << s; - s = va_arg(args, const char *); + s = va_arg(args, const char*); } std::cerr << "\n"; @@ -512,635 +500,638 @@ static void logf(const char* event, void* uid, bool is_error, ...) { const struct client_server_test client_server_tests[] = { // Single root CA. { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, // Multiple root CAs. { - .ca_path = "test-ca-all.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-4.pem", - .server_password = "fdb123", - .server_verify = {""}, + .ca_path = "test-ca-all.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-4.pem", + .server_password = "fdb123", + .server_verify = { "" }, }, { - .ca_path = "test-ca-all.pem", - .client_success = true, - .client_path = "test-client-4.pem", - .client_password = "fdb321", - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = "fdb123", - .server_verify = {""}, + .ca_path = "test-ca-all.pem", + .client_success = true, + .client_path = "test-client-4.pem", + .client_password = "fdb321", + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = "fdb123", + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-4.pem", - .server_password = "fdb123", - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-4.pem", + .server_password = "fdb123", + .server_verify = { "" }, }, { - .ca_path = "test-ca-2.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = false, - .server_path = "test-server-4.pem", - .server_password = "fdb123", - .server_verify = {""}, + .ca_path = "test-ca-2.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = false, + .server_path = "test-server-4.pem", + .server_password = "fdb123", + .server_verify = { "" }, }, // Expired certificates. { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-3.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-3.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-3.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = false, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-3.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = false, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"Check.Unexpired=0"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-3.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "Check.Unexpired=0" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-3.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-3.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"Check.Unexpired=0"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-3.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "Check.Unexpired=0" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"Check.Valid=0"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-3.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "Check.Valid=0" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-3.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-3.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"Check.Valid=0"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-3.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "Check.Valid=0" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1", "I.CN=FDB LibTLS Plugin Test Intermediate CA 2,Check.Unexpired=0"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-3.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1", + "I.CN=FDB LibTLS Plugin Test Intermediate CA 2,Check.Unexpired=0" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-3.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1,Check.Unexpired=0", "I.CN=FDB LibTLS Plugin Test Intermediate CA 2"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-3.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1,Check.Unexpired=0", + "I.CN=FDB LibTLS Plugin Test Intermediate CA 2" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-3.pem", + .server_password = NULL, + .server_verify = { "" }, }, // Match on specific subject and/or issuer. { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"C=US"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "C=US" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"C=US"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "C=US" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"C=AU"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "C=AU" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"C=US", "C=AU"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "C=US", "C=AU" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"C=US", "C=JP"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "C=US", "C=JP" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\04\\>"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\04\\>" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\81 \\<\\01\\+\\02=\\04\\>"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\81 \\<\\01\\+\\02=\\04\\>" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\04"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\04" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {"CN=FDB LibTLS Plugin Test Client 1"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "CN=FDB LibTLS Plugin Test Client 1" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"CN=FDB LibTLS Plugin Test Client 1"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "CN=FDB LibTLS Plugin Test Client 1" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = false, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"O=Apple Pty Limited,OU=FDC Team"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = false, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "O=Apple Pty Limited,OU=FDC Team" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"O=Apple Inc.,OU=FDB Team"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"O=Apple Pty Limited,OU=FDB Team"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "O=Apple Inc.,OU=FDB Team" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "O=Apple Pty Limited,OU=FDB Team" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"O=Apple Inc.,OU=FDC Team"}, - .servername = NULL, - .server_success = false, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"O=Apple Pty Limited,OU=FDC Team"}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "O=Apple Inc.,OU=FDC Team" }, + .servername = NULL, + .server_success = false, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "O=Apple Pty Limited,OU=FDC Team" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDB Team"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDB Team"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDB Team" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDB Team" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDC Team"}, - .servername = NULL, - .server_success = false, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDC Team"}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDC Team" }, + .servername = NULL, + .server_success = false, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDC Team" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 2"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1"}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 2" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 2"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 2" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin Test Intermediate CA 2"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1,O=Apple Inc.,I.C=US,S.C=US"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin " + "Test Intermediate CA 2" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1,O=Apple Inc.,I.C=US,S.C=US" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin Test Intermediate CA 1"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1,O=Apple Inc.,I.C=US,S.C=US"}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin " + "Test Intermediate CA 1" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1,O=Apple Inc.,I.C=US,S.C=US" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 1"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 1"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 1" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 1" }, }, { - .ca_path = "test-ca-all.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 1"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-4.pem", - .server_password = "fdb123", - .server_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 1"}, + .ca_path = "test-ca-all.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 1" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-4.pem", + .server_password = "fdb123", + .server_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 1" }, }, { - .ca_path = "test-ca-all.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 1", "R.CN=FDB LibTLS Plugin Test Root CA 2"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-4.pem", - .server_password = "fdb123", - .server_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 1"}, + .ca_path = "test-ca-all.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 1", "R.CN=FDB LibTLS Plugin Test Root CA 2" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-4.pem", + .server_password = "fdb123", + .server_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 1" }, }, { - .ca_path = "test-ca-all.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 2"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-4.pem", - .server_password = "fdb123", - .server_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 1"}, + .ca_path = "test-ca-all.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 2" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-4.pem", + .server_password = "fdb123", + .server_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 1" }, }, { - .ca_path = "test-ca-all.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"R.OU=FDB Team"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-4.pem", - .server_password = "fdb123", - .server_verify = {"R.OU=FDB Team"}, + .ca_path = "test-ca-all.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "R.OU=FDB Team" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-4.pem", + .server_password = "fdb123", + .server_verify = { "R.OU=FDB Team" }, }, // Client performing name validation via servername. { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {}, - .servername = "test.foundationdb.org", - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = {}, + .servername = "test.foundationdb.org", + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {}, - .servername = "www.foundationdb.org", - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = {}, + .servername = "www.foundationdb.org", + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "" }, }, // Prefix and Suffix Matching { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"O>=Apple Inc.,OU>=FDB"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"O<=Limited,OU<=Team"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "O>=Apple Inc.,OU>=FDB" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "O<=Limited,OU<=Team" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"O<=Apple Inc.,OU<=FDB"}, - .servername = NULL, - .server_success = false, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"O>=Limited,OU>=Team"}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "O<=Apple Inc.,OU<=FDB" }, + .servername = NULL, + .server_success = false, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "O>=Limited,OU>=Team" }, }, // Subject Alternative Name { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"S.subjectAltName=DNS:test.foundationdb.org"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"Check.Valid=0"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "S.subjectAltName=DNS:test.foundationdb.org" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "Check.Valid=0" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"S.subjectAltName>=DNS:test."}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"Check.Valid=0"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "S.subjectAltName>=DNS:test." }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "Check.Valid=0" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"S.subjectAltName<=DNS:.org"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"Check.Valid=0"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "S.subjectAltName<=DNS:.org" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "Check.Valid=0" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"S.subjectAltName<=DNS:.com"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"Check.Valid=0"}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "S.subjectAltName<=DNS:.com" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "Check.Valid=0" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"S.subjectAltName<=EMAIL:.com"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"Check.Valid=0"}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "S.subjectAltName<=EMAIL:.com" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "Check.Valid=0" }, }, }; -int main(int argc, char **argv) -{ - void *pluginSO = NULL; - void *(*getPlugin)(const char*); +int main(int argc, char** argv) { + void* pluginSO = NULL; + void* (*getPlugin)(const char*); int failed = 0; if (argc != 2) { @@ -1154,18 +1145,19 @@ int main(int argc, char **argv) exit(1); } - getPlugin = (void*(*)(const char*))dlsym( pluginSO, "get_plugin" ); + getPlugin = (void* (*)(const char*))dlsym(pluginSO, "get_plugin"); if (getPlugin == NULL) { std::cerr << "plugin '" << argv[1] << "' does not provide get_plugin()\n"; exit(1); } - Reference plugin = Reference((ITLSPlugin *)getPlugin(ITLSPlugin::get_plugin_type_name_and_version())); + Reference plugin = + Reference((ITLSPlugin*)getPlugin(ITLSPlugin::get_plugin_type_name_and_version())); - FDBLibTLSPluginTest *pt = new FDBLibTLSPluginTest(plugin); + FDBLibTLSPluginTest* pt = new FDBLibTLSPluginTest(plugin); int test_num = 1; - for (auto &cst: client_server_tests) { + for (auto& cst : client_server_tests) { std::cerr << "== Test " << test_num++ << " ==\n"; failed |= pt->client_server_test(&cst); } diff --git a/FDBLibTLS/verify-test.cpp b/FDBLibTLS/verify-test.cpp index 1a0cdcbc5c..c1383b5e0b 100644 --- a/FDBLibTLS/verify-test.cpp +++ b/FDBLibTLS/verify-test.cpp @@ -33,11 +33,18 @@ #include "FDBLibTLS/FDBLibTLSPolicy.h" struct FDBLibTLSVerifyTest { - FDBLibTLSVerifyTest(std::string input): - input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}), root_criteria({}) {}; - FDBLibTLSVerifyTest(std::string input, bool verify_cert, bool verify_time, std::map subject, std::map issuer, std::map root): - input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject), issuer_criteria(issuer), root_criteria(root) {}; - ~FDBLibTLSVerifyTest() {}; + FDBLibTLSVerifyTest(std::string input) + : input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}), + root_criteria({}){}; + FDBLibTLSVerifyTest(std::string input, + bool verify_cert, + bool verify_time, + std::map subject, + std::map issuer, + std::map root) + : input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject), + issuer_criteria(issuer), root_criteria(root){}; + ~FDBLibTLSVerifyTest(){}; int run(); @@ -54,20 +61,21 @@ struct FDBLibTLSVerifyTest { static std::string criteriaToString(std::map const& criteria) { std::string s; - for (auto &pair: criteria) { - s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " + boost::lexical_cast((int)pair.second.match_type) + ", " + boost::lexical_cast((int)pair.second.location) + ")}"; + for (auto& pair : criteria) { + s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " + + boost::lexical_cast((int)pair.second.match_type) + ", " + + boost::lexical_cast((int)pair.second.location) + ")}"; } return "{" + s + "}"; } -static void logf(const char* event, void* uid, bool is_error, ...) { -} +static void logf(const char* event, void* uid, bool is_error, ...) {} int FDBLibTLSVerifyTest::run() { Reference verify; try { verify = Reference(new FDBLibTLSVerify(input)); - } catch ( const std::runtime_error& e ) { + } catch (const std::runtime_error& e) { if (valid) { std::cerr << "FAIL: Verify test failed, but should have succeeded - '" << input << "'\n"; return 1; @@ -87,15 +95,18 @@ int FDBLibTLSVerifyTest::run() { return 1; } if (verify->subject_criteria != subject_criteria) { - std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want " << criteriaToString(subject_criteria) << "\n"; + std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want " + << criteriaToString(subject_criteria) << "\n"; return 1; } if (verify->issuer_criteria != issuer_criteria) { - std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want " << criteriaToString(issuer_criteria) << "\n"; + std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want " + << criteriaToString(issuer_criteria) << "\n"; return 1; } if (verify->root_criteria != root_criteria) { - std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want " << criteriaToString(root_criteria) << "\n"; + std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want " + << criteriaToString(root_criteria) << "\n"; return 1; } return 0; @@ -105,7 +116,7 @@ static int policy_verify_test() { Reference plugin = Reference(new FDBLibTLSPlugin()); Reference policy = Reference(new FDBLibTLSPolicy(plugin, (ITLSLogFunc)logf)); - const char *verify_peers[] = { + const char* verify_peers[] = { "S.CN=abc", "I.CN=def", "R.CN=xyz,Check.Unexpired=0", @@ -121,7 +132,7 @@ static int policy_verify_test() { Reference(new FDBLibTLSVerify(std::string(verify_peers[2], verify_peers_len[2]))), }; - if (!policy->set_verify_peers(3, (const uint8_t **)verify_peers, verify_peers_len)) { + if (!policy->set_verify_peers(3, (const uint8_t**)verify_peers, verify_peers_len)) { std::cerr << "FAIL: Policy verify test failed, but should have succeeded\n"; return 1; } @@ -131,25 +142,30 @@ static int policy_verify_test() { } int i = 0; - for (auto &verify_rule: policy->verify_rules) { + for (auto& verify_rule : policy->verify_rules) { if (verify_rule->verify_cert != verify_rules[i]->verify_cert) { - std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want " << verify_rules[i]->verify_cert << "\n"; + std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want " + << verify_rules[i]->verify_cert << "\n"; return 1; } if (verify_rule->verify_time != verify_rules[i]->verify_time) { - std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want " << verify_rules[i]->verify_time << "\n"; + std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want " + << verify_rules[i]->verify_time << "\n"; return 1; } if (verify_rule->subject_criteria != verify_rules[i]->subject_criteria) { - std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want " << criteriaToString(verify_rules[i]->subject_criteria) << "\n"; + std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want " + << criteriaToString(verify_rules[i]->subject_criteria) << "\n"; return 1; } if (verify_rule->issuer_criteria != verify_rules[i]->issuer_criteria) { - std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want " << criteriaToString(verify_rules[i]->issuer_criteria) << "\n"; + std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want " + << criteriaToString(verify_rules[i]->issuer_criteria) << "\n"; return 1; } if (verify_rule->root_criteria != verify_rules[i]->root_criteria) { - std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want " << criteriaToString(verify_rules[i]->root_criteria) << "\n"; + std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want " + << criteriaToString(verify_rules[i]->root_criteria) << "\n"; return 1; } i++; @@ -157,8 +173,7 @@ static int policy_verify_test() { return 0; } -int main(int argc, char **argv) -{ +int main(int argc, char** argv) { int failed = 0; #define EXACT(x) Criteria(x, MatchType::EXACT, X509Location::NAME) @@ -173,29 +188,60 @@ int main(int argc, char **argv) FDBLibTLSVerifyTest("Check.Unexpired=0", true, false, {}, {}, {}), FDBLibTLSVerifyTest("Check.Valid=1,Check.Unexpired=0", true, false, {}, {}, {}), FDBLibTLSVerifyTest("Check.Unexpired=0,Check.Valid=0", false, false, {}, {}, {}), - FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC", true, false, - {{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}), - FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC", true, false, - {{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {{NID_countryName, EXACT("US")}}, {}), - FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC", true, false, - {{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {}, {{NID_countryName, EXACT("US")}}), - FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC", true, false, - {{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}), - FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC", true, false, - {{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}), - FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC", true, true, - {{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}), - FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc", true, true, - {{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, - {{NID_countryName, EXACT("US")}}, - {{NID_commonName, EXACT("abc")}}), - FDBLibTLSVerifyTest("C=\\,S=abc", true, true, {{NID_countryName, EXACT(",S=abc")}}, {}, {}), - FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, {{NID_commonName, EXACT("abc")}}, {}, {}), - FDBLibTLSVerifyTest("CN=a\\62c", true, true, {{NID_commonName, EXACT("abc")}}, {}, {}), - FDBLibTLSVerifyTest("CN=a\\01c", true, true, {{NID_commonName, EXACT("a\001c")}}, {}, {}), - FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp", true, true, {{NID_subject_alt_name, {"XYZCorp", MatchType::EXACT, X509Location::EXTENSION}}}, {}, {}), - FDBLibTLSVerifyTest("S.O>=XYZ", true, true, {{NID_organizationName, PREFIX("XYZ")}}, {}, {}), - FDBLibTLSVerifyTest("S.O<=LLC", true, true, {{NID_organizationName, SUFFIX("LLC")}}, {}, {}), + FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC", + true, + false, + { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } }, + { { NID_countryName, EXACT("US") } }, + {}), + FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC", + true, + false, + { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } }, + { { NID_countryName, EXACT("US") } }, + {}), + FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC", + true, + false, + { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } }, + {}, + { { NID_countryName, EXACT("US") } }), + FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC", + true, + false, + { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } }, + { { NID_countryName, EXACT("US") } }, + {}), + FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC", + true, + false, + { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } }, + { { NID_countryName, EXACT("US") } }, + {}), + FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC", + true, + true, + { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } }, + { { NID_countryName, EXACT("US") } }, + {}), + FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc", + true, + true, + { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } }, + { { NID_countryName, EXACT("US") } }, + { { NID_commonName, EXACT("abc") } }), + FDBLibTLSVerifyTest("C=\\,S=abc", true, true, { { NID_countryName, EXACT(",S=abc") } }, {}, {}), + FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}), + FDBLibTLSVerifyTest("CN=a\\62c", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}), + FDBLibTLSVerifyTest("CN=a\\01c", true, true, { { NID_commonName, EXACT("a\001c") } }, {}, {}), + FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp", + true, + true, + { { NID_subject_alt_name, { "XYZCorp", MatchType::EXACT, X509Location::EXTENSION } } }, + {}, + {}), + FDBLibTLSVerifyTest("S.O>=XYZ", true, true, { { NID_organizationName, PREFIX("XYZ") } }, {}, {}), + FDBLibTLSVerifyTest("S.O<=LLC", true, true, { { NID_organizationName, SUFFIX("LLC") } }, {}, {}), // Invalid cases. FDBLibTLSVerifyTest("Check.Invalid=0"), @@ -212,7 +258,7 @@ int main(int argc, char **argv) #undef PREFIX #undef SUFFIX - for (auto &test: tests) + for (auto& test : tests) failed |= test.run(); failed |= policy_verify_test(); diff --git a/bindings/c/ThreadCleanup.cpp b/bindings/c/ThreadCleanup.cpp index 20b49cf8e5..5531b6c012 100644 --- a/bindings/c/ThreadCleanup.cpp +++ b/bindings/c/ThreadCleanup.cpp @@ -25,14 +25,14 @@ #include -BOOL WINAPI DllMain( HINSTANCE dll, DWORD reason, LPVOID reserved ) { +BOOL WINAPI DllMain(HINSTANCE dll, DWORD reason, LPVOID reserved) { if (reason == DLL_THREAD_DETACH) releaseAllThreadMagazines(); return TRUE; } -#elif defined( __unixish__ ) +#elif defined(__unixish__) static pthread_key_t threadDestructorKey; @@ -41,13 +41,13 @@ static void threadDestructor(void*) { } void registerThread() { - pthread_setspecific( threadDestructorKey, (const void*)1 ); + pthread_setspecific(threadDestructorKey, (const void*)1); } static int initThreadDestructorKey() { if (!pthread_key_create(&threadDestructorKey, &threadDestructor)) { registerThread(); - setFastAllocatorThreadInitFunction( ®isterThread ); + setFastAllocatorThreadInitFunction(®isterThread); } return 0; diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index d31714d2c7..d0067657ff 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -44,133 +44,125 @@ int g_api_version = 0; // Legacy (pre API version 610) #define CLUSTER(c) ((char*)c) -/* - * While we could just use the MultiVersionApi instance directly, this #define allows us to swap in any other IClientApi instance (e.g. from ThreadSafeApi) +/* + * While we could just use the MultiVersionApi instance directly, this #define allows us to swap in any other IClientApi + * instance (e.g. from ThreadSafeApi) */ #define API ((IClientApi*)MultiVersionApi::api) /* This must be true so that we can return the data pointer of a Standalone as an array of FDBKeyValue. */ -static_assert( sizeof(FDBKeyValue) == sizeof(KeyValueRef), - "FDBKeyValue / KeyValueRef size mismatch" ); - +static_assert(sizeof(FDBKeyValue) == sizeof(KeyValueRef), "FDBKeyValue / KeyValueRef size mismatch"); #define TSAV_ERROR(type, error) ((FDBFuture*)(ThreadFuture(error())).extractPtr()) - -extern "C" DLLEXPORT -const char *fdb_get_error( fdb_error_t code ) { - return Error::fromUnvalidatedCode( code ).what(); +extern "C" DLLEXPORT const char* fdb_get_error(fdb_error_t code) { + return Error::fromUnvalidatedCode(code).what(); } -extern "C" DLLEXPORT -fdb_bool_t fdb_error_predicate( int predicate_test, fdb_error_t code ) { - if(predicate_test == FDBErrorPredicates::RETRYABLE) { - return fdb_error_predicate( FDBErrorPredicates::MAYBE_COMMITTED, code ) || - fdb_error_predicate( FDBErrorPredicates::RETRYABLE_NOT_COMMITTED, code ); +extern "C" DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_error_t code) { + if (predicate_test == FDBErrorPredicates::RETRYABLE) { + return fdb_error_predicate(FDBErrorPredicates::MAYBE_COMMITTED, code) || + fdb_error_predicate(FDBErrorPredicates::RETRYABLE_NOT_COMMITTED, code); } - if(predicate_test == FDBErrorPredicates::MAYBE_COMMITTED) { - return code == error_code_commit_unknown_result || - code == error_code_cluster_version_changed; + if (predicate_test == FDBErrorPredicates::MAYBE_COMMITTED) { + return code == error_code_commit_unknown_result || code == error_code_cluster_version_changed; } - if(predicate_test == FDBErrorPredicates::RETRYABLE_NOT_COMMITTED) { - return code == error_code_not_committed || - code == error_code_transaction_too_old || - code == error_code_future_version || - code == error_code_database_locked || - code == error_code_proxy_memory_limit_exceeded || - code == error_code_process_behind; + if (predicate_test == FDBErrorPredicates::RETRYABLE_NOT_COMMITTED) { + return code == error_code_not_committed || code == error_code_transaction_too_old || + code == error_code_future_version || code == error_code_database_locked || + code == error_code_proxy_memory_limit_exceeded || code == error_code_process_behind; } return false; } -#define RETURN_ON_ERROR(code_to_run) \ - try { code_to_run } \ - catch( Error& e) { if (e.code() <= 0) return internal_error().code(); else return e.code(); } \ - catch( ... ) { return error_code_unknown_error; } +#define RETURN_ON_ERROR(code_to_run) \ + try { \ + code_to_run \ + } catch (Error & e) { \ + if (e.code() <= 0) \ + return internal_error().code(); \ + else \ + return e.code(); \ + } catch (...) { \ + return error_code_unknown_error; \ + } -#define CATCH_AND_RETURN(code_to_run) \ - RETURN_ON_ERROR(code_to_run); \ +#define CATCH_AND_RETURN(code_to_run) \ + RETURN_ON_ERROR(code_to_run); \ return error_code_success; -#define CATCH_AND_DIE(code_to_run) \ - try { code_to_run } \ - catch ( Error& e ) { fprintf( stderr, "Unexpected FDB error %d\n", e.code() ); abort(); } \ - catch ( ... ) { fprintf( stderr, "Unexpected FDB unknown error\n" ); abort(); } +#define CATCH_AND_DIE(code_to_run) \ + try { \ + code_to_run \ + } catch (Error & e) { \ + fprintf(stderr, "Unexpected FDB error %d\n", e.code()); \ + abort(); \ + } catch (...) { \ + fprintf(stderr, "Unexpected FDB unknown error\n"); \ + abort(); \ + } -extern "C" DLLEXPORT -fdb_error_t fdb_network_set_option( FDBNetworkOption option, - uint8_t const* value, - int value_length ) -{ - CATCH_AND_RETURN( - API->setNetworkOption( (FDBNetworkOptions::Option)option, value ? StringRef( value, value_length ) : Optional() ); ); +extern "C" DLLEXPORT fdb_error_t fdb_network_set_option(FDBNetworkOption option, + uint8_t const* value, + int value_length) { + CATCH_AND_RETURN(API->setNetworkOption((FDBNetworkOptions::Option)option, + value ? StringRef(value, value_length) : Optional());); } fdb_error_t fdb_setup_network_impl() { - CATCH_AND_RETURN( API->setupNetwork(); ); + CATCH_AND_RETURN(API->setupNetwork();); } -fdb_error_t fdb_setup_network_v13( const char* localAddress ) { - fdb_error_t errorCode = fdb_network_set_option( FDB_NET_OPTION_LOCAL_ADDRESS, (uint8_t const*)localAddress, strlen(localAddress) ); - if(errorCode != 0) +fdb_error_t fdb_setup_network_v13(const char* localAddress) { + fdb_error_t errorCode = + fdb_network_set_option(FDB_NET_OPTION_LOCAL_ADDRESS, (uint8_t const*)localAddress, strlen(localAddress)); + if (errorCode != 0) return errorCode; return fdb_setup_network_impl(); } -extern "C" DLLEXPORT -fdb_error_t fdb_run_network() { - CATCH_AND_RETURN( API->runNetwork(); ); +extern "C" DLLEXPORT fdb_error_t fdb_run_network() { + CATCH_AND_RETURN(API->runNetwork();); } -extern "C" DLLEXPORT -fdb_error_t fdb_stop_network() { - CATCH_AND_RETURN( API->stopNetwork(); ); +extern "C" DLLEXPORT fdb_error_t fdb_stop_network() { + CATCH_AND_RETURN(API->stopNetwork();); } -extern "C" DLLEXPORT -fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void *hook_parameter) { - CATCH_AND_RETURN( API->addNetworkThreadCompletionHook(hook, hook_parameter); ); +extern "C" DLLEXPORT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void* hook_parameter) { + CATCH_AND_RETURN(API->addNetworkThreadCompletionHook(hook, hook_parameter);); } -extern "C" DLLEXPORT -void fdb_future_cancel( FDBFuture* f ) { - CATCH_AND_DIE( - TSAVB(f)->addref(); - TSAVB(f)->cancel(); - ); +extern "C" DLLEXPORT void fdb_future_cancel(FDBFuture* f) { + CATCH_AND_DIE(TSAVB(f)->addref(); TSAVB(f)->cancel();); } -extern "C" DLLEXPORT -void fdb_future_release_memory( FDBFuture* f ) { - CATCH_AND_DIE( TSAVB(f)->releaseMemory(); ); +extern "C" DLLEXPORT void fdb_future_release_memory(FDBFuture* f) { + CATCH_AND_DIE(TSAVB(f)->releaseMemory();); } -extern "C" DLLEXPORT -void fdb_future_destroy( FDBFuture* f ) { - CATCH_AND_DIE( TSAVB(f)->cancel(); ); +extern "C" DLLEXPORT void fdb_future_destroy(FDBFuture* f) { + CATCH_AND_DIE(TSAVB(f)->cancel();); } -extern "C" DLLEXPORT -fdb_error_t fdb_future_block_until_ready( FDBFuture* f ) { +extern "C" DLLEXPORT fdb_error_t fdb_future_block_until_ready(FDBFuture* f) { CATCH_AND_RETURN(TSAVB(f)->blockUntilReadyCheckOnMainThread();); } -fdb_bool_t fdb_future_is_error_v22( FDBFuture* f ) { +fdb_bool_t fdb_future_is_error_v22(FDBFuture* f) { return TSAVB(f)->isError(); } -extern "C" DLLEXPORT -fdb_bool_t fdb_future_is_ready( FDBFuture* f ) { +extern "C" DLLEXPORT fdb_bool_t fdb_future_is_ready(FDBFuture* f) { return TSAVB(f)->isReady(); } class CAPICallback : public ThreadCallback { public: - CAPICallback(void (*callbackf)(FDBFuture*, void*), FDBFuture* f, - void* userdata) - : callbackf(callbackf), f(f), userdata(userdata) {} + CAPICallback(void (*callbackf)(FDBFuture*, void*), FDBFuture* f, void* userdata) + : callbackf(callbackf), f(f), userdata(userdata) {} virtual bool canFire(int notMadeActive) { return true; } virtual void fire(const Void& unused, int& userParam) { @@ -188,249 +180,217 @@ private: void* userdata; }; -extern "C" DLLEXPORT -fdb_error_t fdb_future_set_callback( FDBFuture* f, - void (*callbackf)(FDBFuture*, void*), - void* userdata ) { +extern "C" DLLEXPORT fdb_error_t fdb_future_set_callback(FDBFuture* f, + void (*callbackf)(FDBFuture*, void*), + void* userdata) { CAPICallback* cb = new CAPICallback(callbackf, f, userdata); int ignore; - CATCH_AND_RETURN( TSAVB(f)->callOrSetAsCallback( cb, ignore, 0 ); ); + CATCH_AND_RETURN(TSAVB(f)->callOrSetAsCallback(cb, ignore, 0);); } -fdb_error_t fdb_future_get_error_impl( FDBFuture* f ) { +fdb_error_t fdb_future_get_error_impl(FDBFuture* f) { return TSAVB(f)->getErrorCode(); } -fdb_error_t fdb_future_get_error_v22( FDBFuture* f, const char** description ) { - if ( !( TSAVB(f)->isError() ) ) +fdb_error_t fdb_future_get_error_v22(FDBFuture* f, const char** description) { + if (!(TSAVB(f)->isError())) return error_code_future_not_error; if (description) *description = TSAVB(f)->error.what(); return TSAVB(f)->error.code(); } -fdb_error_t fdb_future_get_version_v619( FDBFuture* f, int64_t* out_version ) { - CATCH_AND_RETURN( *out_version = TSAV(Version, f)->get(); ); +fdb_error_t fdb_future_get_version_v619(FDBFuture* f, int64_t* out_version) { + CATCH_AND_RETURN(*out_version = TSAV(Version, f)->get();); } -extern "C" DLLEXPORT -fdb_error_t fdb_future_get_int64( FDBFuture* f, int64_t* out_value ) { - CATCH_AND_RETURN( *out_value = TSAV(int64_t, f)->get(); ); +extern "C" DLLEXPORT fdb_error_t fdb_future_get_int64(FDBFuture* f, int64_t* out_value) { + CATCH_AND_RETURN(*out_value = TSAV(int64_t, f)->get();); } -extern "C" DLLEXPORT -fdb_error_t fdb_future_get_key( FDBFuture* f, uint8_t const** out_key, - int* out_key_length ) { - CATCH_AND_RETURN( - KeyRef key = TSAV(Key, f)->get(); - *out_key = key.begin(); - *out_key_length = key.size(); ); +extern "C" DLLEXPORT fdb_error_t fdb_future_get_key(FDBFuture* f, uint8_t const** out_key, int* out_key_length) { + CATCH_AND_RETURN(KeyRef key = TSAV(Key, f)->get(); *out_key = key.begin(); *out_key_length = key.size();); } -fdb_error_t fdb_future_get_cluster_v609( FDBFuture* f, FDBCluster** out_cluster ) { - CATCH_AND_RETURN( - *out_cluster = (FDBCluster*) - ( (TSAV( char*, f )->get() ) ); ); +fdb_error_t fdb_future_get_cluster_v609(FDBFuture* f, FDBCluster** out_cluster) { + CATCH_AND_RETURN(*out_cluster = (FDBCluster*)((TSAV(char*, f)->get()));); } -fdb_error_t fdb_future_get_database_v609( FDBFuture* f, FDBDatabase** out_database ) { - CATCH_AND_RETURN( - *out_database = (FDBDatabase*) - ( (TSAV( Reference, f )->get() ).extractPtr() ); ); +fdb_error_t fdb_future_get_database_v609(FDBFuture* f, FDBDatabase** out_database) { + CATCH_AND_RETURN(*out_database = (FDBDatabase*)((TSAV(Reference, f)->get()).extractPtr());); } -extern "C" DLLEXPORT -fdb_error_t fdb_future_get_value( FDBFuture* f, fdb_bool_t* out_present, - uint8_t const** out_value, int* out_value_length ) { - CATCH_AND_RETURN( - Optional v = TSAV(Optional, f)->get(); - *out_present = v.present(); - if (*out_present) { - *out_value = v.get().begin(); - *out_value_length = v.get().size(); - } ); +extern "C" DLLEXPORT fdb_error_t fdb_future_get_value(FDBFuture* f, + fdb_bool_t* out_present, + uint8_t const** out_value, + int* out_value_length) { + CATCH_AND_RETURN(Optional v = TSAV(Optional, f)->get(); *out_present = v.present(); + if (*out_present) { + *out_value = v.get().begin(); + *out_value_length = v.get().size(); + }); } -fdb_error_t fdb_future_get_keyvalue_array_impl( - FDBFuture* f, FDBKeyValue const** out_kv, - int* out_count, fdb_bool_t* out_more ) -{ - CATCH_AND_RETURN( - Standalone rrr = TSAV(Standalone, f)->get(); - *out_kv = (FDBKeyValue*)rrr.begin(); - *out_count = rrr.size(); - *out_more = rrr.more; ); +fdb_error_t fdb_future_get_keyvalue_array_impl(FDBFuture* f, + FDBKeyValue const** out_kv, + int* out_count, + fdb_bool_t* out_more) { + CATCH_AND_RETURN(Standalone rrr = TSAV(Standalone, f)->get(); + *out_kv = (FDBKeyValue*)rrr.begin(); + *out_count = rrr.size(); + *out_more = rrr.more;); } -fdb_error_t fdb_future_get_keyvalue_array_v13( - FDBFuture* f, FDBKeyValue const** out_kv, int* out_count) -{ - CATCH_AND_RETURN( - Standalone rrr = TSAV(Standalone, f)->get(); - *out_kv = (FDBKeyValue*)rrr.begin(); - *out_count = rrr.size(); ); +fdb_error_t fdb_future_get_keyvalue_array_v13(FDBFuture* f, FDBKeyValue const** out_kv, int* out_count) { + CATCH_AND_RETURN(Standalone rrr = TSAV(Standalone, f)->get(); + *out_kv = (FDBKeyValue*)rrr.begin(); + *out_count = rrr.size();); } -extern "C" DLLEXPORT -fdb_error_t fdb_future_get_string_array( - FDBFuture* f, const char*** out_strings, int* out_count) -{ - CATCH_AND_RETURN( - Standalone> na = TSAV(Standalone>, f)->get(); - *out_strings = (const char **) na.begin(); - *out_count = na.size(); - ); +extern "C" DLLEXPORT fdb_error_t fdb_future_get_string_array(FDBFuture* f, const char*** out_strings, int* out_count) { + CATCH_AND_RETURN(Standalone> na = TSAV(Standalone>, f)->get(); + *out_strings = (const char**)na.begin(); + *out_count = na.size();); } -FDBFuture* fdb_create_cluster_v609( const char* cluster_file_path ) { - char *path; - if(cluster_file_path) { +FDBFuture* fdb_create_cluster_v609(const char* cluster_file_path) { + char* path; + if (cluster_file_path) { path = new char[strlen(cluster_file_path) + 1]; strcpy(path, cluster_file_path); - } - else { + } else { path = new char[1]; path[0] = '\0'; } return (FDBFuture*)ThreadFuture(path).extractPtr(); } -fdb_error_t fdb_cluster_set_option_v609( FDBCluster* c, - FDBClusterOption option, - uint8_t const* value, - int value_length ) -{ +fdb_error_t fdb_cluster_set_option_v609(FDBCluster* c, + FDBClusterOption option, + uint8_t const* value, + int value_length) { // There are no cluster options return error_code_success; } -void fdb_cluster_destroy_v609( FDBCluster* c ) { - CATCH_AND_DIE( delete[] CLUSTER(c); ); +void fdb_cluster_destroy_v609(FDBCluster* c) { + CATCH_AND_DIE(delete[] CLUSTER(c);); } // This exists so that fdb_cluster_create_database doesn't need to call the public symbol fdb_create_database. // If it does and this is an external client loaded though the multi-version API, then it may inadvertently call // the version of the function in the primary library if it was loaded into the global symbols. -fdb_error_t fdb_create_database_impl( const char* cluster_file_path, FDBDatabase** out_database ) { - CATCH_AND_RETURN( - *out_database = (FDBDatabase*)API->createDatabase( cluster_file_path ? cluster_file_path : "" ).extractPtr(); - ); +fdb_error_t fdb_create_database_impl(const char* cluster_file_path, FDBDatabase** out_database) { + CATCH_AND_RETURN(*out_database = + (FDBDatabase*)API->createDatabase(cluster_file_path ? cluster_file_path : "").extractPtr();); } -FDBFuture* fdb_cluster_create_database_v609( FDBCluster* c, uint8_t const* db_name, - int db_name_length ) -{ - if(strncmp((const char*)db_name, "DB", db_name_length) != 0) { +FDBFuture* fdb_cluster_create_database_v609(FDBCluster* c, uint8_t const* db_name, int db_name_length) { + if (strncmp((const char*)db_name, "DB", db_name_length) != 0) { return (FDBFuture*)ThreadFuture>(invalid_database_name()).extractPtr(); } - FDBDatabase *db; + FDBDatabase* db; fdb_error_t err = fdb_create_database_impl(CLUSTER(c), &db); - if(err) { + if (err) { return (FDBFuture*)ThreadFuture>(Error(err)).extractPtr(); } return (FDBFuture*)ThreadFuture>(Reference(DB(db))).extractPtr(); } -extern "C" DLLEXPORT -fdb_error_t fdb_create_database( const char* cluster_file_path, FDBDatabase** out_database ) { - return fdb_create_database_impl( cluster_file_path, out_database ); +extern "C" DLLEXPORT fdb_error_t fdb_create_database(const char* cluster_file_path, FDBDatabase** out_database) { + return fdb_create_database_impl(cluster_file_path, out_database); } -extern "C" DLLEXPORT -fdb_error_t fdb_database_set_option( FDBDatabase* d, - FDBDatabaseOption option, - uint8_t const* value, - int value_length ) -{ - CATCH_AND_RETURN( - DB(d)->setOption( (FDBDatabaseOptions::Option)option, value ? StringRef( value, value_length ) : Optional() ); ); +extern "C" DLLEXPORT fdb_error_t fdb_database_set_option(FDBDatabase* d, + FDBDatabaseOption option, + uint8_t const* value, + int value_length) { + CATCH_AND_RETURN(DB(d)->setOption((FDBDatabaseOptions::Option)option, + value ? StringRef(value, value_length) : Optional());); } -extern "C" DLLEXPORT -void fdb_database_destroy( FDBDatabase* d ) { - CATCH_AND_DIE( DB(d)->delref(); ); +extern "C" DLLEXPORT void fdb_database_destroy(FDBDatabase* d) { + CATCH_AND_DIE(DB(d)->delref();); } -extern "C" DLLEXPORT -fdb_error_t fdb_database_create_transaction( FDBDatabase* d, - FDBTransaction** out_transaction ) -{ - CATCH_AND_RETURN( - Reference tr = DB(d)->createTransaction(); - if(g_api_version <= 15) - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - *out_transaction = (FDBTransaction*)tr.extractPtr(); ); +extern "C" DLLEXPORT fdb_error_t fdb_database_create_transaction(FDBDatabase* d, FDBTransaction** out_transaction) { + CATCH_AND_RETURN(Reference tr = DB(d)->createTransaction(); + if (g_api_version <= 15) tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + *out_transaction = (FDBTransaction*)tr.extractPtr();); } - -extern "C" DLLEXPORT -void fdb_transaction_destroy( FDBTransaction* tr ) { +extern "C" DLLEXPORT void fdb_transaction_destroy(FDBTransaction* tr) { try { TXN(tr)->delref(); - } catch ( ... ) { } + } catch (...) { + } } -extern "C" DLLEXPORT -void fdb_transaction_cancel( FDBTransaction* tr ) { - CATCH_AND_DIE( TXN(tr)->cancel(); ); +extern "C" DLLEXPORT void fdb_transaction_cancel(FDBTransaction* tr) { + CATCH_AND_DIE(TXN(tr)->cancel();); } -extern "C" DLLEXPORT -void fdb_transaction_set_read_version( FDBTransaction* tr, int64_t version ) { - CATCH_AND_DIE( TXN(tr)->setVersion( version ); ); +extern "C" DLLEXPORT void fdb_transaction_set_read_version(FDBTransaction* tr, int64_t version) { + CATCH_AND_DIE(TXN(tr)->setVersion(version);); } -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_get_read_version( FDBTransaction* tr ) { - return (FDBFuture*)( TXN(tr)->getReadVersion().extractPtr() ); +extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_read_version(FDBTransaction* tr) { + return (FDBFuture*)(TXN(tr)->getReadVersion().extractPtr()); } -FDBFuture* fdb_transaction_get_impl( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length, fdb_bool_t snapshot ) { - return (FDBFuture*) - ( TXN(tr)->get( KeyRef( key_name, key_name_length ), snapshot ).extractPtr() ); +FDBFuture* fdb_transaction_get_impl(FDBTransaction* tr, + uint8_t const* key_name, + int key_name_length, + fdb_bool_t snapshot) { + return (FDBFuture*)(TXN(tr)->get(KeyRef(key_name, key_name_length), snapshot).extractPtr()); } -FDBFuture* fdb_transaction_get_v13( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length ) -{ - return fdb_transaction_get_impl( tr, key_name, key_name_length, 0 ); +FDBFuture* fdb_transaction_get_v13(FDBTransaction* tr, uint8_t const* key_name, int key_name_length) { + return fdb_transaction_get_impl(tr, key_name, key_name_length, 0); } -FDBFuture* fdb_transaction_get_key_impl( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length, fdb_bool_t or_equal, - int offset, fdb_bool_t snapshot ) { - return (FDBFuture*)( TXN(tr)->getKey( KeySelectorRef( - KeyRef( key_name, - key_name_length ), - or_equal, offset ), - snapshot ).extractPtr() ); +FDBFuture* fdb_transaction_get_key_impl(FDBTransaction* tr, + uint8_t const* key_name, + int key_name_length, + fdb_bool_t or_equal, + int offset, + fdb_bool_t snapshot) { + return (FDBFuture*)(TXN(tr) + ->getKey(KeySelectorRef(KeyRef(key_name, key_name_length), or_equal, offset), snapshot) + .extractPtr()); } -FDBFuture* fdb_transaction_get_key_v13( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length, fdb_bool_t or_equal, - int offset ) { - return fdb_transaction_get_key_impl( tr, key_name, key_name_length, - or_equal, offset, false ); +FDBFuture* fdb_transaction_get_key_v13(FDBTransaction* tr, + uint8_t const* key_name, + int key_name_length, + fdb_bool_t or_equal, + int offset) { + return fdb_transaction_get_key_impl(tr, key_name, key_name_length, or_equal, offset, false); } -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_get_addresses_for_key( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length ){ - return (FDBFuture*)( TXN(tr)->getAddressesForKey( KeyRef(key_name, key_name_length) ).extractPtr() ); - +extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_addresses_for_key(FDBTransaction* tr, + uint8_t const* key_name, + int key_name_length) { + return (FDBFuture*)(TXN(tr)->getAddressesForKey(KeyRef(key_name, key_name_length)).extractPtr()); } -FDBFuture* fdb_transaction_get_range_impl( - FDBTransaction* tr, uint8_t const* begin_key_name, - int begin_key_name_length, fdb_bool_t begin_or_equal, int begin_offset, - uint8_t const* end_key_name, int end_key_name_length, - fdb_bool_t end_or_equal, int end_offset, int limit, int target_bytes, - FDBStreamingMode mode, int iteration, fdb_bool_t snapshot, - fdb_bool_t reverse ) -{ +FDBFuture* fdb_transaction_get_range_impl(FDBTransaction* tr, + uint8_t const* begin_key_name, + int begin_key_name_length, + fdb_bool_t begin_or_equal, + int begin_offset, + uint8_t const* end_key_name, + int end_key_name_length, + fdb_bool_t end_or_equal, + int end_offset, + int limit, + int target_bytes, + FDBStreamingMode mode, + int iteration, + fdb_bool_t snapshot, + fdb_bool_t reverse) { /* This method may be called with a runtime API version of 13, in which negative row limits are a reverse range read */ if (g_api_version <= 13 && limit < 0) { @@ -445,21 +405,24 @@ FDBFuture* fdb_transaction_get_range_impl( target_bytes = CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED; /* Unlimited/unlimited with mode _EXACT isn't permitted */ - if (limit == CLIENT_KNOBS->ROW_LIMIT_UNLIMITED && target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED && mode == FDB_STREAMING_MODE_EXACT) + if (limit == CLIENT_KNOBS->ROW_LIMIT_UNLIMITED && target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED && + mode == FDB_STREAMING_MODE_EXACT) return TSAV_ERROR(Standalone, exact_mode_without_limits); /* _ITERATOR mode maps to one of the known streaming modes depending on iteration */ - static const int mode_bytes_array[] = {CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED, 256, 1000, 4096, 80000}; + static const int mode_bytes_array[] = { CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED, 256, 1000, 4096, 80000 }; /* The progression used for FDB_STREAMING_MODE_ITERATOR. Goes from small -> medium -> large. Then 1.5 * previous until serial. */ - static const int iteration_progression[] = { 256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000 }; + static const int iteration_progression[] = { + 256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000 + }; /* length(iteration_progression) */ static const int max_iteration = sizeof(iteration_progression) / sizeof(int); - if(mode == FDB_STREAMING_MODE_WANT_ALL) + if (mode == FDB_STREAMING_MODE_WANT_ALL) mode = FDB_STREAMING_MODE_SERIAL; int mode_bytes; @@ -469,173 +432,168 @@ FDBFuture* fdb_transaction_get_range_impl( iteration = std::min(iteration, max_iteration); mode_bytes = iteration_progression[iteration - 1]; - } - else if(mode >= 0 && mode <= FDB_STREAMING_MODE_SERIAL) + } else if (mode >= 0 && mode <= FDB_STREAMING_MODE_SERIAL) mode_bytes = mode_bytes_array[mode]; else return TSAV_ERROR(Standalone, client_invalid_operation); - if(target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED) + if (target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED) target_bytes = mode_bytes; - else if(mode_bytes != CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED) + else if (mode_bytes != CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED) target_bytes = std::min(target_bytes, mode_bytes); - return (FDBFuture*)( TXN(tr)->getRange( - KeySelectorRef( - KeyRef( begin_key_name, - begin_key_name_length ), - begin_or_equal, begin_offset ), - KeySelectorRef( - KeyRef( end_key_name, - end_key_name_length ), - end_or_equal, end_offset ), - GetRangeLimits(limit, target_bytes), - snapshot, reverse ).extractPtr() ); + return ( + FDBFuture*)(TXN(tr) + ->getRange( + KeySelectorRef(KeyRef(begin_key_name, begin_key_name_length), begin_or_equal, begin_offset), + KeySelectorRef(KeyRef(end_key_name, end_key_name_length), end_or_equal, end_offset), + GetRangeLimits(limit, target_bytes), + snapshot, + reverse) + .extractPtr()); } -FDBFuture* fdb_transaction_get_range_selector_v13( - FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, - fdb_bool_t begin_or_equal, int begin_offset, uint8_t const* end_key_name, - int end_key_name_length, fdb_bool_t end_or_equal, int end_offset, int limit ) -{ - return fdb_transaction_get_range_impl( - tr, begin_key_name, begin_key_name_length, begin_or_equal, begin_offset, - end_key_name, end_key_name_length, end_or_equal, end_offset, - limit, 0, FDB_STREAMING_MODE_EXACT, 0, false, false); +FDBFuture* fdb_transaction_get_range_selector_v13(FDBTransaction* tr, + uint8_t const* begin_key_name, + int begin_key_name_length, + fdb_bool_t begin_or_equal, + int begin_offset, + uint8_t const* end_key_name, + int end_key_name_length, + fdb_bool_t end_or_equal, + int end_offset, + int limit) { + return fdb_transaction_get_range_impl(tr, + begin_key_name, + begin_key_name_length, + begin_or_equal, + begin_offset, + end_key_name, + end_key_name_length, + end_or_equal, + end_offset, + limit, + 0, + FDB_STREAMING_MODE_EXACT, + 0, + false, + false); } -FDBFuture* fdb_transaction_get_range_v13( - FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, - uint8_t const* end_key_name, int end_key_name_length, int limit ) -{ +FDBFuture* fdb_transaction_get_range_v13(FDBTransaction* tr, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length, + int limit) { return fdb_transaction_get_range_selector_v13( - tr, - FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(begin_key_name, - begin_key_name_length), - FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(end_key_name, - end_key_name_length), - limit ); + tr, + FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(begin_key_name, begin_key_name_length), + FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(end_key_name, end_key_name_length), + limit); } -extern "C" DLLEXPORT -void fdb_transaction_set( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length, uint8_t const* value, - int value_length ) { +extern "C" DLLEXPORT void fdb_transaction_set(FDBTransaction* tr, + uint8_t const* key_name, + int key_name_length, + uint8_t const* value, + int value_length) { + CATCH_AND_DIE(TXN(tr)->set(KeyRef(key_name, key_name_length), ValueRef(value, value_length));); +} + +extern "C" DLLEXPORT void fdb_transaction_atomic_op(FDBTransaction* tr, + uint8_t const* key_name, + int key_name_length, + uint8_t const* param, + int param_length, + FDBMutationType operation_type) { + CATCH_AND_DIE(TXN(tr)->atomicOp( + KeyRef(key_name, key_name_length), ValueRef(param, param_length), (FDBMutationTypes::Option)operation_type);); +} + +extern "C" DLLEXPORT void fdb_transaction_clear(FDBTransaction* tr, uint8_t const* key_name, int key_name_length) { + CATCH_AND_DIE(TXN(tr)->clear(KeyRef(key_name, key_name_length));); +} + +extern "C" DLLEXPORT void fdb_transaction_clear_range(FDBTransaction* tr, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length) { CATCH_AND_DIE( - TXN(tr)->set( KeyRef( key_name, key_name_length ), - ValueRef( value, value_length ) ); ); + TXN(tr)->clear(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));); } -extern "C" DLLEXPORT -void fdb_transaction_atomic_op( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length, uint8_t const* param, - int param_length, FDBMutationType operation_type ) { - CATCH_AND_DIE( - TXN(tr)->atomicOp( KeyRef( key_name, key_name_length ), - ValueRef( param, param_length ), - (FDBMutationTypes::Option) operation_type ); ); +extern "C" DLLEXPORT FDBFuture* fdb_transaction_watch(FDBTransaction* tr, + uint8_t const* key_name, + int key_name_length) { + return (FDBFuture*)(TXN(tr)->watch(KeyRef(key_name, key_name_length)).extractPtr()); } -extern "C" DLLEXPORT -void fdb_transaction_clear( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length ) { - CATCH_AND_DIE( - TXN(tr)->clear( KeyRef( key_name, key_name_length ) ); ); +extern "C" DLLEXPORT FDBFuture* fdb_transaction_commit(FDBTransaction* tr) { + return (FDBFuture*)(TXN(tr)->commit().extractPtr()); } -extern "C" DLLEXPORT -void fdb_transaction_clear_range( - FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, - uint8_t const* end_key_name, int end_key_name_length ) -{ - CATCH_AND_DIE( - TXN(tr)->clear( KeyRef( begin_key_name, - begin_key_name_length ), - KeyRef( end_key_name, - end_key_name_length ) ); ); +extern "C" DLLEXPORT fdb_error_t fdb_transaction_get_committed_version(FDBTransaction* tr, int64_t* out_version) { + CATCH_AND_RETURN(*out_version = TXN(tr)->getCommittedVersion();); } -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_watch( FDBTransaction *tr, uint8_t const* key_name, - int key_name_length) -{ - return (FDBFuture*)( TXN(tr)->watch(KeyRef(key_name, key_name_length)).extractPtr() ); -} - -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_commit( FDBTransaction* tr ) { - return (FDBFuture*)( TXN(tr)->commit().extractPtr() ); -} - -extern "C" DLLEXPORT -fdb_error_t fdb_transaction_get_committed_version( FDBTransaction* tr, - int64_t* out_version ) -{ - CATCH_AND_RETURN( - *out_version = TXN(tr)->getCommittedVersion(); ); -} - -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* tr) { +extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* tr) { return (FDBFuture*)TXN(tr)->getApproximateSize().extractPtr(); } -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_get_versionstamp( FDBTransaction* tr ) -{ +extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_versionstamp(FDBTransaction* tr) { return (FDBFuture*)(TXN(tr)->getVersionstamp().extractPtr()); } -fdb_error_t fdb_transaction_set_option_impl( FDBTransaction* tr, - FDBTransactionOption option, - uint8_t const* value, - int value_length ) -{ +fdb_error_t fdb_transaction_set_option_impl(FDBTransaction* tr, + FDBTransactionOption option, + uint8_t const* value, + int value_length) { + CATCH_AND_RETURN(TXN(tr)->setOption((FDBTransactionOptions::Option)option, + value ? StringRef(value, value_length) : Optional());); +} + +void fdb_transaction_set_option_v13(FDBTransaction* tr, FDBTransactionOption option) { + fdb_transaction_set_option_impl(tr, option, NULL, 0); +} + +extern "C" DLLEXPORT FDBFuture* fdb_transaction_on_error(FDBTransaction* tr, fdb_error_t error) { + return (FDBFuture*)(TXN(tr)->onError(Error::fromUnvalidatedCode(error)).extractPtr()); +} + +extern "C" DLLEXPORT void fdb_transaction_reset(FDBTransaction* tr) { + CATCH_AND_DIE(TXN(tr)->reset();); +} + +extern "C" DLLEXPORT fdb_error_t fdb_transaction_add_conflict_range(FDBTransaction* tr, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length, + FDBConflictRangeType type) { CATCH_AND_RETURN( - TXN(tr)->setOption( (FDBTransactionOptions::Option)option, value ? StringRef( value, value_length ) : Optional() ); ); -} - -void fdb_transaction_set_option_v13( FDBTransaction* tr, - FDBTransactionOption option ) -{ - fdb_transaction_set_option_impl( tr, option, NULL, 0 ); -} - -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_on_error( FDBTransaction* tr, fdb_error_t error ) { - return (FDBFuture*)( TXN(tr)->onError( - Error::fromUnvalidatedCode( error ) ).extractPtr() ); -} - -extern "C" DLLEXPORT -void fdb_transaction_reset( FDBTransaction* tr ) { - CATCH_AND_DIE( TXN(tr)->reset(); ); -} - -extern "C" DLLEXPORT -fdb_error_t fdb_transaction_add_conflict_range( FDBTransaction*tr, uint8_t const* begin_key_name, - int begin_key_name_length, uint8_t const* end_key_name, - int end_key_name_length, FDBConflictRangeType type) { - CATCH_AND_RETURN( - KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); - if(type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_READ) - TXN(tr)->addReadConflictRange(range); - else if(type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_WRITE) - TXN(tr)->addWriteConflictRange(range); - else - return error_code_client_invalid_operation; - ); - + KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); + if (type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_READ) TXN(tr)->addReadConflictRange(range); + else if (type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_WRITE) TXN(tr)->addWriteConflictRange(range); + else return error_code_client_invalid_operation;); } #include "fdb_c_function_pointers.g.h" -#define FDB_API_CHANGED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) fdb_api_ptr_##func = (void*)&(func##_impl); +#define FDB_API_CHANGED(func, ver) \ + if (header_version < ver) \ + fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); \ + else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) \ + fdb_api_ptr_##func = (void*)&(func##_impl); -#define FDB_API_REMOVED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else fdb_api_ptr_##func = (void*)&fdb_api_ptr_removed; +#define FDB_API_REMOVED(func, ver) \ + if (header_version < ver) \ + fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); \ + else \ + fdb_api_ptr_##func = (void*)&fdb_api_ptr_removed; -extern "C" DLLEXPORT -fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version ) { +extern "C" DLLEXPORT fdb_error_t fdb_select_api_version_impl(int runtime_version, int header_version) { /* Can only call this once */ if (g_api_version != 0) return error_code_api_version_already_set; @@ -652,9 +610,7 @@ fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version if (runtime_version < 13) return error_code_api_version_not_supported; - RETURN_ON_ERROR( - API->selectApiVersion(runtime_version); - ); + RETURN_ON_ERROR(API->selectApiVersion(runtime_version);); g_api_version = runtime_version; @@ -662,52 +618,50 @@ fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version Error::init(); // Versioned API changes -- descending order by version (new changes at top) - // FDB_API_CHANGED( function, ver ) means there is a new implementation as of ver, and a function function_(ver-1) is the old implementation - // FDB_API_REMOVED( function, ver ) means the function was removed as of ver, and function_(ver-1) is the old implementation + // FDB_API_CHANGED( function, ver ) means there is a new implementation as of ver, and a function function_(ver-1) + // is the old implementation FDB_API_REMOVED( function, ver ) means the function was removed as of ver, and + // function_(ver-1) is the old implementation // - // WARNING: use caution when implementing removed functions by calling public API functions. This can lead to undesired behavior when - // using the multi-version API. Instead, it is better to have both the removed and public functions call an internal implementation function. - // See fdb_create_database_impl for an example. - FDB_API_REMOVED( fdb_future_get_version, 620 ); - FDB_API_REMOVED( fdb_create_cluster, 610 ); - FDB_API_REMOVED( fdb_cluster_create_database, 610 ); - FDB_API_REMOVED( fdb_cluster_set_option, 610 ); - FDB_API_REMOVED( fdb_cluster_destroy, 610 ); - FDB_API_REMOVED( fdb_future_get_cluster, 610 ); - FDB_API_REMOVED( fdb_future_get_database, 610 ); - FDB_API_CHANGED( fdb_future_get_error, 23 ); - FDB_API_REMOVED( fdb_future_is_error, 23 ); - FDB_API_CHANGED( fdb_future_get_keyvalue_array, 14 ); - FDB_API_CHANGED( fdb_transaction_get_key, 14 ); - FDB_API_CHANGED( fdb_transaction_get_range, 14 ); - FDB_API_REMOVED( fdb_transaction_get_range_selector, 14 ); - FDB_API_CHANGED( fdb_transaction_get, 14 ); - FDB_API_CHANGED( fdb_setup_network, 14 ); - FDB_API_CHANGED( fdb_transaction_set_option, 14 ); + // WARNING: use caution when implementing removed functions by calling public API functions. This can lead to + // undesired behavior when using the multi-version API. Instead, it is better to have both the removed and public + // functions call an internal implementation function. See fdb_create_database_impl for an example. + FDB_API_REMOVED(fdb_future_get_version, 620); + FDB_API_REMOVED(fdb_create_cluster, 610); + FDB_API_REMOVED(fdb_cluster_create_database, 610); + FDB_API_REMOVED(fdb_cluster_set_option, 610); + FDB_API_REMOVED(fdb_cluster_destroy, 610); + FDB_API_REMOVED(fdb_future_get_cluster, 610); + FDB_API_REMOVED(fdb_future_get_database, 610); + FDB_API_CHANGED(fdb_future_get_error, 23); + FDB_API_REMOVED(fdb_future_is_error, 23); + FDB_API_CHANGED(fdb_future_get_keyvalue_array, 14); + FDB_API_CHANGED(fdb_transaction_get_key, 14); + FDB_API_CHANGED(fdb_transaction_get_range, 14); + FDB_API_REMOVED(fdb_transaction_get_range_selector, 14); + FDB_API_CHANGED(fdb_transaction_get, 14); + FDB_API_CHANGED(fdb_setup_network, 14); + FDB_API_CHANGED(fdb_transaction_set_option, 14); /* End versioned API changes */ return error_code_success; } -extern "C" DLLEXPORT -int fdb_get_max_api_version() { +extern "C" DLLEXPORT int fdb_get_max_api_version() { return FDB_API_VERSION; } -extern "C" DLLEXPORT -const char* fdb_get_client_version() { +extern "C" DLLEXPORT const char* fdb_get_client_version() { return API->getClientVersion(); } #if defined(__APPLE__) #include -__attribute__((constructor)) -static void initialize() { - //OS X ld doesn't support -z nodelete, so we dlopen to increment the reference count of this module +__attribute__((constructor)) static void initialize() { + // OS X ld doesn't support -z nodelete, so we dlopen to increment the reference count of this module Dl_info info; int ret = dladdr((void*)&fdb_select_api_version_impl, &info); - if(!ret || !info.dli_fname) - return; //If we get here somehow, we face the risk of seg faults if somebody unloads our library + if (!ret || !info.dli_fname) + return; // If we get here somehow, we face the risk of seg faults if somebody unloads our library dlopen(info.dli_fname, RTLD_NOLOAD | RTLD_NODELETE); } diff --git a/bindings/c/foundationdb/ClientWorkload.h b/bindings/c/foundationdb/ClientWorkload.h index edaa35ccdf..0b785e1f31 100644 --- a/bindings/c/foundationdb/ClientWorkload.h +++ b/bindings/c/foundationdb/ClientWorkload.h @@ -44,7 +44,8 @@ enum class FDBSeverity { Debug, Info, Warn, WarnAlways, Error }; class FDBLogger { public: - virtual void trace(FDBSeverity sev, const std::string& name, + virtual void trace(FDBSeverity sev, + const std::string& name, const std::vector>& details) = 0; }; diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c old mode 100755 new mode 100644 index fce05ccf44..b1ed067e12 --- a/bindings/c/test/mako/mako.c +++ b/bindings/c/test/mako/mako.c @@ -22,1323 +22,1280 @@ #include "utils.h" #include "zipf.h" -#define check_fdb_error(_e) \ - do { \ - if (_e) { \ - fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, \ - fdb_get_error(_e)); \ - goto FDB_FAIL; \ - } \ - } while (0) +#define check_fdb_error(_e) \ + do { \ + if (_e) { \ + fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(_e)); \ + goto FDB_FAIL; \ + } \ + } while (0) -#define fdb_block_wait(_f) \ - do { \ - if ((fdb_future_block_until_ready(_f)) != 0) { \ - fprintf(stderr, "ERROR: fdb_future_block_until_ready failed at %s:%d\n", \ - __FILE__, __LINE__); \ - goto FDB_FAIL; \ - } \ - } while (0) +#define fdb_block_wait(_f) \ + do { \ + if ((fdb_future_block_until_ready(_f)) != 0) { \ + fprintf(stderr, "ERROR: fdb_future_block_until_ready failed at %s:%d\n", __FILE__, __LINE__); \ + goto FDB_FAIL; \ + } \ + } while (0) -fdb_error_t wait_future(FDBFuture *f) { - fdb_error_t err; +fdb_error_t wait_future(FDBFuture* f) { + fdb_error_t err; - err = fdb_future_block_until_ready(f); - if (err) { - return err; /* error from fdb_future_block_until_ready() */ - } - return fdb_future_get_error(f); + err = fdb_future_block_until_ready(f); + if (err) { + return err; /* error from fdb_future_block_until_ready() */ + } + return fdb_future_get_error(f); } +int commit_transaction(FDBTransaction* transaction, mako_stats_t* stats) { + FDBFuture* f; + fdb_error_t err = 0; + int retry = DEFAULT_RETRY_COUNT; -int commit_transaction(FDBTransaction *transaction, mako_stats_t *stats) { - FDBFuture *f; - fdb_error_t err = 0; - int retry = DEFAULT_RETRY_COUNT; + do { + f = fdb_transaction_commit(transaction); + err = wait_future(f); + fdb_future_destroy(f); + if (stats) { + if (err == 1020 /* not_committed */) + stats->conflicts++; + else { + stats->errors[OP_COMMIT]++; + } + } - do { - f = fdb_transaction_commit(transaction); - err = wait_future(f); - fdb_future_destroy(f); - if (stats) { - if (err == 1020 /* not_committed */) - stats->conflicts++; - else { - stats->errors[OP_COMMIT]++; - } - } - - if (err) { - fprintf(stderr, "ERROR: Error %d occured at fdb_transaction_commit\n", - err); - f = fdb_transaction_on_error(transaction, err); - err = wait_future(f); - fdb_future_destroy(f); - if (err) { - /* not retryable */ - fprintf(stderr, - "ERROR: fdb_transaction_on_error returned %d at %s:%d\n", - err, __FILE__, __LINE__); - break; - } - } else { - if (stats) - stats->ops[OP_COMMIT]++; - break; - } - } while (err && retry--); - - return err; + if (err) { + fprintf(stderr, "ERROR: Error %d occured at fdb_transaction_commit\n", err); + f = fdb_transaction_on_error(transaction, err); + err = wait_future(f); + fdb_future_destroy(f); + if (err) { + /* not retryable */ + fprintf(stderr, "ERROR: fdb_transaction_on_error returned %d at %s:%d\n", err, __FILE__, __LINE__); + break; + } + } else { + if (stats) + stats->ops[OP_COMMIT]++; + break; + } + } while (err && retry--); + + return err; } -void update_op_stats(struct timespec *start, struct timespec *end, int op, - mako_stats_t *stats) { - uint64_t latencyus; +void update_op_stats(struct timespec* start, struct timespec* end, int op, mako_stats_t* stats) { + uint64_t latencyus; - latencyus = (((uint64_t)end->tv_sec * 1000000000 + end->tv_nsec) - - ((uint64_t)start->tv_sec * 1000000000 + start->tv_nsec)) / - 1000; - stats->latency_samples[op]++; - stats->latency_us_total[op] += latencyus; - if (latencyus < stats->latency_us_min[op]) { - stats->latency_us_min[op] = latencyus; - } - if (latencyus > stats->latency_us_max[op]) { - stats->latency_us_max[op] = latencyus; - } + latencyus = (((uint64_t)end->tv_sec * 1000000000 + end->tv_nsec) - + ((uint64_t)start->tv_sec * 1000000000 + start->tv_nsec)) / + 1000; + stats->latency_samples[op]++; + stats->latency_us_total[op] += latencyus; + if (latencyus < stats->latency_us_min[op]) { + stats->latency_us_min[op] = latencyus; + } + if (latencyus > stats->latency_us_max[op]) { + stats->latency_us_max[op] = latencyus; + } } /* FDB network thread */ -void *fdb_network_thread(void *args) { - fdb_error_t err; +void* fdb_network_thread(void* args) { + fdb_error_t err; - if (((mako_args_t *)args)->verbose == VERBOSE_DEBUG) { - printf("DEBUG: fdb_network_thread started\n"); - } + if (((mako_args_t*)args)->verbose == VERBOSE_DEBUG) { + printf("DEBUG: fdb_network_thread started\n"); + } - err = fdb_run_network(); - if (err) { - fprintf(stderr, "ERROR: fdb_run_network: %s\n", fdb_get_error(err)); - } + err = fdb_run_network(); + if (err) { + fprintf(stderr, "ERROR: fdb_run_network: %s\n", fdb_get_error(err)); + } - return 0; + return 0; } /* cleanup database */ -int cleanup(FDBTransaction *transaction, mako_args_t *args) { - struct timespec timer_start, timer_end; - char beginstr[7]; - char endstr[7]; +int cleanup(FDBTransaction* transaction, mako_args_t* args) { + struct timespec timer_start, timer_end; + char beginstr[7]; + char endstr[7]; - strncpy(beginstr, "mako", 4); - beginstr[4] = 0x00; - strncpy(endstr, "mako", 4); - endstr[4] = 0xff; - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); - fdb_transaction_clear_range(transaction, (uint8_t *)beginstr, 5, - (uint8_t *)endstr, 5); - if (commit_transaction(transaction, NULL)) - goto FDB_FAIL; + strncpy(beginstr, "mako", 4); + beginstr[4] = 0x00; + strncpy(endstr, "mako", 4); + endstr[4] = 0xff; + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); + fdb_transaction_clear_range(transaction, (uint8_t*)beginstr, 5, (uint8_t*)endstr, 5); + if (commit_transaction(transaction, NULL)) + goto FDB_FAIL; - fdb_transaction_reset(transaction); - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_end); - if (args->verbose >= VERBOSE_DEFAULT) { - printf("INFO: Clear range: %6.3f sec\n", - ((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + - timer_end.tv_nsec - timer_start.tv_nsec) / - 1000000000); - } - return 0; + fdb_transaction_reset(transaction); + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_end); + if (args->verbose >= VERBOSE_DEFAULT) { + printf("INFO: Clear range: %6.3f sec\n", + ((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) / + 1000000000); + } + return 0; FDB_FAIL: - fprintf(stderr, "ERROR: FDB failure in cleanup()\n"); - return -1; + fprintf(stderr, "ERROR: FDB failure in cleanup()\n"); + return -1; } /* populate database */ -int populate(FDBTransaction *transaction, mako_args_t *args, int worker_id, - int thread_id, int thread_tps, mako_stats_t *stats) { - int i; - struct timespec timer_start, timer_end; - struct timespec timer_prev, timer_now; /* for throttling */ - struct timespec timer_per_xact_start, timer_per_xact_end; - char *keystr; - char *valstr; +int populate(FDBTransaction* transaction, + mako_args_t* args, + int worker_id, + int thread_id, + int thread_tps, + mako_stats_t* stats) { + int i; + struct timespec timer_start, timer_end; + struct timespec timer_prev, timer_now; /* for throttling */ + struct timespec timer_per_xact_start, timer_per_xact_end; + char* keystr; + char* valstr; - int begin = insert_begin(args->rows, worker_id, thread_id, - args->num_processes, args->num_threads); - int end = insert_end(args->rows, worker_id, thread_id, args->num_processes, - args->num_threads); - int xacts = 0; + int begin = insert_begin(args->rows, worker_id, thread_id, args->num_processes, args->num_threads); + int end = insert_end(args->rows, worker_id, thread_id, args->num_processes, args->num_threads); + int xacts = 0; - keystr = (char *)malloc(sizeof(char) * args->key_length + 1); - if (!keystr) - return -1; - valstr = (char *)malloc(sizeof(char) * args->value_length + 1); - if (!valstr) { - free(keystr); - return -1; - } + keystr = (char*)malloc(sizeof(char) * args->key_length + 1); + if (!keystr) + return -1; + valstr = (char*)malloc(sizeof(char) * args->value_length + 1); + if (!valstr) { + free(keystr); + return -1; + } - clock_gettime(CLOCK_MONOTONIC, &timer_start); - timer_prev.tv_sec = timer_start.tv_sec; - timer_prev.tv_nsec = timer_start.tv_nsec; - timer_per_xact_start.tv_sec = timer_start.tv_sec; - timer_per_xact_start.tv_nsec = timer_start.tv_nsec; + clock_gettime(CLOCK_MONOTONIC, &timer_start); + timer_prev.tv_sec = timer_start.tv_sec; + timer_prev.tv_nsec = timer_start.tv_nsec; + timer_per_xact_start.tv_sec = timer_start.tv_sec; + timer_per_xact_start.tv_nsec = timer_start.tv_nsec; - for (i = begin; i <= end; i++) { + for (i = begin; i <= end; i++) { - if ((thread_tps > 0) && (xacts >= thread_tps)) { - /* throttling is on */ + if ((thread_tps > 0) && (xacts >= thread_tps)) { + /* throttling is on */ - throttle: - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); - if ((timer_now.tv_sec > timer_prev.tv_sec + 1) || - ((timer_now.tv_sec == timer_prev.tv_sec + 1) && - (timer_now.tv_nsec > timer_prev.tv_nsec))) { - /* more than 1 second passed, no need to throttle */ - xacts = 0; - timer_prev.tv_sec = timer_now.tv_sec; - timer_prev.tv_nsec = timer_now.tv_nsec; - } else { - /* 1 second not passed, throttle */ - usleep(1000); /* sleep for 1ms */ - goto throttle; - } - } /* throttle */ + throttle: + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); + if ((timer_now.tv_sec > timer_prev.tv_sec + 1) || + ((timer_now.tv_sec == timer_prev.tv_sec + 1) && (timer_now.tv_nsec > timer_prev.tv_nsec))) { + /* more than 1 second passed, no need to throttle */ + xacts = 0; + timer_prev.tv_sec = timer_now.tv_sec; + timer_prev.tv_nsec = timer_now.tv_nsec; + } else { + /* 1 second not passed, throttle */ + usleep(1000); /* sleep for 1ms */ + goto throttle; + } + } /* throttle */ - /* sequential keys */ - genkey(keystr, i, args->rows, args->key_length + 1); - /* random values */ - randstr(valstr, args->value_length + 1); + /* sequential keys */ + genkey(keystr, i, args->rows, args->key_length + 1); + /* random values */ + randstr(valstr, args->value_length + 1); - /* insert (SET) */ - fdb_transaction_set(transaction, (uint8_t *)keystr, strlen(keystr), - (uint8_t *)valstr, strlen(valstr)); - stats->ops[OP_INSERT]++; + /* insert (SET) */ + fdb_transaction_set(transaction, (uint8_t*)keystr, strlen(keystr), (uint8_t*)valstr, strlen(valstr)); + stats->ops[OP_INSERT]++; - /* commit every 100 inserts (default) */ - if (i % args->txnspec.ops[OP_INSERT][OP_COUNT] == 0) { + /* commit every 100 inserts (default) */ + if (i % args->txnspec.ops[OP_INSERT][OP_COUNT] == 0) { - if (commit_transaction(transaction, NULL)) - goto FDB_FAIL; + if (commit_transaction(transaction, NULL)) + goto FDB_FAIL; - /* xact latency stats */ - clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); - update_op_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, - stats); - stats->ops[OP_COMMIT]++; - clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_start); + /* xact latency stats */ + clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); + update_op_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, stats); + stats->ops[OP_COMMIT]++; + clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_start); - fdb_transaction_reset(transaction); - stats->xacts++; - xacts++; /* for throttling */ - } - } + fdb_transaction_reset(transaction); + stats->xacts++; + xacts++; /* for throttling */ + } + } - if (commit_transaction(transaction, NULL)) - goto FDB_FAIL; + if (commit_transaction(transaction, NULL)) + goto FDB_FAIL; - /* xact latency stats */ - clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); - update_op_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, stats); + /* xact latency stats */ + clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); + update_op_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, stats); - clock_gettime(CLOCK_MONOTONIC, &timer_end); - stats->xacts++; + clock_gettime(CLOCK_MONOTONIC, &timer_end); + stats->xacts++; - if (args->verbose == VERBOSE_DEBUG) { - printf("DEBUG: Populated %d rows (%d-%d): %6.3f sec\n", end - begin, begin, - end, - ((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + - timer_end.tv_nsec - timer_start.tv_nsec) / - 1000000000); - } + if (args->verbose == VERBOSE_DEBUG) { + printf("DEBUG: Populated %d rows (%d-%d): %6.3f sec\n", + end - begin, + begin, + end, + ((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) / + 1000000000); + } - free(keystr); - free(valstr); - return 0; + free(keystr); + free(valstr); + return 0; FDB_FAIL: - if (keystr) - free(keystr); - if (valstr) - free(valstr); - fprintf(stderr, "ERROR: FDB failure in populate()\n"); - return -1; + if (keystr) + free(keystr); + if (valstr) + free(valstr); + fprintf(stderr, "ERROR: FDB failure in populate()\n"); + return -1; } -int64_t run_op_getreadversion(FDBTransaction *transaction) { - int64_t rv = 0; - FDBFuture *f; - fdb_error_t err; - int retry = DEFAULT_RETRY_COUNT; +int64_t run_op_getreadversion(FDBTransaction* transaction) { + int64_t rv = 0; + FDBFuture* f; + fdb_error_t err; + int retry = DEFAULT_RETRY_COUNT; - do { - f = fdb_transaction_get_read_version(transaction); - err = wait_future(f); + do { + f = fdb_transaction_get_read_version(transaction); + err = wait_future(f); - if (err) { - fdb_future_destroy(f); - f = fdb_transaction_on_error(transaction, err); - err = wait_future(f); - fdb_future_destroy(f); - if (err) { - /* not retryable */ - break; - } - } - } while (err && retry--); + if (err) { + fdb_future_destroy(f); + f = fdb_transaction_on_error(transaction, err); + err = wait_future(f); + fdb_future_destroy(f); + if (err) { + /* not retryable */ + break; + } + } + } while (err && retry--); - if (err) { - fprintf(stderr, "ERROR: fdb_transaction_get_read_version: %s\n", fdb_get_error(err)); - return -1; - } + if (err) { + fprintf(stderr, "ERROR: fdb_transaction_get_read_version: %s\n", fdb_get_error(err)); + return -1; + } #if FDB_API_VERSION < 620 - err = fdb_future_get_version(f, &rv); + err = fdb_future_get_version(f, &rv); #else - err = fdb_future_get_int64(f, &rv); + err = fdb_future_get_int64(f, &rv); #endif - if (err) { + if (err) { #if FDB_API_VERSION < 620 - fprintf(stderr, "ERROR: fdb_future_get_version: %s\n", fdb_get_error(err)); + fprintf(stderr, "ERROR: fdb_future_get_version: %s\n", fdb_get_error(err)); #else - fprintf(stderr, "ERROR: fdb_future_get_int64: %s\n", fdb_get_error(err)); + fprintf(stderr, "ERROR: fdb_future_get_int64: %s\n", fdb_get_error(err)); #endif - } - fdb_future_destroy(f); - return rv; + } + fdb_future_destroy(f); + return rv; } -int run_op_get(FDBTransaction *transaction, char *keystr, char *valstr, - int snapshot) { - FDBFuture *f; - int out_present; - char *val; - int vallen; - fdb_error_t err; - int retry = DEFAULT_RETRY_COUNT; +int run_op_get(FDBTransaction* transaction, char* keystr, char* valstr, int snapshot) { + FDBFuture* f; + int out_present; + char* val; + int vallen; + fdb_error_t err; + int retry = DEFAULT_RETRY_COUNT; - do { - f = fdb_transaction_get(transaction, (uint8_t *)keystr, strlen(keystr), - snapshot); - err = wait_future(f); + do { + f = fdb_transaction_get(transaction, (uint8_t*)keystr, strlen(keystr), snapshot); + err = wait_future(f); - if (err) { - fdb_future_destroy(f); - f = fdb_transaction_on_error(transaction, err); - err = wait_future(f); - fdb_future_destroy(f); - if (err) { - /* not retryable */ - break; - } - } - } while (err && retry--); + if (err) { + fdb_future_destroy(f); + f = fdb_transaction_on_error(transaction, err); + err = wait_future(f); + fdb_future_destroy(f); + if (err) { + /* not retryable */ + break; + } + } + } while (err && retry--); - if (err) { - fprintf(stderr, "ERROR: fdb_transaction_get: %s\n", fdb_get_error(err)); - return -1; - } + if (err) { + fprintf(stderr, "ERROR: fdb_transaction_get: %s\n", fdb_get_error(err)); + return -1; + } - err = fdb_future_get_value(f, &out_present, (const uint8_t **)&val, &vallen); - fdb_future_destroy(f); - if (err || !out_present) { - /* error or value not present */ - return -1; - } - strncpy(valstr, val, vallen); - valstr[vallen] = '\0'; - return 0; + err = fdb_future_get_value(f, &out_present, (const uint8_t**)&val, &vallen); + fdb_future_destroy(f); + if (err || !out_present) { + /* error or value not present */ + return -1; + } + strncpy(valstr, val, vallen); + valstr[vallen] = '\0'; + return 0; } -int run_op_getrange(FDBTransaction *transaction, char *keystr, char *keystr2, - char *valstr, int snapshot, int reverse) { - FDBFuture *f; - fdb_error_t err; - FDBKeyValue const *out_kv; - int out_count; - int out_more; - int retry = DEFAULT_RETRY_COUNT; +int run_op_getrange(FDBTransaction* transaction, char* keystr, char* keystr2, char* valstr, int snapshot, int reverse) { + FDBFuture* f; + fdb_error_t err; + FDBKeyValue const* out_kv; + int out_count; + int out_more; + int retry = DEFAULT_RETRY_COUNT; - do { - f = fdb_transaction_get_range( - transaction, - FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t *)keystr, strlen(keystr)), - FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t *)keystr2, strlen(keystr2)) + 1, - 0 /* limit */, 0 /* target_bytes */, - FDB_STREAMING_MODE_WANT_ALL /* FDBStreamingMode */, 0 /* iteration */, - snapshot, reverse /* reverse */); - err = wait_future(f); + do { + f = fdb_transaction_get_range(transaction, + FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)), + FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1, + 0 /* limit */, + 0 /* target_bytes */, + FDB_STREAMING_MODE_WANT_ALL /* FDBStreamingMode */, + 0 /* iteration */, + snapshot, + reverse /* reverse */); + err = wait_future(f); - if (err) { - fdb_future_destroy(f); - f = fdb_transaction_on_error(transaction, err); - err = wait_future(f); - fdb_future_destroy(f); - if (err) { - /* not retryable */ - break; - } - } - } while (err && retry--); + if (err) { + fdb_future_destroy(f); + f = fdb_transaction_on_error(transaction, err); + err = wait_future(f); + fdb_future_destroy(f); + if (err) { + /* not retryable */ + break; + } + } + } while (err && retry--); - if (err) { - fprintf(stderr, "ERROR: fdb_transaction_get_range: %s\n", fdb_get_error(err)); - return -1; - } + if (err) { + fprintf(stderr, "ERROR: fdb_transaction_get_range: %s\n", fdb_get_error(err)); + return -1; + } - err = fdb_future_get_keyvalue_array(f, &out_kv, &out_count, &out_more); - if (err) { - fprintf(stderr, "ERROR: fdb_future_get_keyvalue_array: %s\n", - fdb_get_error(err)); - fdb_future_destroy(f); - return -1; - } - fdb_future_destroy(f); - return 0; + err = fdb_future_get_keyvalue_array(f, &out_kv, &out_count, &out_more); + if (err) { + fprintf(stderr, "ERROR: fdb_future_get_keyvalue_array: %s\n", fdb_get_error(err)); + fdb_future_destroy(f); + return -1; + } + fdb_future_destroy(f); + return 0; } -int run_op_update(FDBTransaction *transaction, char *keystr, char *valstr) { - FDBFuture *f; - int out_present; - char *val; - int vallen; - fdb_error_t err; - int retry = DEFAULT_RETRY_COUNT; +int run_op_update(FDBTransaction* transaction, char* keystr, char* valstr) { + FDBFuture* f; + int out_present; + char* val; + int vallen; + fdb_error_t err; + int retry = DEFAULT_RETRY_COUNT; - /* GET first */ - do { - f = fdb_transaction_get(transaction, (uint8_t *)keystr, strlen(keystr), 0); - err = wait_future(f); + /* GET first */ + do { + f = fdb_transaction_get(transaction, (uint8_t*)keystr, strlen(keystr), 0); + err = wait_future(f); - if (err) { - fdb_future_destroy(f); - f = fdb_transaction_on_error(transaction, err); - err = wait_future(f); - fdb_future_destroy(f); - if (err) { - /* not retryable */ - break; - } - } - } while (err && retry--); + if (err) { + fdb_future_destroy(f); + f = fdb_transaction_on_error(transaction, err); + err = wait_future(f); + fdb_future_destroy(f); + if (err) { + /* not retryable */ + break; + } + } + } while (err && retry--); - if (err) { - fprintf(stderr, "ERROR: fdb_transaction_get: %s\n", fdb_get_error(err)); - return -1; - } + if (err) { + fprintf(stderr, "ERROR: fdb_transaction_get: %s\n", fdb_get_error(err)); + return -1; + } - err = fdb_future_get_value(f, &out_present, (const uint8_t **)&val, &vallen); - fdb_future_destroy(f); - if (err || !out_present) { - /* error or value not present */ - return -1; - } + err = fdb_future_get_value(f, &out_present, (const uint8_t**)&val, &vallen); + fdb_future_destroy(f); + if (err || !out_present) { + /* error or value not present */ + return -1; + } - /* Update Value (SET) */ - fdb_transaction_set(transaction, (uint8_t *)keystr, strlen(keystr), - (uint8_t *)valstr, strlen(valstr)); - return 0; + /* Update Value (SET) */ + fdb_transaction_set(transaction, (uint8_t*)keystr, strlen(keystr), (uint8_t*)valstr, strlen(valstr)); + return 0; } -int run_op_insert(FDBTransaction *transaction, char *keystr, char *valstr) { - fdb_transaction_set(transaction, (uint8_t *)keystr, strlen(keystr), - (uint8_t *)valstr, strlen(valstr)); - return 0; +int run_op_insert(FDBTransaction* transaction, char* keystr, char* valstr) { + fdb_transaction_set(transaction, (uint8_t*)keystr, strlen(keystr), (uint8_t*)valstr, strlen(valstr)); + return 0; } -int run_op_clear(FDBTransaction *transaction, char *keystr) { - fdb_transaction_clear(transaction, (uint8_t *)keystr, strlen(keystr)); - return 0; +int run_op_clear(FDBTransaction* transaction, char* keystr) { + fdb_transaction_clear(transaction, (uint8_t*)keystr, strlen(keystr)); + return 0; } -int run_op_clearrange(FDBTransaction *transaction, char *keystr, - char *keystr2) { - fdb_transaction_clear_range(transaction, (uint8_t *)keystr, strlen(keystr), - (uint8_t *)keystr2, strlen(keystr2)); - return 0; +int run_op_clearrange(FDBTransaction* transaction, char* keystr, char* keystr2) { + fdb_transaction_clear_range(transaction, (uint8_t*)keystr, strlen(keystr), (uint8_t*)keystr2, strlen(keystr2)); + return 0; } /* run one transaction */ -int run_transaction(FDBTransaction *transaction, mako_args_t *args, - mako_stats_t *stats, char *keystr, char *keystr2, - char *valstr) { - int i; - int count; - int rc; - struct timespec timer_start, timer_end; - struct timespec timer_per_xact_start, timer_per_xact_end; - int docommit = 0; - int keynum; - int keyend; - int64_t readversion; - int randstrlen; - int rangei; +int run_transaction(FDBTransaction* transaction, + mako_args_t* args, + mako_stats_t* stats, + char* keystr, + char* keystr2, + char* valstr) { + int i; + int count; + int rc; + struct timespec timer_start, timer_end; + struct timespec timer_per_xact_start, timer_per_xact_end; + int docommit = 0; + int keynum; + int keyend; + int64_t readversion; + int randstrlen; + int rangei; - /* transaction */ - clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_start); - for (i = 0; i < MAX_OP; i++) { + /* transaction */ + clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_start); + for (i = 0; i < MAX_OP; i++) { - if ((args->txnspec.ops[i][OP_COUNT] > 0) && (i != OP_COMMIT)) { - for (count = 0; count < args->txnspec.ops[i][OP_COUNT]; count++) { + if ((args->txnspec.ops[i][OP_COUNT] > 0) && (i != OP_COMMIT)) { + for (count = 0; count < args->txnspec.ops[i][OP_COUNT]; count++) { - /* pick a random key(s) */ - if (args->zipf) { - keynum = zipfian_next(); - } else { - keynum = urand(0, args->rows - 1); - } - genkey(keystr, keynum, args->rows, args->key_length + 1); + /* pick a random key(s) */ + if (args->zipf) { + keynum = zipfian_next(); + } else { + keynum = urand(0, args->rows - 1); + } + genkey(keystr, keynum, args->rows, args->key_length + 1); - /* range */ - if (args->txnspec.ops[i][OP_RANGE] > 0) { - keyend = keynum + args->txnspec.ops[i][OP_RANGE] - 1; /* inclusive */ - if (keyend > args->rows - 1) { - keyend = args->rows - 1; - } - genkey(keystr2, keyend, args->rows, args->key_length + 1); - } + /* range */ + if (args->txnspec.ops[i][OP_RANGE] > 0) { + keyend = keynum + args->txnspec.ops[i][OP_RANGE] - 1; /* inclusive */ + if (keyend > args->rows - 1) { + keyend = args->rows - 1; + } + genkey(keystr2, keyend, args->rows, args->key_length + 1); + } - if (stats->xacts % args->sampling == 0) { - /* per op latency */ - clock_gettime(CLOCK_MONOTONIC, &timer_start); - } + if (stats->xacts % args->sampling == 0) { + /* per op latency */ + clock_gettime(CLOCK_MONOTONIC, &timer_start); + } - switch (i) { - case OP_GETREADVERSION: - readversion = run_op_getreadversion(transaction); - if (!readversion) { - rc = -1; - } - break; - case OP_GET: - rc = run_op_get(transaction, keystr, valstr, 0); - break; - case OP_GETRANGE: - rc = run_op_getrange(transaction, keystr, keystr2, valstr, 0, - args->txnspec.ops[i][OP_REVERSE]); - break; - case OP_SGET: - rc = run_op_get(transaction, keystr, valstr, 1); - break; - case OP_SGETRANGE: - rc = run_op_getrange(transaction, keystr, keystr2, valstr, 1, - args->txnspec.ops[i][OP_REVERSE]); - break; - case OP_UPDATE: - randstr(valstr, args->value_length + 1); - rc = run_op_update(transaction, keystr, valstr); - docommit = 1; - break; - case OP_INSERT: - randstr(keystr + KEYPREFIXLEN, args->key_length - KEYPREFIXLEN + - 1); /* make it (almost) unique */ - randstr(valstr, args->value_length + 1); - rc = run_op_insert(transaction, keystr, valstr); - docommit = 1; - break; - case OP_INSERTRANGE: - randstrlen = args->key_length - KEYPREFIXLEN - - digits(args->txnspec.ops[i][OP_RANGE]); - randstr(keystr + KEYPREFIXLEN, - randstrlen + 1); /* make it (almost) unique */ - randstr(valstr, args->value_length + 1); - for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) { - sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", - digits(args->txnspec.ops[i][OP_RANGE]), rangei); - rc = run_op_insert(transaction, keystr, valstr); - if (rc != 0) - break; - } - docommit = 1; - break; - case OP_CLEAR: - rc = run_op_clear(transaction, keystr); - docommit = 1; - break; - case OP_SETCLEAR: - randstr(keystr + KEYPREFIXLEN, args->key_length - KEYPREFIXLEN + - 1); /* make it (almost) unique */ - randstr(valstr, args->value_length + 1); - rc = run_op_insert(transaction, keystr, valstr); - if (rc == 0) { - /* commit insert so mutation goes to storage */ - if (commit_transaction(transaction, stats) == 0) { - clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); - update_op_stats(&timer_per_xact_start, &timer_per_xact_end, - OP_COMMIT, stats); - } - fdb_transaction_reset(transaction); - rc = run_op_clear(transaction, keystr); - } - docommit = 1; - break; - case OP_CLEARRANGE: - rc = run_op_clearrange(transaction, keystr, keystr2); - docommit = 1; - break; - case OP_SETCLEARRANGE: - randstrlen = args->key_length - KEYPREFIXLEN - - digits(args->txnspec.ops[i][OP_RANGE]); - randstr(keystr + KEYPREFIXLEN, - randstrlen + 1); /* make it (almost) unique */ - randstr(valstr, args->value_length + 1); - for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) { - sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", - digits(args->txnspec.ops[i][OP_RANGE]), rangei); - if (rangei == 0) { - strcpy(keystr2, keystr); - keystr2[strlen(keystr)] = '\0'; - } - rc = run_op_insert(transaction, keystr, valstr); - if (rc != 0) { - /* rollback not necessary, transaction will be reset */ - break; - } - } - /* commit inserts so mutation goes to storage */ - if (commit_transaction(transaction, stats) == 0) { - clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); - update_op_stats(&timer_per_xact_start, &timer_per_xact_end, - OP_COMMIT, stats); - } - fdb_transaction_reset(transaction); - rc = run_op_clearrange(transaction, keystr2, keystr); - docommit = 1; - break; - default: - fprintf(stderr, "ERROR: Unknown Operation %d\n", i); - break; - } + switch (i) { + case OP_GETREADVERSION: + readversion = run_op_getreadversion(transaction); + if (!readversion) { + rc = -1; + } + break; + case OP_GET: + rc = run_op_get(transaction, keystr, valstr, 0); + break; + case OP_GETRANGE: + rc = run_op_getrange(transaction, keystr, keystr2, valstr, 0, args->txnspec.ops[i][OP_REVERSE]); + break; + case OP_SGET: + rc = run_op_get(transaction, keystr, valstr, 1); + break; + case OP_SGETRANGE: + rc = run_op_getrange(transaction, keystr, keystr2, valstr, 1, args->txnspec.ops[i][OP_REVERSE]); + break; + case OP_UPDATE: + randstr(valstr, args->value_length + 1); + rc = run_op_update(transaction, keystr, valstr); + docommit = 1; + break; + case OP_INSERT: + randstr(keystr + KEYPREFIXLEN, args->key_length - KEYPREFIXLEN + 1); /* make it (almost) unique */ + randstr(valstr, args->value_length + 1); + rc = run_op_insert(transaction, keystr, valstr); + docommit = 1; + break; + case OP_INSERTRANGE: + randstrlen = args->key_length - KEYPREFIXLEN - digits(args->txnspec.ops[i][OP_RANGE]); + randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */ + randstr(valstr, args->value_length + 1); + for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) { + sprintf(keystr + KEYPREFIXLEN + randstrlen, + "%0.*d", + digits(args->txnspec.ops[i][OP_RANGE]), + rangei); + rc = run_op_insert(transaction, keystr, valstr); + if (rc != 0) + break; + } + docommit = 1; + break; + case OP_CLEAR: + rc = run_op_clear(transaction, keystr); + docommit = 1; + break; + case OP_SETCLEAR: + randstr(keystr + KEYPREFIXLEN, args->key_length - KEYPREFIXLEN + 1); /* make it (almost) unique */ + randstr(valstr, args->value_length + 1); + rc = run_op_insert(transaction, keystr, valstr); + if (rc == 0) { + /* commit insert so mutation goes to storage */ + if (commit_transaction(transaction, stats) == 0) { + clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); + update_op_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, stats); + } + fdb_transaction_reset(transaction); + rc = run_op_clear(transaction, keystr); + } + docommit = 1; + break; + case OP_CLEARRANGE: + rc = run_op_clearrange(transaction, keystr, keystr2); + docommit = 1; + break; + case OP_SETCLEARRANGE: + randstrlen = args->key_length - KEYPREFIXLEN - digits(args->txnspec.ops[i][OP_RANGE]); + randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */ + randstr(valstr, args->value_length + 1); + for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) { + sprintf(keystr + KEYPREFIXLEN + randstrlen, + "%0.*d", + digits(args->txnspec.ops[i][OP_RANGE]), + rangei); + if (rangei == 0) { + strcpy(keystr2, keystr); + keystr2[strlen(keystr)] = '\0'; + } + rc = run_op_insert(transaction, keystr, valstr); + if (rc != 0) { + /* rollback not necessary, transaction will be reset */ + break; + } + } + /* commit inserts so mutation goes to storage */ + if (commit_transaction(transaction, stats) == 0) { + clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); + update_op_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, stats); + } + fdb_transaction_reset(transaction); + rc = run_op_clearrange(transaction, keystr2, keystr); + docommit = 1; + break; + default: + fprintf(stderr, "ERROR: Unknown Operation %d\n", i); + break; + } - if (stats->xacts % args->sampling == 0) { - clock_gettime(CLOCK_MONOTONIC, &timer_end); - if (rc == 0) { - /* per op latency */ - update_op_stats(&timer_start, &timer_end, i, stats); - } - } + if (stats->xacts % args->sampling == 0) { + clock_gettime(CLOCK_MONOTONIC, &timer_end); + if (rc == 0) { + /* per op latency */ + update_op_stats(&timer_start, &timer_end, i, stats); + } + } - /* check rc */ - if (rc != 0) { - stats->errors[i]++; - } else { - stats->ops[i]++; - } - } - } - } - if (docommit | args->commit_get) { - if (commit_transaction(transaction, stats) == 0) { - clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); - update_op_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, - stats); - } - } - stats->xacts++; + /* check rc */ + if (rc != 0) { + stats->errors[i]++; + } else { + stats->ops[i]++; + } + } + } + } + if (docommit | args->commit_get) { + if (commit_transaction(transaction, stats) == 0) { + clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); + update_op_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, stats); + } + } + stats->xacts++; - fdb_transaction_reset(transaction); - return 0; + fdb_transaction_reset(transaction); + return 0; } -int run_workload(FDBTransaction *transaction, mako_args_t *args, int thread_tps, - int thread_iters, volatile int *signal, mako_stats_t *stats) { - int xacts = 0; - int rc = 0; - struct timespec timer_prev, timer_now; - char *keystr; - char *keystr2; - char *valstr; +int run_workload(FDBTransaction* transaction, + mako_args_t* args, + int thread_tps, + int thread_iters, + volatile int* signal, + mako_stats_t* stats) { + int xacts = 0; + int rc = 0; + struct timespec timer_prev, timer_now; + char* keystr; + char* keystr2; + char* valstr; - if (thread_tps < 0) - return 0; + if (thread_tps < 0) + return 0; - keystr = (char *)malloc(sizeof(char) * args->key_length + 1); - if (!keystr) - return -1; - keystr2 = (char *)malloc(sizeof(char) * args->key_length + 1); - if (!keystr2) { - free(keystr); - return -1; - } - valstr = (char *)malloc(sizeof(char) * args->value_length + 1); - if (!valstr) { - free(keystr); - free(keystr2); - return -1; - } + keystr = (char*)malloc(sizeof(char) * args->key_length + 1); + if (!keystr) + return -1; + keystr2 = (char*)malloc(sizeof(char) * args->key_length + 1); + if (!keystr2) { + free(keystr); + return -1; + } + valstr = (char*)malloc(sizeof(char) * args->value_length + 1); + if (!valstr) { + free(keystr); + free(keystr2); + return -1; + } - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_prev); + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_prev); - while (1) { + while (1) { - if ((thread_tps > 0) && (xacts >= thread_tps)) { - /* throttling is on */ + if ((thread_tps > 0) && (xacts >= thread_tps)) { + /* throttling is on */ - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); - if ((timer_now.tv_sec > timer_prev.tv_sec + 1) || - ((timer_now.tv_sec == timer_prev.tv_sec + 1) && - (timer_now.tv_nsec > timer_prev.tv_nsec))) { - /* more than 1 second passed, no need to throttle */ - xacts = 0; - timer_prev.tv_sec = timer_now.tv_sec; - timer_prev.tv_nsec = timer_now.tv_nsec; - } else { - /* 1 second not passed, throttle */ - usleep(1000); - continue; - } - } + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); + if ((timer_now.tv_sec > timer_prev.tv_sec + 1) || + ((timer_now.tv_sec == timer_prev.tv_sec + 1) && (timer_now.tv_nsec > timer_prev.tv_nsec))) { + /* more than 1 second passed, no need to throttle */ + xacts = 0; + timer_prev.tv_sec = timer_now.tv_sec; + timer_prev.tv_nsec = timer_now.tv_nsec; + } else { + /* 1 second not passed, throttle */ + usleep(1000); + continue; + } + } - rc = run_transaction(transaction, args, stats, keystr, keystr2, valstr); - if (rc) { - /* should never get here */ - fprintf(stderr, "ERROR: run_transaction failed (%d)\n", rc); - } + rc = run_transaction(transaction, args, stats, keystr, keystr2, valstr); + if (rc) { + /* should never get here */ + fprintf(stderr, "ERROR: run_transaction failed (%d)\n", rc); + } - if (thread_iters > 0) { - if (thread_iters == xacts) { - break; - } - } else if (*signal == SIGNAL_RED) { - break; - } - xacts++; - } - free(keystr); - free(keystr2); - free(valstr); + if (thread_iters > 0) { + if (thread_iters == xacts) { + break; + } + } else if (*signal == SIGNAL_RED) { + break; + } + xacts++; + } + free(keystr); + free(keystr2); + free(valstr); - return rc; + return rc; } /* mako worker thread */ -void *worker_thread(void *thread_args) { - int worker_id = ((thread_args_t *)thread_args)->process->worker_id; - int thread_id = ((thread_args_t *)thread_args)->thread_id; - mako_args_t *args = ((thread_args_t *)thread_args)->process->args; - FDBDatabase *database = ((thread_args_t *)thread_args)->process->database; - fdb_error_t err; - int rc; - FDBTransaction *transaction; - int thread_tps; - int thread_iters = 0; - int op; - volatile int *signal = &((thread_args_t *)thread_args)->process->shm->signal; - volatile int *readycount = - &((thread_args_t *)thread_args)->process->shm->readycount; - mako_stats_t *stats = - (void *)((thread_args_t *)thread_args)->process->shm + - sizeof(mako_shmhdr_t) /* skip header */ - + (sizeof(mako_stats_t) * (worker_id * args->num_threads + thread_id)); +void* worker_thread(void* thread_args) { + int worker_id = ((thread_args_t*)thread_args)->process->worker_id; + int thread_id = ((thread_args_t*)thread_args)->thread_id; + mako_args_t* args = ((thread_args_t*)thread_args)->process->args; + FDBDatabase* database = ((thread_args_t*)thread_args)->process->database; + fdb_error_t err; + int rc; + FDBTransaction* transaction; + int thread_tps; + int thread_iters = 0; + int op; + volatile int* signal = &((thread_args_t*)thread_args)->process->shm->signal; + volatile int* readycount = &((thread_args_t*)thread_args)->process->shm->readycount; + mako_stats_t* stats = (void*)((thread_args_t*)thread_args)->process->shm + sizeof(mako_shmhdr_t) /* skip header */ + + (sizeof(mako_stats_t) * (worker_id * args->num_threads + thread_id)); - /* init latency */ - for (op = 0; op < MAX_OP; op++) { - stats->latency_us_min[op] = 0xFFFFFFFFFFFFFFFF; /* uint64_t */ - stats->latency_us_max[op] = 0; - stats->latency_us_total[op] = 0; - } + /* init latency */ + for (op = 0; op < MAX_OP; op++) { + stats->latency_us_min[op] = 0xFFFFFFFFFFFFFFFF; /* uint64_t */ + stats->latency_us_max[op] = 0; + stats->latency_us_total[op] = 0; + } - if (args->verbose == VERBOSE_DEBUG) { - printf("DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n", worker_id, - args->num_processes, thread_id, args->num_threads, - (unsigned int)pthread_self()); - } + if (args->verbose == VERBOSE_DEBUG) { + printf("DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n", + worker_id, + args->num_processes, + thread_id, + args->num_threads, + (unsigned int)pthread_self()); + } - if (args->tps) { - thread_tps = compute_thread_tps(args->tps, worker_id, thread_id, - args->num_processes, args->num_threads); - } + if (args->tps) { + thread_tps = compute_thread_tps(args->tps, worker_id, thread_id, args->num_processes, args->num_threads); + } - if (args->iteration) { - thread_iters = compute_thread_iters(args->iteration, worker_id, thread_id, - args->num_processes, args->num_threads); - } + if (args->iteration) { + thread_iters = + compute_thread_iters(args->iteration, worker_id, thread_id, args->num_processes, args->num_threads); + } - /* create my own transaction object */ - err = fdb_database_create_transaction(database, &transaction); - check_fdb_error(err); + /* create my own transaction object */ + err = fdb_database_create_transaction(database, &transaction); + check_fdb_error(err); - /* i'm ready */ - __sync_fetch_and_add(readycount, 1); - while (*signal == SIGNAL_OFF) { - usleep(10000); /* 10ms */ - } + /* i'm ready */ + __sync_fetch_and_add(readycount, 1); + while (*signal == SIGNAL_OFF) { + usleep(10000); /* 10ms */ + } - /* clean */ - if (args->mode == MODE_CLEAN) { - rc = cleanup(transaction, args); - if (rc < 0) { - fprintf(stderr, "ERROR: cleanup failed\n"); - } - } + /* clean */ + if (args->mode == MODE_CLEAN) { + rc = cleanup(transaction, args); + if (rc < 0) { + fprintf(stderr, "ERROR: cleanup failed\n"); + } + } - /* build/popualte */ - else if (args->mode == MODE_BUILD) { - rc = populate(transaction, args, worker_id, thread_id, thread_tps, stats); - if (rc < 0) { - fprintf(stderr, "ERROR: populate failed\n"); - } - } + /* build/popualte */ + else if (args->mode == MODE_BUILD) { + rc = populate(transaction, args, worker_id, thread_id, thread_tps, stats); + if (rc < 0) { + fprintf(stderr, "ERROR: populate failed\n"); + } + } - /* run the workload */ - else if (args->mode == MODE_RUN) { - rc = run_workload(transaction, args, thread_tps, thread_iters, signal, - stats); - if (rc < 0) { - fprintf(stderr, "ERROR: run_workload failed\n"); - } - } + /* run the workload */ + else if (args->mode == MODE_RUN) { + rc = run_workload(transaction, args, thread_tps, thread_iters, signal, stats); + if (rc < 0) { + fprintf(stderr, "ERROR: run_workload failed\n"); + } + } - /* fall through */ + /* fall through */ FDB_FAIL: - fdb_transaction_destroy(transaction); - pthread_exit(0); + fdb_transaction_destroy(transaction); + pthread_exit(0); } /* mako worker process */ -int worker_process_main(mako_args_t *args, int worker_id, mako_shmhdr_t *shm) { - int i; - pthread_t - network_thread; /* handle for thread which invoked fdb_run_network() */ - pthread_t *worker_threads = NULL; +int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) { + int i; + pthread_t network_thread; /* handle for thread which invoked fdb_run_network() */ + pthread_t* worker_threads = NULL; #if FDB_API_VERSION < 610 - FDBCluster *cluster; + FDBCluster* cluster; #endif - process_info_t process; - thread_args_t *thread_args = NULL; - int rc; - fdb_error_t err; + process_info_t process; + thread_args_t* thread_args = NULL; + int rc; + fdb_error_t err; - process.worker_id = worker_id; - process.args = args; - process.shm = (mako_shmhdr_t *)shm; + process.worker_id = worker_id; + process.args = args; + process.shm = (mako_shmhdr_t*)shm; - if (args->verbose == VERBOSE_DEBUG) { - printf("DEBUG: worker %d started\n", worker_id); - } + if (args->verbose == VERBOSE_DEBUG) { + printf("DEBUG: worker %d started\n", worker_id); + } - /* Everything starts from here */ - /* Let's use the maximum API version */ - // fprintf(stderr, "fdb_get_max_api_version: %d\n", - // fdb_get_max_api_version()); - err = fdb_select_api_version(fdb_get_max_api_version()); - if (err) { - fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err)); - return -1; - } + /* Everything starts from here */ + /* Let's use the maximum API version */ + // fprintf(stderr, "fdb_get_max_api_version: %d\n", + // fdb_get_max_api_version()); + err = fdb_select_api_version(fdb_get_max_api_version()); + if (err) { + fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err)); + return -1; + } - /* enable flatbuffers if specified */ - if (args->flatbuffers) { + /* enable flatbuffers if specified */ + if (args->flatbuffers) { #ifdef FDB_NET_OPTION_USE_FLATBUFFERS - if (args->verbose >= VERBOSE_DEBUG) { - printf("DEBUG: Using flatbuffers\n"); - } - err = - fdb_network_set_option(FDB_NET_OPTION_USE_FLATBUFFERS, - (uint8_t *)&args->flatbuffers, sizeof(uint8_t)); - if (err) { - fprintf( - stderr, - "ERROR: fdb_network_set_option(FDB_NET_OPTION_USE_FLATBUFFERS): %s\n", - fdb_get_error(err)); - } + if (args->verbose >= VERBOSE_DEBUG) { + printf("DEBUG: Using flatbuffers\n"); + } + err = fdb_network_set_option(FDB_NET_OPTION_USE_FLATBUFFERS, (uint8_t*)&args->flatbuffers, sizeof(uint8_t)); + if (err) { + fprintf(stderr, "ERROR: fdb_network_set_option(FDB_NET_OPTION_USE_FLATBUFFERS): %s\n", fdb_get_error(err)); + } #else - if (args->verbose >= VERBOSE_DEFAULT) { - printf("INFO: flatbuffers is not supported in FDB API version %d\n", - FDB_API_VERSION); - } + if (args->verbose >= VERBOSE_DEFAULT) { + printf("INFO: flatbuffers is not supported in FDB API version %d\n", FDB_API_VERSION); + } #endif - } + } - /* enable tracing if specified */ - if (args->trace) { - if (args->verbose >= VERBOSE_DEBUG) { - printf("DEBUG: Enable Tracing (%s)\n", (args->tracepath[0] == '\0') - ? "current directory" - : args->tracepath); - } - err = fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE, - (uint8_t *)args->tracepath, - strlen(args->tracepath)); - if (err) { - fprintf( - stderr, - "ERROR: fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE): %s\n", - fdb_get_error(err)); - } - } + /* enable tracing if specified */ + if (args->trace) { + if (args->verbose >= VERBOSE_DEBUG) { + printf("DEBUG: Enable Tracing (%s)\n", + (args->tracepath[0] == '\0') ? "current directory" : args->tracepath); + } + err = fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE, (uint8_t*)args->tracepath, strlen(args->tracepath)); + if (err) { + fprintf(stderr, "ERROR: fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE): %s\n", fdb_get_error(err)); + } + } - /* enable knobs if specified */ - if (args->knobs[0] != '\0') { - if (args->verbose >= VERBOSE_DEBUG) { - printf("DEBUG: Setting client konbs: %s\n", args->knobs); - } - err = fdb_network_set_option(FDB_NET_OPTION_KNOB, (uint8_t *)args->knobs, - strlen(args->knobs)); - if (err) { - fprintf(stderr, "ERROR: fdb_network_set_option: %s\n", - fdb_get_error(err)); - } - } + /* enable knobs if specified */ + if (args->knobs[0] != '\0') { + if (args->verbose >= VERBOSE_DEBUG) { + printf("DEBUG: Setting client konbs: %s\n", args->knobs); + } + err = fdb_network_set_option(FDB_NET_OPTION_KNOB, (uint8_t*)args->knobs, strlen(args->knobs)); + if (err) { + fprintf(stderr, "ERROR: fdb_network_set_option: %s\n", fdb_get_error(err)); + } + } - /* Network thread must be setup before doing anything */ - if (args->verbose == VERBOSE_DEBUG) { - printf("DEBUG: fdb_setup_network\n"); - } - err = fdb_setup_network(); - if (err) { - fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err)); - return -1; - } + /* Network thread must be setup before doing anything */ + if (args->verbose == VERBOSE_DEBUG) { + printf("DEBUG: fdb_setup_network\n"); + } + err = fdb_setup_network(); + if (err) { + fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err)); + return -1; + } - /* Each worker process will have its own network thread */ - if (args->verbose >= VERBOSE_DEBUG) { - printf("DEBUG: creating network thread\n"); - } - rc = pthread_create(&network_thread, NULL, fdb_network_thread, (void *)args); - if (rc != 0) { - fprintf(stderr, "ERROR: Cannot create a network thread\n"); - return -1; - } + /* Each worker process will have its own network thread */ + if (args->verbose >= VERBOSE_DEBUG) { + printf("DEBUG: creating network thread\n"); + } + rc = pthread_create(&network_thread, NULL, fdb_network_thread, (void*)args); + if (rc != 0) { + fprintf(stderr, "ERROR: Cannot create a network thread\n"); + return -1; + } - /*** let's party! ***/ + /*** let's party! ***/ - /* set up cluster and datbase for workder threads */ + /* set up cluster and datbase for workder threads */ #if FDB_API_VERSION < 610 - /* cluster */ - f = fdb_create_cluster(args->cluster_file); - fdb_block_wait(f); - err = fdb_future_get_cluster(f, &cluster); - check_fdb_error(err); - fdb_future_destroy(f); + /* cluster */ + f = fdb_create_cluster(args->cluster_file); + fdb_block_wait(f); + err = fdb_future_get_cluster(f, &cluster); + check_fdb_error(err); + fdb_future_destroy(f); - /* database */ - /* big mystery -- do we ever have a database named other than "DB"? */ - f = fdb_cluster_create_database(cluster, (uint8_t *)"DB", 2); - fdb_block_wait(f); - err = fdb_future_get_database(f, &process.database); - check_fdb_error(err); - fdb_future_destroy(f); + /* database */ + /* big mystery -- do we ever have a database named other than "DB"? */ + f = fdb_cluster_create_database(cluster, (uint8_t*)"DB", 2); + fdb_block_wait(f); + err = fdb_future_get_database(f, &process.database); + check_fdb_error(err); + fdb_future_destroy(f); #else /* >= 610 */ - fdb_create_database(args->cluster_file, &process.database); + fdb_create_database(args->cluster_file, &process.database); #endif - if (args->verbose >= VERBOSE_DEBUG) { - printf("DEBUG: creating %d worker threads\n", args->num_threads); - } - worker_threads = (pthread_t *)calloc(sizeof(pthread_t), args->num_threads); - if (!worker_threads) { - fprintf(stderr, "ERROR: cannot allocate worker_threads\n"); - goto EXIT; - } + if (args->verbose >= VERBOSE_DEBUG) { + printf("DEBUG: creating %d worker threads\n", args->num_threads); + } + worker_threads = (pthread_t*)calloc(sizeof(pthread_t), args->num_threads); + if (!worker_threads) { + fprintf(stderr, "ERROR: cannot allocate worker_threads\n"); + goto EXIT; + } - /* spawn worker threads */ - thread_args = - (thread_args_t *)calloc(sizeof(thread_args_t), args->num_threads); - if (!thread_args) { - fprintf(stderr, "ERROR: cannot allocate thread_args\n"); - goto EXIT; - } + /* spawn worker threads */ + thread_args = (thread_args_t*)calloc(sizeof(thread_args_t), args->num_threads); + if (!thread_args) { + fprintf(stderr, "ERROR: cannot allocate thread_args\n"); + goto EXIT; + } - for (i = 0; i < args->num_threads; i++) { - thread_args[i].thread_id = i; - thread_args[i].process = &process; - rc = pthread_create(&worker_threads[i], NULL, worker_thread, - (void *)&thread_args[i]); - if (rc != 0) { - fprintf(stderr, "ERROR: cannot create a new worker thread %d\n", i); - /* ignore this thread? */ - } - } + for (i = 0; i < args->num_threads; i++) { + thread_args[i].thread_id = i; + thread_args[i].process = &process; + rc = pthread_create(&worker_threads[i], NULL, worker_thread, (void*)&thread_args[i]); + if (rc != 0) { + fprintf(stderr, "ERROR: cannot create a new worker thread %d\n", i); + /* ignore this thread? */ + } + } - /*** party is over ***/ + /*** party is over ***/ - /* wait for everyone to finish */ - for (i = 0; i < args->num_threads; i++) { - if (args->verbose >= VERBOSE_DEBUG) { - printf("DEBUG: worker_thread %d joining\n", i); - } - rc = pthread_join(worker_threads[i], NULL); - if (rc != 0) { - fprintf(stderr, "ERROR: threads %d failed to join\n", i); - } - } + /* wait for everyone to finish */ + for (i = 0; i < args->num_threads; i++) { + if (args->verbose >= VERBOSE_DEBUG) { + printf("DEBUG: worker_thread %d joining\n", i); + } + rc = pthread_join(worker_threads[i], NULL); + if (rc != 0) { + fprintf(stderr, "ERROR: threads %d failed to join\n", i); + } + } EXIT: - if (worker_threads) - free(worker_threads); - if (thread_args) - free(thread_args); + if (worker_threads) + free(worker_threads); + if (thread_args) + free(thread_args); - /* clean up database and cluster */ - fdb_database_destroy(process.database); + /* clean up database and cluster */ + fdb_database_destroy(process.database); #if FDB_API_VERSION < 610 - fdb_cluster_destroy(cluster); + fdb_cluster_destroy(cluster); #endif - /* stop the network thread */ - if (args->verbose >= VERBOSE_DEBUG) { - printf("DEBUG: fdb_stop_network\n"); - } - err = fdb_stop_network(); - check_fdb_error(err); + /* stop the network thread */ + if (args->verbose >= VERBOSE_DEBUG) { + printf("DEBUG: fdb_stop_network\n"); + } + err = fdb_stop_network(); + check_fdb_error(err); FDB_FAIL: - /* wait for the network thread to join */ - if (args->verbose >= VERBOSE_DEBUG) { - printf("DEBUG: network_thread joining\n"); - } - rc = pthread_join(network_thread, NULL); - if (rc != 0) { - fprintf(stderr, "ERROR: network thread failed to join\n"); - } + /* wait for the network thread to join */ + if (args->verbose >= VERBOSE_DEBUG) { + printf("DEBUG: network_thread joining\n"); + } + rc = pthread_join(network_thread, NULL); + if (rc != 0) { + fprintf(stderr, "ERROR: network thread failed to join\n"); + } - return 0; + return 0; } /* initialize the parameters with default values */ -int init_args(mako_args_t *args) { - int i; - if (!args) - return -1; - memset(args, 0, sizeof(mako_args_t)); /* zero-out everything */ - args->json = 0; - args->num_processes = 1; - args->num_threads = 1; - args->mode = MODE_INVALID; - args->rows = 10000; - args->seconds = 0; - args->iteration = 0; - args->tps = 0; - args->sampling = 1000; - args->key_length = 16; - args->value_length = 16; - args->zipf = 0; - args->commit_get = 0; - args->verbose = 1; - args->flatbuffers = 0; - args->knobs[0] = '\0'; - args->trace = 0; - args->tracepath[0] = '\0'; - for (i = 0; i < MAX_OP; i++) { - args->txnspec.ops[i][OP_COUNT] = 0; - } - return 0; +int init_args(mako_args_t* args) { + int i; + if (!args) + return -1; + memset(args, 0, sizeof(mako_args_t)); /* zero-out everything */ + args->json = 0; + args->num_processes = 1; + args->num_threads = 1; + args->mode = MODE_INVALID; + args->rows = 10000; + args->seconds = 0; + args->iteration = 0; + args->tps = 0; + args->sampling = 1000; + args->key_length = 16; + args->value_length = 16; + args->zipf = 0; + args->commit_get = 0; + args->verbose = 1; + args->flatbuffers = 0; + args->knobs[0] = '\0'; + args->trace = 0; + args->tracepath[0] = '\0'; + for (i = 0; i < MAX_OP; i++) { + args->txnspec.ops[i][OP_COUNT] = 0; + } + return 0; } /* parse transaction specification */ -int parse_transaction(mako_args_t *args, char *optarg) { - char *ptr = optarg; - int op = 0; - int rangeop = 0; - int num; - int error = 0; +int parse_transaction(mako_args_t* args, char* optarg) { + char* ptr = optarg; + int op = 0; + int rangeop = 0; + int num; + int error = 0; - for (op = 0; op < MAX_OP; op++) { - args->txnspec.ops[op][OP_COUNT] = 0; - args->txnspec.ops[op][OP_RANGE] = 0; - } + for (op = 0; op < MAX_OP; op++) { + args->txnspec.ops[op][OP_COUNT] = 0; + args->txnspec.ops[op][OP_RANGE] = 0; + } - op = 0; - while (*ptr) { - if (strncmp(ptr, "grv", 3) == 0) { - op = OP_GETREADVERSION; - ptr += 3; - } else if (strncmp(ptr, "gr", 2) == 0) { - op = OP_GETRANGE; - rangeop = 1; - ptr += 2; - } else if (strncmp(ptr, "g", 1) == 0) { - op = OP_GET; - ptr++; - } else if (strncmp(ptr, "sgr", 3) == 0) { - op = OP_SGETRANGE; - rangeop = 1; - ptr += 3; - } else if (strncmp(ptr, "sg", 2) == 0) { - op = OP_SGET; - ptr += 2; - } else if (strncmp(ptr, "u", 1) == 0) { - op = OP_UPDATE; - ptr++; - } else if (strncmp(ptr, "ir", 2) == 0) { - op = OP_INSERTRANGE; - rangeop = 1; - ptr += 2; - } else if (strncmp(ptr, "i", 1) == 0) { - op = OP_INSERT; - ptr++; - } else if (strncmp(ptr, "cr", 2) == 0) { - op = OP_CLEARRANGE; - rangeop = 1; - ptr += 2; - } else if (strncmp(ptr, "c", 1) == 0) { - op = OP_CLEAR; - ptr++; - } else if (strncmp(ptr, "scr", 3) == 0) { - op = OP_SETCLEARRANGE; - rangeop = 1; - ptr += 3; - } else if (strncmp(ptr, "sc", 2) == 0) { - op = OP_SETCLEAR; - ptr += 2; - } else { - if (args->verbose == VERBOSE_DEBUG) { - printf("Error: Invalid transaction spec: %s\n", ptr); - } - error = 1; - break; - } + op = 0; + while (*ptr) { + if (strncmp(ptr, "grv", 3) == 0) { + op = OP_GETREADVERSION; + ptr += 3; + } else if (strncmp(ptr, "gr", 2) == 0) { + op = OP_GETRANGE; + rangeop = 1; + ptr += 2; + } else if (strncmp(ptr, "g", 1) == 0) { + op = OP_GET; + ptr++; + } else if (strncmp(ptr, "sgr", 3) == 0) { + op = OP_SGETRANGE; + rangeop = 1; + ptr += 3; + } else if (strncmp(ptr, "sg", 2) == 0) { + op = OP_SGET; + ptr += 2; + } else if (strncmp(ptr, "u", 1) == 0) { + op = OP_UPDATE; + ptr++; + } else if (strncmp(ptr, "ir", 2) == 0) { + op = OP_INSERTRANGE; + rangeop = 1; + ptr += 2; + } else if (strncmp(ptr, "i", 1) == 0) { + op = OP_INSERT; + ptr++; + } else if (strncmp(ptr, "cr", 2) == 0) { + op = OP_CLEARRANGE; + rangeop = 1; + ptr += 2; + } else if (strncmp(ptr, "c", 1) == 0) { + op = OP_CLEAR; + ptr++; + } else if (strncmp(ptr, "scr", 3) == 0) { + op = OP_SETCLEARRANGE; + rangeop = 1; + ptr += 3; + } else if (strncmp(ptr, "sc", 2) == 0) { + op = OP_SETCLEAR; + ptr += 2; + } else { + if (args->verbose == VERBOSE_DEBUG) { + printf("Error: Invalid transaction spec: %s\n", ptr); + } + error = 1; + break; + } - /* count */ - num = 0; - if ((*ptr < '0') || (*ptr > '9')) { - num = 1; /* if omitted, set it to 1 */ - } else { - while ((*ptr >= '0') && (*ptr <= '9')) { - num = num * 10 + *ptr - '0'; - ptr++; - } - } - /* set count */ - args->txnspec.ops[op][OP_COUNT] = num; + /* count */ + num = 0; + if ((*ptr < '0') || (*ptr > '9')) { + num = 1; /* if omitted, set it to 1 */ + } else { + while ((*ptr >= '0') && (*ptr <= '9')) { + num = num * 10 + *ptr - '0'; + ptr++; + } + } + /* set count */ + args->txnspec.ops[op][OP_COUNT] = num; - if (rangeop) { - if (*ptr != ':') { - error = 1; - break; - } else { - ptr++; /* skip ':' */ - /* check negative '-' sign */ - if (*ptr == '-') { - args->txnspec.ops[op][OP_REVERSE] = 1; - ptr++; - } else { - args->txnspec.ops[op][OP_REVERSE] = 0; - } - num = 0; - if ((*ptr < '0') || (*ptr > '9')) { - error = 1; - break; - } - while ((*ptr >= '0') && (*ptr <= '9')) { - num = num * 10 + *ptr - '0'; - ptr++; - } - /* set range */ - args->txnspec.ops[op][OP_RANGE] = num; - } - } - rangeop = 0; - } + if (rangeop) { + if (*ptr != ':') { + error = 1; + break; + } else { + ptr++; /* skip ':' */ + /* check negative '-' sign */ + if (*ptr == '-') { + args->txnspec.ops[op][OP_REVERSE] = 1; + ptr++; + } else { + args->txnspec.ops[op][OP_REVERSE] = 0; + } + num = 0; + if ((*ptr < '0') || (*ptr > '9')) { + error = 1; + break; + } + while ((*ptr >= '0') && (*ptr <= '9')) { + num = num * 10 + *ptr - '0'; + ptr++; + } + /* set range */ + args->txnspec.ops[op][OP_RANGE] = num; + } + } + rangeop = 0; + } - if (error) { - fprintf(stderr, "ERROR: invalid transaction specification %s\n", optarg); - return -1; - } + if (error) { + fprintf(stderr, "ERROR: invalid transaction specification %s\n", optarg); + return -1; + } - if (args->verbose == VERBOSE_DEBUG) { - for (op = 0; op < MAX_OP; op++) { - printf("DEBUG: OP: %d: %d: %d\n", op, args->txnspec.ops[op][0], - args->txnspec.ops[op][1]); - } - } + if (args->verbose == VERBOSE_DEBUG) { + for (op = 0; op < MAX_OP; op++) { + printf("DEBUG: OP: %d: %d: %d\n", op, args->txnspec.ops[op][0], args->txnspec.ops[op][1]); + } + } - return 0; + return 0; } void usage() { - printf("Usage:\n"); - printf("%-24s%s\n", "-h, --help", "Print this message"); - printf("%-24s%s\n", " --version", "Print FDB version"); - printf("%-24s%s\n", "-v, --verbose", "Specify verbosity"); - printf("%-24s%s\n", "-c, --cluster=FILE", "Specify FDB cluster file"); - printf("%-24s%s\n", "-p, --procs=PROCS", - "Specify number of worker processes"); - printf("%-24s%s\n", "-t, --threads=THREADS", - "Specify number of worker threads"); - printf("%-24s%s\n", "-r, --rows=ROWS", "Specify number of records"); - printf("%-24s%s\n", "-s, --seconds=SECONDS", - "Specify the test duration in seconds\n"); - printf("%-24s%s\n", "", "This option cannot be specified with --iteration."); - printf("%-24s%s\n", "-i, --iteration=ITERS", - "Specify the number of iterations.\n"); - printf("%-24s%s\n", "", "This option cannot be specified with --seconds."); - printf("%-24s%s\n", " --keylen=LENGTH", "Specify the key lengths"); - printf("%-24s%s\n", " --vallen=LENGTH", "Specify the value lengths"); - printf("%-24s%s\n", "-x, --transaction=SPEC", "Transaction specification"); - printf("%-24s%s\n", " --tps=TPS", "Specify the target TPS"); - printf("%-24s%s\n", " --sampling=RATE", - "Specify the sampling rate for latency stats"); - printf("%-24s%s\n", "-m, --mode=MODE", - "Specify the mode (build, run, clean)"); - printf("%-24s%s\n", "-z, --zipf", - "Use zipfian distribution instead of uniform distribution"); - printf("%-24s%s\n", " --commitget", "Commit GETs"); - printf("%-24s%s\n", " --trace", "Enable tracing"); - printf("%-24s%s\n", " --tracepath=PATH", "Set trace file path"); - printf("%-24s%s\n", " --knobs=KNOBS", "Set client knobs"); - printf("%-24s%s\n", " --flatbuffers", "Use flatbuffers"); + printf("Usage:\n"); + printf("%-24s%s\n", "-h, --help", "Print this message"); + printf("%-24s%s\n", " --version", "Print FDB version"); + printf("%-24s%s\n", "-v, --verbose", "Specify verbosity"); + printf("%-24s%s\n", "-c, --cluster=FILE", "Specify FDB cluster file"); + printf("%-24s%s\n", "-p, --procs=PROCS", "Specify number of worker processes"); + printf("%-24s%s\n", "-t, --threads=THREADS", "Specify number of worker threads"); + printf("%-24s%s\n", "-r, --rows=ROWS", "Specify number of records"); + printf("%-24s%s\n", "-s, --seconds=SECONDS", "Specify the test duration in seconds\n"); + printf("%-24s%s\n", "", "This option cannot be specified with --iteration."); + printf("%-24s%s\n", "-i, --iteration=ITERS", "Specify the number of iterations.\n"); + printf("%-24s%s\n", "", "This option cannot be specified with --seconds."); + printf("%-24s%s\n", " --keylen=LENGTH", "Specify the key lengths"); + printf("%-24s%s\n", " --vallen=LENGTH", "Specify the value lengths"); + printf("%-24s%s\n", "-x, --transaction=SPEC", "Transaction specification"); + printf("%-24s%s\n", " --tps=TPS", "Specify the target TPS"); + printf("%-24s%s\n", " --sampling=RATE", "Specify the sampling rate for latency stats"); + printf("%-24s%s\n", "-m, --mode=MODE", "Specify the mode (build, run, clean)"); + printf("%-24s%s\n", "-z, --zipf", "Use zipfian distribution instead of uniform distribution"); + printf("%-24s%s\n", " --commitget", "Commit GETs"); + printf("%-24s%s\n", " --trace", "Enable tracing"); + printf("%-24s%s\n", " --tracepath=PATH", "Set trace file path"); + printf("%-24s%s\n", " --knobs=KNOBS", "Set client knobs"); + printf("%-24s%s\n", " --flatbuffers", "Use flatbuffers"); } /* parse benchmark paramters */ -int parse_args(int argc, char *argv[], mako_args_t *args) { - int rc; - int c; - int idx; - while (1) { - const char *short_options = "c:p:t:r:s:i:x:v:m:hjz"; - static struct option long_options[] = { - /* name, has_arg, flag, val */ - {"cluster", required_argument, NULL, 'c'}, - {"procs", required_argument, NULL, 'p'}, - {"threads", required_argument, NULL, 't'}, - {"rows", required_argument, NULL, 'r'}, - {"seconds", required_argument, NULL, 's'}, - {"iteration", required_argument, NULL, 'i'}, - {"keylen", required_argument, NULL, ARG_KEYLEN}, - {"vallen", required_argument, NULL, ARG_VALLEN}, - {"transaction", required_argument, NULL, 'x'}, - {"tps", required_argument, NULL, ARG_TPS}, - {"sampling", required_argument, NULL, ARG_SAMPLING}, - {"verbose", required_argument, NULL, 'v'}, - {"mode", required_argument, NULL, 'm'}, - {"knobs", required_argument, NULL, ARG_KNOBS}, - {"tracepath", required_argument, NULL, ARG_TRACEPATH}, - /* no args */ - {"help", no_argument, NULL, 'h'}, - {"json", no_argument, NULL, 'j'}, - {"zipf", no_argument, NULL, 'z'}, - {"commitget", no_argument, NULL, ARG_COMMITGET}, - {"flatbuffers", no_argument, NULL, ARG_FLATBUFFERS}, - {"trace", no_argument, NULL, ARG_TRACE}, - {"version", no_argument, NULL, ARG_VERSION}, - {NULL, 0, NULL, 0}}; - idx = 0; - c = getopt_long(argc, argv, short_options, long_options, &idx); - if (c < 0) - break; - switch (c) { - case '?': - case 'h': - usage(); - return -1; - case 'c': - strcpy(args->cluster_file, optarg); - break; - case 'p': - args->num_processes = atoi(optarg); - break; - case 't': - args->num_threads = atoi(optarg); - break; - case 'r': - args->rows = atoi(optarg); - break; - case 's': - args->seconds = atoi(optarg); - break; - case 'i': - args->iteration = atoi(optarg); - break; - case 'x': - rc = parse_transaction(args, optarg); - if (rc < 0) - return -1; - break; - case 'v': - args->verbose = atoi(optarg); - break; - case 'z': - args->zipf = 1; - break; - case 'm': - if (strcmp(optarg, "clean") == 0) { - args->mode = MODE_CLEAN; - } else if (strcmp(optarg, "build") == 0) { - args->mode = MODE_BUILD; - } else if (strcmp(optarg, "run") == 0) { - args->mode = MODE_RUN; - } - break; - case ARG_KEYLEN: - args->key_length = atoi(optarg); - break; - case ARG_VALLEN: - args->value_length = atoi(optarg); - break; - case ARG_TPS: - args->tps = atoi(optarg); - break; - case ARG_SAMPLING: - args->sampling = atoi(optarg); - break; - case ARG_VERSION: - fprintf(stderr, "Version: %d\n", FDB_API_VERSION); - exit(0); - break; - case ARG_COMMITGET: - args->commit_get = 1; - break; - case ARG_FLATBUFFERS: - args->flatbuffers = 1; - break; - case ARG_KNOBS: - memcpy(args->knobs, optarg, strlen(optarg) + 1); - break; - case ARG_TRACE: - args->trace = 1; - break; - case ARG_TRACEPATH: - args->trace = 1; - memcpy(args->tracepath, optarg, strlen(optarg) + 1); - break; - } - } - return 0; +int parse_args(int argc, char* argv[], mako_args_t* args) { + int rc; + int c; + int idx; + while (1) { + const char* short_options = "c:p:t:r:s:i:x:v:m:hjz"; + static struct option long_options[] = { /* name, has_arg, flag, val */ + { "cluster", required_argument, NULL, 'c' }, + { "procs", required_argument, NULL, 'p' }, + { "threads", required_argument, NULL, 't' }, + { "rows", required_argument, NULL, 'r' }, + { "seconds", required_argument, NULL, 's' }, + { "iteration", required_argument, NULL, 'i' }, + { "keylen", required_argument, NULL, ARG_KEYLEN }, + { "vallen", required_argument, NULL, ARG_VALLEN }, + { "transaction", required_argument, NULL, 'x' }, + { "tps", required_argument, NULL, ARG_TPS }, + { "sampling", required_argument, NULL, ARG_SAMPLING }, + { "verbose", required_argument, NULL, 'v' }, + { "mode", required_argument, NULL, 'm' }, + { "knobs", required_argument, NULL, ARG_KNOBS }, + { "tracepath", required_argument, NULL, ARG_TRACEPATH }, + /* no args */ + { "help", no_argument, NULL, 'h' }, + { "json", no_argument, NULL, 'j' }, + { "zipf", no_argument, NULL, 'z' }, + { "commitget", no_argument, NULL, ARG_COMMITGET }, + { "flatbuffers", no_argument, NULL, ARG_FLATBUFFERS }, + { "trace", no_argument, NULL, ARG_TRACE }, + { "version", no_argument, NULL, ARG_VERSION }, + { NULL, 0, NULL, 0 } + }; + idx = 0; + c = getopt_long(argc, argv, short_options, long_options, &idx); + if (c < 0) + break; + switch (c) { + case '?': + case 'h': + usage(); + return -1; + case 'c': + strcpy(args->cluster_file, optarg); + break; + case 'p': + args->num_processes = atoi(optarg); + break; + case 't': + args->num_threads = atoi(optarg); + break; + case 'r': + args->rows = atoi(optarg); + break; + case 's': + args->seconds = atoi(optarg); + break; + case 'i': + args->iteration = atoi(optarg); + break; + case 'x': + rc = parse_transaction(args, optarg); + if (rc < 0) + return -1; + break; + case 'v': + args->verbose = atoi(optarg); + break; + case 'z': + args->zipf = 1; + break; + case 'm': + if (strcmp(optarg, "clean") == 0) { + args->mode = MODE_CLEAN; + } else if (strcmp(optarg, "build") == 0) { + args->mode = MODE_BUILD; + } else if (strcmp(optarg, "run") == 0) { + args->mode = MODE_RUN; + } + break; + case ARG_KEYLEN: + args->key_length = atoi(optarg); + break; + case ARG_VALLEN: + args->value_length = atoi(optarg); + break; + case ARG_TPS: + args->tps = atoi(optarg); + break; + case ARG_SAMPLING: + args->sampling = atoi(optarg); + break; + case ARG_VERSION: + fprintf(stderr, "Version: %d\n", FDB_API_VERSION); + exit(0); + break; + case ARG_COMMITGET: + args->commit_get = 1; + break; + case ARG_FLATBUFFERS: + args->flatbuffers = 1; + break; + case ARG_KNOBS: + memcpy(args->knobs, optarg, strlen(optarg) + 1); + break; + case ARG_TRACE: + args->trace = 1; + break; + case ARG_TRACEPATH: + args->trace = 1; + memcpy(args->tracepath, optarg, strlen(optarg) + 1); + break; + } + } + return 0; } -int validate_args(mako_args_t *args) { - if (args->mode == MODE_INVALID) { - fprintf(stderr, "ERROR: --mode has to be set\n"); - return -1; - } - if (args->rows <= 0) { - fprintf(stderr, "ERROR: --rows must be a positive integer\n"); - return -1; - } - if (args->key_length < 0) { - fprintf(stderr, "ERROR: --keylen must be a positive integer\n"); - return -1; - } - if (args->value_length < 0) { - fprintf(stderr, "ERROR: --vallen must be a positive integer\n"); - return -1; - } - if (args->key_length < 4 /* "mako" */ + digits(args->rows)) { - fprintf(stderr, - "ERROR: --keylen must be larger than %d to store \"mako\" prefix " - "and maximum row number\n", - 4 + digits(args->rows)); - return -1; - } - if (args->mode == MODE_RUN) { - if ((args->seconds > 0) && (args->iteration > 0)) { - fprintf(stderr, "ERROR: Cannot specify seconds and iteration together\n"); - return -1; - } - if ((args->seconds == 0) && (args->iteration == 0)) { - fprintf(stderr, "ERROR: Must specify either seconds or iteration\n"); - return -1; - } - } - return 0; +int validate_args(mako_args_t* args) { + if (args->mode == MODE_INVALID) { + fprintf(stderr, "ERROR: --mode has to be set\n"); + return -1; + } + if (args->rows <= 0) { + fprintf(stderr, "ERROR: --rows must be a positive integer\n"); + return -1; + } + if (args->key_length < 0) { + fprintf(stderr, "ERROR: --keylen must be a positive integer\n"); + return -1; + } + if (args->value_length < 0) { + fprintf(stderr, "ERROR: --vallen must be a positive integer\n"); + return -1; + } + if (args->key_length < 4 /* "mako" */ + digits(args->rows)) { + fprintf(stderr, + "ERROR: --keylen must be larger than %d to store \"mako\" prefix " + "and maximum row number\n", + 4 + digits(args->rows)); + return -1; + } + if (args->mode == MODE_RUN) { + if ((args->seconds > 0) && (args->iteration > 0)) { + fprintf(stderr, "ERROR: Cannot specify seconds and iteration together\n"); + return -1; + } + if ((args->seconds == 0) && (args->iteration == 0)) { + fprintf(stderr, "ERROR: Must specify either seconds or iteration\n"); + return -1; + } + } + return 0; } /* stats output formatting */ @@ -1347,507 +1304,489 @@ int validate_args(mako_args_t *args) { #define STATS_TITLE_WIDTH 12 #define STATS_FIELD_WIDTH 12 -void print_stats(mako_args_t *args, mako_stats_t *stats, struct timespec *now, - struct timespec *prev) { - int i, j; - int op; - int print_err; - static uint64_t ops_total_prev[MAX_OP] = {0}; - uint64_t ops_total[MAX_OP] = {0}; - static uint64_t errors_total_prev[MAX_OP] = {0}; - uint64_t errors_total[MAX_OP] = {0}; - uint64_t errors_diff[MAX_OP] = {0}; - static uint64_t totalxacts_prev = 0; - uint64_t totalxacts = 0; - static uint64_t conflicts_prev = 0; - uint64_t conflicts = 0; - double durationns = (now->tv_sec - prev->tv_sec) * 1000000000.0 + - (now->tv_nsec - prev->tv_nsec); +void print_stats(mako_args_t* args, mako_stats_t* stats, struct timespec* now, struct timespec* prev) { + int i, j; + int op; + int print_err; + static uint64_t ops_total_prev[MAX_OP] = { 0 }; + uint64_t ops_total[MAX_OP] = { 0 }; + static uint64_t errors_total_prev[MAX_OP] = { 0 }; + uint64_t errors_total[MAX_OP] = { 0 }; + uint64_t errors_diff[MAX_OP] = { 0 }; + static uint64_t totalxacts_prev = 0; + uint64_t totalxacts = 0; + static uint64_t conflicts_prev = 0; + uint64_t conflicts = 0; + double durationns = (now->tv_sec - prev->tv_sec) * 1000000000.0 + (now->tv_nsec - prev->tv_nsec); - for (i = 0; i < args->num_processes; i++) { - for (j = 0; j < args->num_threads; j++) { - totalxacts += stats[(i * args->num_threads) + j].xacts; - conflicts += stats[(i * args->num_threads) + j].conflicts; - for (op = 0; op < MAX_OP; op++) { - ops_total[op] += stats[(i * args->num_threads) + j].ops[op]; - errors_total[op] += stats[(i * args->num_threads) + j].errors[op]; - } - } - } - printf("%" STR(STATS_TITLE_WIDTH) "s ", "OPS"); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", - ops_total[op] - ops_total_prev[op]); - errors_diff[op] = errors_total[op] - errors_total_prev[op]; - print_err = (errors_diff[op] > 0); - ops_total_prev[op] = ops_total[op]; - errors_total_prev[op] = errors_total[op]; - } - } - /* TPS */ - printf("%" STR(STATS_FIELD_WIDTH) ".2f ", - (totalxacts - totalxacts_prev) * 1000000000.0 / durationns); - totalxacts_prev = totalxacts; + for (i = 0; i < args->num_processes; i++) { + for (j = 0; j < args->num_threads; j++) { + totalxacts += stats[(i * args->num_threads) + j].xacts; + conflicts += stats[(i * args->num_threads) + j].conflicts; + for (op = 0; op < MAX_OP; op++) { + ops_total[op] += stats[(i * args->num_threads) + j].ops[op]; + errors_total[op] += stats[(i * args->num_threads) + j].errors[op]; + } + } + } + printf("%" STR(STATS_TITLE_WIDTH) "s ", "OPS"); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0) { + printf("%" STR(STATS_FIELD_WIDTH) "lld ", ops_total[op] - ops_total_prev[op]); + errors_diff[op] = errors_total[op] - errors_total_prev[op]; + print_err = (errors_diff[op] > 0); + ops_total_prev[op] = ops_total[op]; + errors_total_prev[op] = errors_total[op]; + } + } + /* TPS */ + printf("%" STR(STATS_FIELD_WIDTH) ".2f ", (totalxacts - totalxacts_prev) * 1000000000.0 / durationns); + totalxacts_prev = totalxacts; - /* Conflicts */ - printf("%" STR(STATS_FIELD_WIDTH) ".2f\n", - (conflicts - conflicts_prev) * 1000000000.0 / durationns); - conflicts_prev = conflicts; + /* Conflicts */ + printf("%" STR(STATS_FIELD_WIDTH) ".2f\n", (conflicts - conflicts_prev) * 1000000000.0 / durationns); + conflicts_prev = conflicts; - if (print_err) { - printf("%" STR(STATS_TITLE_WIDTH) "s ", "Errors"); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", errors_diff[op]); - } - } - printf("\n"); - } - return; + if (print_err) { + printf("%" STR(STATS_TITLE_WIDTH) "s ", "Errors"); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0) { + printf("%" STR(STATS_FIELD_WIDTH) "lld ", errors_diff[op]); + } + } + printf("\n"); + } + return; } -void print_stats_header(mako_args_t *args) { - int op; - int i; +void print_stats_header(mako_args_t* args) { + int op; + int i; - /* header */ - for (i = 0; i <= STATS_TITLE_WIDTH; i++) - printf(" "); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0) { - switch (op) { - case OP_GETREADVERSION: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "GRV"); - break; - case OP_GET: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "GET"); - break; - case OP_GETRANGE: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "GETRANGE"); - break; - case OP_SGET: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "SGET"); - break; - case OP_SGETRANGE: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "SGETRANGE"); - break; - case OP_UPDATE: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "UPDATE"); - break; - case OP_INSERT: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "INSERT"); - break; - case OP_INSERTRANGE: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "INSERTRANGE"); - break; - case OP_CLEAR: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "CLEAR"); - break; - case OP_SETCLEAR: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "SETCLEAR"); - break; - case OP_CLEARRANGE: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "CLEARRANGE"); - break; - case OP_SETCLEARRANGE: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "SETCLRRANGE"); - break; - } - } - } - printf("%" STR(STATS_FIELD_WIDTH) "s ", "TPS"); - printf("%" STR(STATS_FIELD_WIDTH) "s\n", "Conflicts/s"); + /* header */ + for (i = 0; i <= STATS_TITLE_WIDTH; i++) + printf(" "); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0) { + switch (op) { + case OP_GETREADVERSION: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "GRV"); + break; + case OP_GET: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "GET"); + break; + case OP_GETRANGE: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "GETRANGE"); + break; + case OP_SGET: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "SGET"); + break; + case OP_SGETRANGE: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "SGETRANGE"); + break; + case OP_UPDATE: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "UPDATE"); + break; + case OP_INSERT: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "INSERT"); + break; + case OP_INSERTRANGE: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "INSERTRANGE"); + break; + case OP_CLEAR: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "CLEAR"); + break; + case OP_SETCLEAR: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "SETCLEAR"); + break; + case OP_CLEARRANGE: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "CLEARRANGE"); + break; + case OP_SETCLEARRANGE: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "SETCLRRANGE"); + break; + } + } + } + printf("%" STR(STATS_FIELD_WIDTH) "s ", "TPS"); + printf("%" STR(STATS_FIELD_WIDTH) "s\n", "Conflicts/s"); - for (i = 0; i < STATS_TITLE_WIDTH; i++) - printf("="); - printf(" "); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0) { - for (i = 0; i < STATS_FIELD_WIDTH; i++) - printf("="); - printf(" "); - } - } - /* TPS */ - for (i = 0; i < STATS_FIELD_WIDTH; i++) - printf("="); - printf(" "); - /* Conflicts */ - for (i = 0; i < STATS_FIELD_WIDTH; i++) - printf("="); - printf("\n"); + for (i = 0; i < STATS_TITLE_WIDTH; i++) + printf("="); + printf(" "); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0) { + for (i = 0; i < STATS_FIELD_WIDTH; i++) + printf("="); + printf(" "); + } + } + /* TPS */ + for (i = 0; i < STATS_FIELD_WIDTH; i++) + printf("="); + printf(" "); + /* Conflicts */ + for (i = 0; i < STATS_FIELD_WIDTH; i++) + printf("="); + printf("\n"); } -void print_report(mako_args_t *args, mako_stats_t *stats, - struct timespec *timer_now, struct timespec *timer_start) { - int i, j, op; - uint64_t totalxacts = 0; - uint64_t conflicts = 0; - uint64_t totalerrors = 0; - uint64_t ops_total[MAX_OP] = {0}; - uint64_t errors_total[MAX_OP] = {0}; - uint64_t lat_min[MAX_OP] = {0}; - uint64_t lat_total[MAX_OP] = {0}; - uint64_t lat_samples[MAX_OP] = {0}; - uint64_t lat_max[MAX_OP] = {0}; +void print_report(mako_args_t* args, mako_stats_t* stats, struct timespec* timer_now, struct timespec* timer_start) { + int i, j, op; + uint64_t totalxacts = 0; + uint64_t conflicts = 0; + uint64_t totalerrors = 0; + uint64_t ops_total[MAX_OP] = { 0 }; + uint64_t errors_total[MAX_OP] = { 0 }; + uint64_t lat_min[MAX_OP] = { 0 }; + uint64_t lat_total[MAX_OP] = { 0 }; + uint64_t lat_samples[MAX_OP] = { 0 }; + uint64_t lat_max[MAX_OP] = { 0 }; - uint64_t durationns = (timer_now->tv_sec - timer_start->tv_sec) * 1000000000 + - (timer_now->tv_nsec - timer_start->tv_nsec); + uint64_t durationns = + (timer_now->tv_sec - timer_start->tv_sec) * 1000000000 + (timer_now->tv_nsec - timer_start->tv_nsec); - for (op = 0; op < MAX_OP; op++) { - lat_min[op] = 0xFFFFFFFFFFFFFFFF; /* uint64_t */ - lat_max[op] = 0; - lat_total[op] = 0; - lat_samples[op] = 0; - } + for (op = 0; op < MAX_OP; op++) { + lat_min[op] = 0xFFFFFFFFFFFFFFFF; /* uint64_t */ + lat_max[op] = 0; + lat_total[op] = 0; + lat_samples[op] = 0; + } - for (i = 0; i < args->num_processes; i++) { - for (j = 0; j < args->num_threads; j++) { - int idx = i * args->num_threads + j; - totalxacts += stats[idx].xacts; - conflicts += stats[idx].conflicts; - for (op = 0; op < MAX_OP; op++) { - if ((args->txnspec.ops[op][OP_COUNT] > 0) || (op == OP_COMMIT)) { - totalerrors += stats[idx].errors[op]; - ops_total[op] += stats[idx].ops[op]; - errors_total[op] += stats[idx].errors[op]; - lat_total[op] += stats[idx].latency_us_total[op]; - lat_samples[op] += stats[idx].latency_samples[op]; - if (stats[idx].latency_us_min[op] < lat_min[op]) { - lat_min[op] = stats[idx].latency_us_min[op]; - } - if (stats[idx].latency_us_max[op] > lat_max[op]) { - lat_max[op] = stats[idx].latency_us_max[op]; - } - } /* if count > 0 */ - } - } - } + for (i = 0; i < args->num_processes; i++) { + for (j = 0; j < args->num_threads; j++) { + int idx = i * args->num_threads + j; + totalxacts += stats[idx].xacts; + conflicts += stats[idx].conflicts; + for (op = 0; op < MAX_OP; op++) { + if ((args->txnspec.ops[op][OP_COUNT] > 0) || (op == OP_COMMIT)) { + totalerrors += stats[idx].errors[op]; + ops_total[op] += stats[idx].ops[op]; + errors_total[op] += stats[idx].errors[op]; + lat_total[op] += stats[idx].latency_us_total[op]; + lat_samples[op] += stats[idx].latency_samples[op]; + if (stats[idx].latency_us_min[op] < lat_min[op]) { + lat_min[op] = stats[idx].latency_us_min[op]; + } + if (stats[idx].latency_us_max[op] > lat_max[op]) { + lat_max[op] = stats[idx].latency_us_max[op]; + } + } /* if count > 0 */ + } + } + } - /* overall stats */ - printf("\n====== Total Duration %6.3f sec ======\n\n", - (double)durationns / 1000000000); - printf("Total Processes: %8d\n", args->num_processes); - printf("Total Threads: %8d\n", args->num_threads); - printf("Target TPS: %8d\n", args->tps); - printf("Total Xacts: %8lld\n", totalxacts); - printf("Total Conflicts: %8lld\n", conflicts); - printf("Total Errors: %8lld\n", totalerrors); - printf("Overall TPS: %8lld\n\n", totalxacts * 1000000000 / durationns); + /* overall stats */ + printf("\n====== Total Duration %6.3f sec ======\n\n", (double)durationns / 1000000000); + printf("Total Processes: %8d\n", args->num_processes); + printf("Total Threads: %8d\n", args->num_threads); + printf("Target TPS: %8d\n", args->tps); + printf("Total Xacts: %8lld\n", totalxacts); + printf("Total Conflicts: %8lld\n", conflicts); + printf("Total Errors: %8lld\n", totalerrors); + printf("Overall TPS: %8lld\n\n", totalxacts * 1000000000 / durationns); - /* per-op stats */ - print_stats_header(args); + /* per-op stats */ + print_stats_header(args); - /* OPS */ - printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Total OPS"); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_COMMIT) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", ops_total[op]); - } - } - /* TPS */ - printf("%" STR(STATS_FIELD_WIDTH) ".2f ", - totalxacts * 1000000000.0 / durationns); - /* Conflicts */ - printf("%" STR(STATS_FIELD_WIDTH) ".2f\n", - conflicts * 1000000000.0 / durationns); + /* OPS */ + printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Total OPS"); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_COMMIT) { + printf("%" STR(STATS_FIELD_WIDTH) "lld ", ops_total[op]); + } + } + /* TPS */ + printf("%" STR(STATS_FIELD_WIDTH) ".2f ", totalxacts * 1000000000.0 / durationns); + /* Conflicts */ + printf("%" STR(STATS_FIELD_WIDTH) ".2f\n", conflicts * 1000000000.0 / durationns); - /* Errors */ - printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Errors"); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_COMMIT) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", errors_total[op]); - } - } - printf("\n"); + /* Errors */ + printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Errors"); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_COMMIT) { + printf("%" STR(STATS_FIELD_WIDTH) "lld ", errors_total[op]); + } + } + printf("\n"); - /* Min Latency */ - printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Lat Min (us)"); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_COMMIT) { - if (lat_min[op] == -1) { - printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); - } else { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_min[op]); - } - } - } - printf("\n"); + /* Min Latency */ + printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Lat Min (us)"); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_COMMIT) { + if (lat_min[op] == -1) { + printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); + } else { + printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_min[op]); + } + } + } + printf("\n"); - /* Avg Latency */ - printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Lat Avg (us)"); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_COMMIT) { - if (lat_total[op]) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", - lat_total[op] / lat_samples[op]); - } else { - printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); - } - } - } - printf("\n"); + /* Avg Latency */ + printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Lat Avg (us)"); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_COMMIT) { + if (lat_total[op]) { + printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_total[op] / lat_samples[op]); + } else { + printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); + } + } + } + printf("\n"); - /* Max Latency */ - printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Lat Max (us)"); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_COMMIT) { - if (lat_max[op] == 0) { - printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); - } else { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_max[op]); - } - } - } - printf("\n"); + /* Max Latency */ + printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Lat Max (us)"); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_COMMIT) { + if (lat_max[op] == 0) { + printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); + } else { + printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_max[op]); + } + } + } + printf("\n"); } -int stats_process_main(mako_args_t *args, mako_stats_t *stats, - volatile int *signal) { - struct timespec timer_start, timer_prev, timer_now; +int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile int* signal) { + struct timespec timer_start, timer_prev, timer_now; - /* wait until the signal turn on */ - while (*signal == SIGNAL_OFF) { - usleep(10000); /* 10ms */ - } + /* wait until the signal turn on */ + while (*signal == SIGNAL_OFF) { + usleep(10000); /* 10ms */ + } - if (args->verbose >= VERBOSE_DEFAULT) - print_stats_header(args); + if (args->verbose >= VERBOSE_DEFAULT) + print_stats_header(args); - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); - timer_prev.tv_sec = timer_start.tv_sec; - timer_prev.tv_nsec = timer_start.tv_nsec; - while (*signal != SIGNAL_RED) { - usleep(100000); /* sleep for 100ms */ - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); - /* roughly 1 sec */ - if (timer_now.tv_sec > timer_prev.tv_sec) { - if (args->verbose >= VERBOSE_DEFAULT) - print_stats(args, stats, &timer_now, &timer_prev); - timer_prev.tv_sec = timer_now.tv_sec; - timer_prev.tv_nsec = timer_now.tv_nsec; - } - } + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); + timer_prev.tv_sec = timer_start.tv_sec; + timer_prev.tv_nsec = timer_start.tv_nsec; + while (*signal != SIGNAL_RED) { + usleep(100000); /* sleep for 100ms */ + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); + /* roughly 1 sec */ + if (timer_now.tv_sec > timer_prev.tv_sec) { + if (args->verbose >= VERBOSE_DEFAULT) + print_stats(args, stats, &timer_now, &timer_prev); + timer_prev.tv_sec = timer_now.tv_sec; + timer_prev.tv_nsec = timer_now.tv_nsec; + } + } - /* print report */ - if (args->verbose >= VERBOSE_DEFAULT) { - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); - print_report(args, stats, &timer_now, &timer_start); - } + /* print report */ + if (args->verbose >= VERBOSE_DEFAULT) { + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); + print_report(args, stats, &timer_now, &timer_start); + } - return 0; + return 0; } -int main(int argc, char *argv[]) { - int rc; - mako_args_t args; - int p; - pid_t *worker_pids = NULL; - proc_type_t proc_type = proc_master; - int worker_id; - pid_t pid; - int status; - mako_shmhdr_t *shm; /* shmhdr + stats */ - int shmfd; - char shmpath[NAME_MAX]; - size_t shmsize; - mako_stats_t *stats; +int main(int argc, char* argv[]) { + int rc; + mako_args_t args; + int p; + pid_t* worker_pids = NULL; + proc_type_t proc_type = proc_master; + int worker_id; + pid_t pid; + int status; + mako_shmhdr_t* shm; /* shmhdr + stats */ + int shmfd; + char shmpath[NAME_MAX]; + size_t shmsize; + mako_stats_t* stats; - rc = init_args(&args); - if (rc < 0) { - fprintf(stderr, "ERROR: init_args failed\n"); - return -1; - } - rc = parse_args(argc, argv, &args); - if (rc < 0) { - /* usage printed */ - return 0; - } + rc = init_args(&args); + if (rc < 0) { + fprintf(stderr, "ERROR: init_args failed\n"); + return -1; + } + rc = parse_args(argc, argv, &args); + if (rc < 0) { + /* usage printed */ + return 0; + } - rc = validate_args(&args); - if (rc < 0) - return -1; + rc = validate_args(&args); + if (rc < 0) + return -1; - if (args.mode == MODE_CLEAN) { - /* cleanup will be done from a single thread */ - args.num_processes = 1; - args.num_threads = 1; - } + if (args.mode == MODE_CLEAN) { + /* cleanup will be done from a single thread */ + args.num_processes = 1; + args.num_threads = 1; + } - if (args.mode == MODE_BUILD) { - if (args.txnspec.ops[OP_INSERT][OP_COUNT] == 0) { - parse_transaction(&args, "i100"); - } - } + if (args.mode == MODE_BUILD) { + if (args.txnspec.ops[OP_INSERT][OP_COUNT] == 0) { + parse_transaction(&args, "i100"); + } + } - /* create the shared memory for stats */ - sprintf(shmpath, "mako%d", getpid()); - shmfd = shm_open(shmpath, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); - if (shmfd < 0) { - fprintf(stderr, "ERROR: shm_open failed\n"); - return -1; - } + /* create the shared memory for stats */ + sprintf(shmpath, "mako%d", getpid()); + shmfd = shm_open(shmpath, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); + if (shmfd < 0) { + fprintf(stderr, "ERROR: shm_open failed\n"); + return -1; + } - /* allocate */ - shmsize = sizeof(mako_shmhdr_t) + - (sizeof(mako_stats_t) * args.num_processes * args.num_threads); - if (ftruncate(shmfd, shmsize) < 0) { - shm = MAP_FAILED; - fprintf(stderr, "ERROR: ftruncate (fd:%d size:%llu) failed\n", shmfd, - (unsigned long long)shmsize); - goto EXIT; - } + /* allocate */ + shmsize = sizeof(mako_shmhdr_t) + (sizeof(mako_stats_t) * args.num_processes * args.num_threads); + if (ftruncate(shmfd, shmsize) < 0) { + shm = MAP_FAILED; + fprintf(stderr, "ERROR: ftruncate (fd:%d size:%llu) failed\n", shmfd, (unsigned long long)shmsize); + goto EXIT; + } - /* map it */ - shm = (mako_shmhdr_t *)mmap(NULL, shmsize, PROT_READ | PROT_WRITE, MAP_SHARED, - shmfd, 0); - if (shm == MAP_FAILED) { - fprintf(stderr, "ERROR: mmap (fd:%d size:%llu) failed\n", shmfd, - (unsigned long long)shmsize); - goto EXIT; - } + /* map it */ + shm = (mako_shmhdr_t*)mmap(NULL, shmsize, PROT_READ | PROT_WRITE, MAP_SHARED, shmfd, 0); + if (shm == MAP_FAILED) { + fprintf(stderr, "ERROR: mmap (fd:%d size:%llu) failed\n", shmfd, (unsigned long long)shmsize); + goto EXIT; + } - stats = (mako_stats_t *)((void *)shm + sizeof(mako_shmhdr_t)); + stats = (mako_stats_t*)((void*)shm + sizeof(mako_shmhdr_t)); - /* initialize the shared memory */ - memset(shm, 0, shmsize); + /* initialize the shared memory */ + memset(shm, 0, shmsize); - /* get ready */ - shm->signal = SIGNAL_OFF; - shm->readycount = 0; + /* get ready */ + shm->signal = SIGNAL_OFF; + shm->readycount = 0; - /* fork worker processes */ - worker_pids = (pid_t*)calloc(sizeof(pid_t), args.num_processes + 1); - if (!worker_pids) { - fprintf(stderr, "ERROR: cannot allocate worker_pids (%d processes)\n", - args.num_processes); - goto EXIT; - } + /* fork worker processes */ + worker_pids = (pid_t*)calloc(sizeof(pid_t), args.num_processes + 1); + if (!worker_pids) { + fprintf(stderr, "ERROR: cannot allocate worker_pids (%d processes)\n", args.num_processes); + goto EXIT; + } - /* forking (num_process + 1) children */ - /* last process is the stats handler */ - for (p = 0; p < args.num_processes + 1; p++) { - pid = fork(); - if (pid != 0) { - /* master */ - worker_pids[p] = pid; - if (args.verbose == VERBOSE_DEBUG) { - printf("DEBUG: worker %d (PID:%d) forked\n", p, worker_pids[p]); - } - } else { - if (p < args.num_processes) { - /* worker process */ - proc_type = proc_worker; - worker_id = p; - } else { - /* stats */ - proc_type = proc_stats; - } - break; - } - } + /* forking (num_process + 1) children */ + /* last process is the stats handler */ + for (p = 0; p < args.num_processes + 1; p++) { + pid = fork(); + if (pid != 0) { + /* master */ + worker_pids[p] = pid; + if (args.verbose == VERBOSE_DEBUG) { + printf("DEBUG: worker %d (PID:%d) forked\n", p, worker_pids[p]); + } + } else { + if (p < args.num_processes) { + /* worker process */ + proc_type = proc_worker; + worker_id = p; + } else { + /* stats */ + proc_type = proc_stats; + } + break; + } + } - /* initialize the randomizer */ - srand(time(0) * getpid()); + /* initialize the randomizer */ + srand(time(0) * getpid()); - /* initialize zipfian if necessary (per-process) */ - if (args.zipf) { - zipfian_generator(args.rows); - } + /* initialize zipfian if necessary (per-process) */ + if (args.zipf) { + zipfian_generator(args.rows); + } - if (proc_type == proc_worker) { - /* worker process */ - worker_process_main(&args, worker_id, shm); - /* worker can exit here */ - exit(0); - } else if (proc_type == proc_stats) { - /* stats */ - if (args.mode == MODE_CLEAN) { - /* no stats needed for clean mode */ - exit(0); - } - stats_process_main(&args, stats, &shm->signal); - exit(0); - } + if (proc_type == proc_worker) { + /* worker process */ + worker_process_main(&args, worker_id, shm); + /* worker can exit here */ + exit(0); + } else if (proc_type == proc_stats) { + /* stats */ + if (args.mode == MODE_CLEAN) { + /* no stats needed for clean mode */ + exit(0); + } + stats_process_main(&args, stats, &shm->signal); + exit(0); + } - /* master */ + /* master */ - /* wait for everyone to be ready */ - while (shm->readycount < (args.num_processes * args.num_threads)) { - usleep(1000); - } - shm->signal = SIGNAL_GREEN; + /* wait for everyone to be ready */ + while (shm->readycount < (args.num_processes * args.num_threads)) { + usleep(1000); + } + shm->signal = SIGNAL_GREEN; - if (args.mode == MODE_RUN) { - struct timespec timer_start, timer_now; + if (args.mode == MODE_RUN) { + struct timespec timer_start, timer_now; - /* run the benchamrk */ + /* run the benchamrk */ - /* if seconds is specified, stop child processes after the specified - * duration */ - if (args.seconds > 0) { - if (args.verbose == VERBOSE_DEBUG) { - printf("DEBUG: master sleeping for %d seconds\n", args.seconds); - } + /* if seconds is specified, stop child processes after the specified + * duration */ + if (args.seconds > 0) { + if (args.verbose == VERBOSE_DEBUG) { + printf("DEBUG: master sleeping for %d seconds\n", args.seconds); + } - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); - while (1) { - usleep(100000); /* sleep for 100ms */ - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); - /* doesn't have to be precise */ - if (timer_now.tv_sec - timer_start.tv_sec > args.seconds) { - if (args.verbose == VERBOSE_DEBUG) { - printf("DEBUG: time's up (%d seconds)\n", args.seconds); - } - break; - } - } + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); + while (1) { + usleep(100000); /* sleep for 100ms */ + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); + /* doesn't have to be precise */ + if (timer_now.tv_sec - timer_start.tv_sec > args.seconds) { + if (args.verbose == VERBOSE_DEBUG) { + printf("DEBUG: time's up (%d seconds)\n", args.seconds); + } + break; + } + } - /* notify everyone the time's up */ - shm->signal = SIGNAL_RED; - } - } + /* notify everyone the time's up */ + shm->signal = SIGNAL_RED; + } + } - /* wait for worker processes to exit */ - for (p = 0; p < args.num_processes; p++) { - if (args.verbose == VERBOSE_DEBUG) { - printf("DEBUG: waiting worker %d (PID:%d) to exit\n", p, worker_pids[p]); - } - pid = waitpid(worker_pids[p], &status, 0 /* or what? */); - if (pid < 0) { - fprintf(stderr, "ERROR: waitpid failed for worker process PID %d\n", - worker_pids[p]); - } - if (args.verbose == VERBOSE_DEBUG) { - printf("DEBUG: worker %d (PID:%d) exited\n", p, worker_pids[p]); - } - } + /* wait for worker processes to exit */ + for (p = 0; p < args.num_processes; p++) { + if (args.verbose == VERBOSE_DEBUG) { + printf("DEBUG: waiting worker %d (PID:%d) to exit\n", p, worker_pids[p]); + } + pid = waitpid(worker_pids[p], &status, 0 /* or what? */); + if (pid < 0) { + fprintf(stderr, "ERROR: waitpid failed for worker process PID %d\n", worker_pids[p]); + } + if (args.verbose == VERBOSE_DEBUG) { + printf("DEBUG: worker %d (PID:%d) exited\n", p, worker_pids[p]); + } + } - /* all worker threads finished, stop the stats */ - if (args.mode == MODE_BUILD || args.iteration > 0) { - shm->signal = SIGNAL_RED; - } + /* all worker threads finished, stop the stats */ + if (args.mode == MODE_BUILD || args.iteration > 0) { + shm->signal = SIGNAL_RED; + } - /* wait for stats to stop */ - pid = waitpid(worker_pids[args.num_processes], &status, 0 /* or what? */); - if (pid < 0) { - fprintf(stderr, "ERROR: waitpid failed for stats process PID %d\n", - worker_pids[args.num_processes]); - } + /* wait for stats to stop */ + pid = waitpid(worker_pids[args.num_processes], &status, 0 /* or what? */); + if (pid < 0) { + fprintf(stderr, "ERROR: waitpid failed for stats process PID %d\n", worker_pids[args.num_processes]); + } EXIT: - if (worker_pids) - free(worker_pids); + if (worker_pids) + free(worker_pids); - if (shm != MAP_FAILED) - munmap(shm, shmsize); + if (shm != MAP_FAILED) + munmap(shm, shmsize); - if (shmfd) { - close(shmfd); - shm_unlink(shmpath); - } + if (shmfd) { + close(shmfd); + shm_unlink(shmpath); + } - return 0; + return 0; } diff --git a/bindings/c/test/mako/mako.h b/bindings/c/test/mako/mako.h old mode 100755 new mode 100644 index b8515092df..b944dc1a73 --- a/bindings/c/test/mako/mako.h +++ b/bindings/c/test/mako/mako.h @@ -69,34 +69,34 @@ #define KEYPREFIXLEN 4 typedef struct { - /* for each operation, it stores "count", "range" and "reverse" */ - int ops[MAX_OP][3]; + /* for each operation, it stores "count", "range" and "reverse" */ + int ops[MAX_OP][3]; } mako_txnspec_t; #define KNOB_MAX 256 /* benchmark parameters */ typedef struct { - int json; - int num_processes; - int num_threads; - int mode; - int rows; /* is 2 billion enough? */ - int seconds; - int iteration; - int tps; - int sampling; - int key_length; - int value_length; - int zipf; - int commit_get; - int verbose; - mako_txnspec_t txnspec; - char cluster_file[PATH_MAX]; - int trace; - char tracepath[PATH_MAX]; - char knobs[KNOB_MAX]; - uint8_t flatbuffers; + int json; + int num_processes; + int num_threads; + int mode; + int rows; /* is 2 billion enough? */ + int seconds; + int iteration; + int tps; + int sampling; + int key_length; + int value_length; + int zipf; + int commit_get; + int verbose; + mako_txnspec_t txnspec; + char cluster_file[PATH_MAX]; + int trace; + char tracepath[PATH_MAX]; + char knobs[KNOB_MAX]; + uint8_t flatbuffers; } mako_args_t; /* shared memory */ @@ -105,33 +105,33 @@ typedef struct { #define SIGNAL_OFF 2 typedef struct { - int signal; - int readycount; + int signal; + int readycount; } mako_shmhdr_t; typedef struct { - uint64_t xacts; - uint64_t conflicts; - uint64_t ops[MAX_OP]; - uint64_t errors[MAX_OP]; - uint64_t latency_samples[MAX_OP]; - uint64_t latency_us_total[MAX_OP]; - uint64_t latency_us_min[MAX_OP]; - uint64_t latency_us_max[MAX_OP]; + uint64_t xacts; + uint64_t conflicts; + uint64_t ops[MAX_OP]; + uint64_t errors[MAX_OP]; + uint64_t latency_samples[MAX_OP]; + uint64_t latency_us_total[MAX_OP]; + uint64_t latency_us_min[MAX_OP]; + uint64_t latency_us_max[MAX_OP]; } mako_stats_t; /* per-process information */ typedef struct { - int worker_id; - FDBDatabase *database; - mako_args_t *args; - mako_shmhdr_t *shm; + int worker_id; + FDBDatabase* database; + mako_args_t* args; + mako_shmhdr_t* shm; } process_info_t; /* args for threads */ typedef struct { - int thread_id; - process_info_t *process; + int thread_id; + process_info_t* process; } thread_args_t; /* process type */ diff --git a/bindings/c/test/mako/utils.c b/bindings/c/test/mako/utils.c old mode 100755 new mode 100644 index 9af57d58a4..c323e1965a --- a/bindings/c/test/mako/utils.c +++ b/bindings/c/test/mako/utils.c @@ -6,76 +6,74 @@ /* uniform-distribution random */ int urand(int low, int high) { - double r = rand() / (1.0 + RAND_MAX); - int range = high - low + 1; - return (int)((r * range) + low); + double r = rand() / (1.0 + RAND_MAX); + int range = high - low + 1; + return (int)((r * range) + low); } /* random string */ /* len is the buffer size, must include null */ -void randstr(char *str, int len) { - int i; - for (i = 0; i < len-1; i++) { - str[i] = '!' + urand(0, 'z'-'!'); /* generage a char from '!' to 'z' */ - } - str[len-1] = '\0'; +void randstr(char* str, int len) { + int i; + for (i = 0; i < len - 1; i++) { + str[i] = '!' + urand(0, 'z' - '!'); /* generage a char from '!' to 'z' */ + } + str[len - 1] = '\0'; } /* random numeric string */ /* len is the buffer size, must include null */ -void randnumstr(char *str, int len) { - int i; - for (i = 0; i < len-1; i++) { - str[i] = '0' + urand(0, 9); /* generage a char from '!' to 'z' */ - } - str[len-1] = '\0'; +void randnumstr(char* str, int len) { + int i; + for (i = 0; i < len - 1; i++) { + str[i] = '0' + urand(0, 9); /* generage a char from '!' to 'z' */ + } + str[len - 1] = '\0'; } /* return the first key to be inserted */ int insert_begin(int rows, int p_idx, int t_idx, int total_p, int total_t) { - double interval = (double)rows / total_p / total_t; - return (int)(round(interval * ((p_idx * total_t) + t_idx))); + double interval = (double)rows / total_p / total_t; + return (int)(round(interval * ((p_idx * total_t) + t_idx))); } /* return the last key to be inserted */ int insert_end(int rows, int p_idx, int t_idx, int total_p, int total_t) { - double interval = (double)rows / total_p / total_t; - return (int)(round(interval * ((p_idx * total_t) + t_idx + 1) - 1)); + double interval = (double)rows / total_p / total_t; + return (int)(round(interval * ((p_idx * total_t) + t_idx + 1) - 1)); } /* devide val equally among threads */ int compute_thread_portion(int val, int p_idx, int t_idx, int total_p, int total_t) { - int interval = val / total_p / total_t; - int remaining = val - (interval * total_p * total_t); - if ((p_idx * total_t + t_idx) < remaining) { - return interval+1; - } else if (interval == 0) { - return -1; - } - /* else */ - return interval; + int interval = val / total_p / total_t; + int remaining = val - (interval * total_p * total_t); + if ((p_idx * total_t + t_idx) < remaining) { + return interval + 1; + } else if (interval == 0) { + return -1; + } + /* else */ + return interval; } /* number of digits */ int digits(int num) { - int digits = 0; - while (num > 0) { - num /= 10; - digits++; - } - return digits; + int digits = 0; + while (num > 0) { + num /= 10; + digits++; + } + return digits; } - /* generate a key for a given key number */ /* len is the buffer size, key length + null */ -void genkey(char *str, int num, int rows, int len) { - int i; - int rowdigit = digits(rows); - sprintf(str, KEYPREFIX "%0.*d", rowdigit, num); - for (i = (KEYPREFIXLEN + rowdigit); i < len-1; i++) { - str[i] = 'x'; - } - str[len-1] = '\0'; +void genkey(char* str, int num, int rows, int len) { + int i; + int rowdigit = digits(rows); + sprintf(str, KEYPREFIX "%0.*d", rowdigit, num); + for (i = (KEYPREFIXLEN + rowdigit); i < len - 1; i++) { + str[i] = 'x'; + } + str[len - 1] = '\0'; } - diff --git a/bindings/c/test/mako/utils.h b/bindings/c/test/mako/utils.h old mode 100755 new mode 100644 index a95607ff6a..5fde4ef25a --- a/bindings/c/test/mako/utils.h +++ b/bindings/c/test/mako/utils.h @@ -9,12 +9,12 @@ int urand(int low, int high); /* write a random string of the length of (len-1) to memory pointed by str * with a null-termination character at str[len-1]. */ -void randstr(char *str, int len); +void randstr(char* str, int len); /* write a random numeric string of the length of (len-1) to memory pointed by str * with a null-termination character at str[len-1]. */ -void randnumstr(char *str, int len); +void randnumstr(char* str, int len); /* given the total number of rows to be inserted, * the worker process index p_idx and the thread index t_idx (both 0-based), @@ -27,26 +27,25 @@ int insert_begin(int rows, int p_idx, int t_idx, int total_p, int total_t); int insert_end(int rows, int p_idx, int t_idx, int total_p, int total_t); /* devide a value equally among threads */ -int compute_thread_portion(int val, int p_idx, int t_idx, int total_p, - int total_t); +int compute_thread_portion(int val, int p_idx, int t_idx, int total_p, int total_t); /* similar to insert_begin/end, compute_thread_tps computes * the per-thread target TPS for given configuration. */ -#define compute_thread_tps(val, p_idx, t_idx, total_p, total_t) \ - compute_thread_portion(val, p_idx, t_idx, total_p, total_t) +#define compute_thread_tps(val, p_idx, t_idx, total_p, total_t) \ + compute_thread_portion(val, p_idx, t_idx, total_p, total_t) /* similar to compute_thread_tps, * compute_thread_iters computs the number of iterations. */ -#define compute_thread_iters(val, p_idx, t_idx, total_p, total_t) \ - compute_thread_portion(val, p_idx, t_idx, total_p, total_t) +#define compute_thread_iters(val, p_idx, t_idx, total_p, total_t) \ + compute_thread_portion(val, p_idx, t_idx, total_p, total_t) /* get the number of digits */ int digits(int num); /* generate a key for a given key number */ /* len is the buffer size, key length + null */ -void genkey(char *str, int num, int rows, int len); +void genkey(char* str, int num, int rows, int len); #endif /* UTILS_H */ diff --git a/bindings/c/test/mako/zipf.c b/bindings/c/test/mako/zipf.c index 3d6656f476..91006263dc 100644 --- a/bindings/c/test/mako/zipf.c +++ b/bindings/c/test/mako/zipf.c @@ -23,93 +23,90 @@ double zeta2(int st, int n, double theta_val, double initialsum); double zetastatic(int n, double theta); double zeta(int n, double theta_val); - double rand_double() { - return (double)rand() / (double)RAND_MAX; + return (double)rand() / (double)RAND_MAX; } - int next_int(int itemcount) { - double u, uz; - int ret; + double u, uz; + int ret; - if (itemcount != countforzeta) { - zetan = zeta2(countforzeta, itemcount, theta, zetan); - } else if ((itemcount < countforzeta) && (allowitemcountdecrease)) { - zetan = zeta(itemcount, theta); - } - eta = (1 - pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan); + if (itemcount != countforzeta) { + zetan = zeta2(countforzeta, itemcount, theta, zetan); + } else if ((itemcount < countforzeta) && (allowitemcountdecrease)) { + zetan = zeta(itemcount, theta); + } + eta = (1 - pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan); - u = rand_double(); - uz = u * zetan; + u = rand_double(); + uz = u * zetan; - if (uz < 1.0) { - return base; - } + if (uz < 1.0) { + return base; + } - if (uz < 1.0 + pow(0.5, theta)) { - return base + 1; - } + if (uz < 1.0 + pow(0.5, theta)) { + return base + 1; + } - ret = base + (int)(itemcount * pow(eta * u - eta + 1, alpha)); - return ret; + ret = base + (int)(itemcount * pow(eta * u - eta + 1, alpha)); + return ret; } int zipfian_next() { - return next_int(items); + return next_int(items); } double zetastatic2(int st, int n, double theta, double initialsum) { - int i; - double sum = initialsum; - for (i = st; i < n; i++) { - sum += 1 / pow(i + 1, theta); - } - return sum; + int i; + double sum = initialsum; + for (i = st; i < n; i++) { + sum += 1 / pow(i + 1, theta); + } + return sum; } double zeta2(int st, int n, double theta_val, double initialsum) { - countforzeta = n; - return zetastatic2(st, n, theta_val, initialsum); + countforzeta = n; + return zetastatic2(st, n, theta_val, initialsum); } double zetastatic(int n, double theta) { - return zetastatic2(0, n, theta, 0); + return zetastatic2(0, n, theta, 0); } double zeta(int n, double theta_val) { - countforzeta = n; - return zetastatic(n, theta_val); + countforzeta = n; + return zetastatic(n, theta_val); } void zipfian_generator4(int min, int max, double _zipfianconstant, double _zetan) { - items = max - min + 1; - base = min; - zipfianconstant = _zipfianconstant; + items = max - min + 1; + base = min; + zipfianconstant = _zipfianconstant; - theta = zipfianconstant; - zeta2theta = zeta(2, theta); - alpha = 1.0 / (1.0 - theta); - zetan = _zetan; - countforzeta = items; - eta = (1 - pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan); + theta = zipfianconstant; + zeta2theta = zeta(2, theta); + alpha = 1.0 / (1.0 - theta); + zetan = _zetan; + countforzeta = items; + eta = (1 - pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan); - zipfian_next(); + zipfian_next(); } void zipfian_generator3(int min, int max, double zipfianconstant) { - zipfian_generator4(min, max, zipfianconstant, zetastatic(max - min + 1, zipfianconstant)); + zipfian_generator4(min, max, zipfianconstant, zetastatic(max - min + 1, zipfianconstant)); } void zipfian_generator2(int min, int max) { - zipfian_generator3(min, max, ZIPFIAN_CONSTANT); + zipfian_generator3(min, max, ZIPFIAN_CONSTANT); } void zipfian_generator(int items) { - zipfian_generator2(0, items - 1); + zipfian_generator2(0, items - 1); } - #if 0 /* test */ void main() { int i = 0; diff --git a/bindings/c/test/performance_test.c b/bindings/c/test/performance_test.c index edfb9a96d1..5f20d78996 100644 --- a/bindings/c/test/performance_test.c +++ b/bindings/c/test/performance_test.c @@ -31,38 +31,40 @@ int numKeys = 1000000; int keySize = 16; uint8_t** keys = NULL; int valueSize = 100; -uint8_t *valueStr = NULL; +uint8_t* valueStr = NULL; -fdb_error_t waitError(FDBFuture *f) { +fdb_error_t waitError(FDBFuture* f) { fdb_error_t blockError = fdb_future_block_until_ready(f); - if(!blockError) { + if (!blockError) { return fdb_future_get_error(f); } else { return blockError; } } -struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) { - FDBTransaction *tr = NULL; +struct RunResult run(struct ResultSet* rs, + FDBDatabase* db, + struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) { + FDBTransaction* tr = NULL; fdb_error_t e = fdb_database_create_transaction(db, &tr); checkError(e, "create transaction", rs); - while(1) { + while (1) { struct RunResult r = func(rs, tr); e = r.e; - if(!e) { - FDBFuture *f = fdb_transaction_commit(tr); + if (!e) { + FDBFuture* f = fdb_transaction_commit(tr); e = waitError(f); fdb_future_destroy(f); } - if(e) { - FDBFuture *f = fdb_transaction_on_error(tr, e); + if (e) { + FDBFuture* f = fdb_transaction_on_error(tr, e); fdb_error_t retryE = waitError(f); fdb_future_destroy(f); if (retryE) { fdb_transaction_destroy(tr); - return (struct RunResult) {0, retryE}; + return (struct RunResult){ 0, retryE }; } } else { fdb_transaction_destroy(tr); @@ -73,19 +75,22 @@ struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*f return RES(0, 4100); // internal_error ; we should never get here } -int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) { +int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), + FDBDatabase* db, + struct ResultSet* rs, + const char* kpiName) { int numRuns = 25; - int *results = malloc(sizeof(int)*numRuns); + int* results = malloc(sizeof(int) * numRuns); int i = 0; - for(; i < numRuns; ++i) { + for (; i < numRuns; ++i) { struct RunResult res = run(rs, db, testFxn); - if(res.e) { + if (res.e) { logError(res.e, kpiName, rs); free(results); return 0; } results[i] = res.res; - if(results[i] < 0) { + if (results[i] < 0) { free(results); return -1; } @@ -99,19 +104,22 @@ int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDB return result; } -int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) { +int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), + FDBDatabase* db, + struct ResultSet* rs, + const char* kpiName) { int numRuns = 25; - int *results = malloc(sizeof(int)*numRuns); + int* results = malloc(sizeof(int) * numRuns); int i = 0; - for(; i < numRuns; ++i) { + for (; i < numRuns; ++i) { struct RunResult res = testFxn(rs, db); - if(res.e) { + if (res.e) { logError(res.e, kpiName, rs); free(results); return 0; } results[i] = res.res; - if(results[i] < 0) { + if (results[i] < 0) { free(results); return -1; } @@ -125,139 +133,144 @@ int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBD return result; } - -struct RunResult clearAll(struct ResultSet *rs, FDBTransaction *tr) { +struct RunResult clearAll(struct ResultSet* rs, FDBTransaction* tr) { fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1); return RES(0, 0); } uint32_t start = 0; uint32_t stop = 0; -struct RunResult insertRange(struct ResultSet *rs, FDBTransaction *tr) { +struct RunResult insertRange(struct ResultSet* rs, FDBTransaction* tr) { int i; - for(i = start; i < stop; i++) { + for (i = start; i < stop; i++) { fdb_transaction_set(tr, keys[i], keySize, valueStr, valueSize); } return RES(0, 0); } -void insertData(struct ResultSet *rs, FDBDatabase *db) { +void insertData(struct ResultSet* rs, FDBDatabase* db) { checkError(run(rs, db, &clearAll).e, "clearing database", rs); // TODO: Do this asynchronously. start = 0; - while(start < numKeys) { + while (start < numKeys) { stop = start + 1000; - if(stop > numKeys) stop = numKeys; + if (stop > numKeys) + stop = numKeys; checkError(run(rs, db, &insertRange).e, "inserting data range", rs); start = stop; } } -fdb_error_t setRetryLimit(struct ResultSet *rs, FDBTransaction *tr, uint64_t limit) { +fdb_error_t setRetryLimit(struct ResultSet* rs, FDBTransaction* tr, uint64_t limit) { return fdb_transaction_set_option(tr, FDB_TR_OPTION_RETRY_LIMIT, (const uint8_t*)&limit, sizeof(uint64_t)); } uint32_t FUTURE_LATENCY_COUNT = 100000; -const char *FUTURE_LATENCY_KPI = "C future throughput (local client)"; -struct RunResult futureLatency(struct ResultSet *rs, FDBTransaction *tr) { +const char* FUTURE_LATENCY_KPI = "C future throughput (local client)"; +struct RunResult futureLatency(struct ResultSet* rs, FDBTransaction* tr) { fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); - FDBFuture *f = fdb_transaction_get_read_version(tr); + FDBFuture* f = fdb_transaction_get_read_version(tr); e = waitError(f); fdb_future_destroy(f); maybeLogError(e, "getting initial read version", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); double start = getTime(); int i; - for(i = 0; i < FUTURE_LATENCY_COUNT; i++) { - FDBFuture *f = fdb_transaction_get_read_version(tr); + for (i = 0; i < FUTURE_LATENCY_COUNT; i++) { + FDBFuture* f = fdb_transaction_get_read_version(tr); e = waitError(f); fdb_future_destroy(f); maybeLogError(e, "getting read version", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); } double end = getTime(); - return RES(FUTURE_LATENCY_COUNT/(end - start), 0); + return RES(FUTURE_LATENCY_COUNT / (end - start), 0); } uint32_t CLEAR_COUNT = 100000; -const char *CLEAR_KPI = "C clear throughput (local client)"; -struct RunResult clear(struct ResultSet *rs, FDBTransaction *tr) { +const char* CLEAR_KPI = "C clear throughput (local client)"; +struct RunResult clear(struct ResultSet* rs, FDBTransaction* tr) { double start = getTime(); int i; - for(i = 0; i < CLEAR_COUNT; i++) { + for (i = 0; i < CLEAR_COUNT; i++) { int k = ((uint64_t)rand()) % numKeys; fdb_transaction_clear(tr, keys[k], keySize); } double end = getTime(); fdb_transaction_reset(tr); // Don't actually clear things. - return RES(CLEAR_COUNT/(end - start), 0); + return RES(CLEAR_COUNT / (end - start), 0); } uint32_t CLEAR_RANGE_COUNT = 100000; -const char *CLEAR_RANGE_KPI = "C clear range throughput (local client)"; -struct RunResult clearRange(struct ResultSet *rs, FDBTransaction *tr) { +const char* CLEAR_RANGE_KPI = "C clear range throughput (local client)"; +struct RunResult clearRange(struct ResultSet* rs, FDBTransaction* tr) { double start = getTime(); int i; - for(i = 0; i < CLEAR_RANGE_COUNT; i++) { + for (i = 0; i < CLEAR_RANGE_COUNT; i++) { int k = ((uint64_t)rand()) % (numKeys - 1); - fdb_transaction_clear_range(tr, keys[k], keySize, keys[k+1], keySize); + fdb_transaction_clear_range(tr, keys[k], keySize, keys[k + 1], keySize); } double end = getTime(); fdb_transaction_reset(tr); // Don't actually clear things. - return RES(CLEAR_RANGE_COUNT/(end - start), 0); + return RES(CLEAR_RANGE_COUNT / (end - start), 0); } uint32_t SET_COUNT = 100000; -const char *SET_KPI = "C set throughput (local client)"; -struct RunResult set(struct ResultSet *rs, FDBTransaction *tr) { +const char* SET_KPI = "C set throughput (local client)"; +struct RunResult set(struct ResultSet* rs, FDBTransaction* tr) { double start = getTime(); int i; - for(i = 0; i < SET_COUNT; i++) { + for (i = 0; i < SET_COUNT; i++) { int k = ((uint64_t)rand()) % numKeys; fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize); } double end = getTime(); fdb_transaction_reset(tr); // Don't actually set things. - return RES(SET_COUNT/(end - start), 0); + return RES(SET_COUNT / (end - start), 0); } uint32_t PARALLEL_GET_COUNT = 10000; -const char *PARALLEL_GET_KPI = "C parallel get throughput (local client)"; -struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) { +const char* PARALLEL_GET_KPI = "C parallel get throughput (local client)"; +struct RunResult parallelGet(struct ResultSet* rs, FDBTransaction* tr) { fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); - FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT); + FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT); double start = getTime(); int i; - for(i = 0; i < PARALLEL_GET_COUNT; i++) { + for (i = 0; i < PARALLEL_GET_COUNT; i++) { int k = ((uint64_t)rand()) % numKeys; futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0); } fdb_bool_t present; - uint8_t const *outValue; + uint8_t const* outValue; int outValueLength; - for(i = 0; i < PARALLEL_GET_COUNT; i++) { + for (i = 0; i < PARALLEL_GET_COUNT; i++) { e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs); - if(e) { + if (e) { fdb_future_destroy(futures[i]); return RES(0, e); } - e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs); - if(e) { + e = maybeLogError( + fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs); + if (e) { fdb_future_destroy(futures[i]); return RES(0, e); } @@ -268,39 +281,41 @@ struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) { double end = getTime(); free(futures); - return RES(PARALLEL_GET_COUNT/(end - start), 0); + return RES(PARALLEL_GET_COUNT / (end - start), 0); } uint32_t ALTERNATING_GET_SET_COUNT = 2000; -const char *ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)"; -struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) { +const char* ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)"; +struct RunResult alternatingGetSet(struct ResultSet* rs, FDBTransaction* tr) { fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); - FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT); + FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT); double start = getTime(); int i; - for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) { + for (i = 0; i < ALTERNATING_GET_SET_COUNT; i++) { int k = ((uint64_t)rand()) % numKeys; fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize); futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0); } fdb_bool_t present; - uint8_t const *outValue; + uint8_t const* outValue; int outValueLength; - for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) { + for (i = 0; i < ALTERNATING_GET_SET_COUNT; i++) { e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs); - if(e) { + if (e) { fdb_future_destroy(futures[i]); return RES(0, e); } - e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs); - if(e) { + e = maybeLogError( + fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs); + if (e) { fdb_future_destroy(futures[i]); return RES(0, e); } @@ -311,38 +326,39 @@ struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) { double end = getTime(); free(futures); - return RES(ALTERNATING_GET_SET_COUNT/(end - start), 0); + return RES(ALTERNATING_GET_SET_COUNT / (end - start), 0); } uint32_t SERIAL_GET_COUNT = 2000; -const char *SERIAL_GET_KPI = "C serial get throughput (local client)"; -struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) { +const char* SERIAL_GET_KPI = "C serial get throughput (local client)"; +struct RunResult serialGet(struct ResultSet* rs, FDBTransaction* tr) { fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); int i; - uint32_t *keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT); + uint32_t* keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT); - if(SERIAL_GET_COUNT > numKeys/2) { - for(i = 0; i < SERIAL_GET_COUNT; i++) { + if (SERIAL_GET_COUNT > numKeys / 2) { + for (i = 0; i < SERIAL_GET_COUNT; i++) { keyIndices[i] = ((uint64_t)rand()) % numKeys; } } else { - for(i = 0; i < SERIAL_GET_COUNT; i++) { - while(1) { + for (i = 0; i < SERIAL_GET_COUNT; i++) { + while (1) { // Yes, this is a linear scan. This happens outside // the part we are measuring. uint32_t index = ((uint64_t)rand()) % numKeys; int j; fdb_bool_t found = 0; - for(j = 0; j < i; j++) { - if(keyIndices[j] == index) { + for (j = 0; j < i; j++) { + if (keyIndices[j] == index) { found = 1; break; } } - if(!found) { + if (!found) { keyIndices[i] = index; break; } @@ -353,13 +369,13 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) { double start = getTime(); fdb_bool_t present; - uint8_t const *outValue; + uint8_t const* outValue; int outValueLength; - for(i = 0; i < SERIAL_GET_COUNT; i++) { - FDBFuture *f = fdb_transaction_get(tr, keys[keyIndices[i]], keySize, 0); + for (i = 0; i < SERIAL_GET_COUNT; i++) { + FDBFuture* f = fdb_transaction_get(tr, keys[keyIndices[i]], keySize, 0); fdb_error_t e = maybeLogError(fdb_future_block_until_ready(f), "getting key in serial", rs); - if(e) { + if (e) { free(keyIndices); fdb_future_destroy(f); return RES(0, e); @@ -367,7 +383,7 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) { e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs); fdb_future_destroy(f); - if(e) { + if (e) { free(keyIndices); return RES(0, e); } @@ -376,66 +392,87 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) { double end = getTime(); free(keyIndices); - return RES(SERIAL_GET_COUNT/(end - start), 0); + return RES(SERIAL_GET_COUNT / (end - start), 0); } uint32_t GET_RANGE_COUNT = 100000; -const char *GET_RANGE_KPI = "C get range throughput (local client)"; -struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) { +const char* GET_RANGE_KPI = "C get range throughput (local client)"; +struct RunResult getRange(struct ResultSet* rs, FDBTransaction* tr) { fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); uint32_t startKey = ((uint64_t)rand()) % (numKeys - GET_RANGE_COUNT - 1); double start = getTime(); - const FDBKeyValue *outKv; + const FDBKeyValue* outKv; int outCount; fdb_bool_t outMore = 1; int totalOut = 0; int iteration = 0; - FDBFuture *f = fdb_transaction_get_range(tr, - keys[startKey], keySize, 1, 0, - keys[startKey + GET_RANGE_COUNT], keySize, 1, 0, - 0, 0, - FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0); + FDBFuture* f = fdb_transaction_get_range(tr, + keys[startKey], + keySize, + 1, + 0, + keys[startKey + GET_RANGE_COUNT], + keySize, + 1, + 0, + 0, + 0, + FDB_STREAMING_MODE_WANT_ALL, + ++iteration, + 0, + 0); - while(outMore) { + while (outMore) { e = maybeLogError(fdb_future_block_until_ready(f), "getting range", rs); - if(e) { + if (e) { fdb_future_destroy(f); return RES(0, e); } e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading range array", rs); - if(e) { + if (e) { fdb_future_destroy(f); return RES(0, e); } totalOut += outCount; - if(outMore) { - FDBFuture *f2 = fdb_transaction_get_range(tr, - outKv[outCount - 1].key, outKv[outCount - 1].key_length, 1, 1, - keys[startKey + GET_RANGE_COUNT], keySize, 1, 0, - 0, 0, - FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0); + if (outMore) { + FDBFuture* f2 = fdb_transaction_get_range(tr, + outKv[outCount - 1].key, + outKv[outCount - 1].key_length, + 1, + 1, + keys[startKey + GET_RANGE_COUNT], + keySize, + 1, + 0, + 0, + 0, + FDB_STREAMING_MODE_WANT_ALL, + ++iteration, + 0, + 0); fdb_future_destroy(f); f = f2; } } - if(totalOut != GET_RANGE_COUNT) { - char *msg = (char*)malloc((sizeof(char)) * 200); + if (totalOut != GET_RANGE_COUNT) { + char* msg = (char*)malloc((sizeof(char)) * 200); sprintf(msg, "verifying out count (%d != %d)", totalOut, GET_RANGE_COUNT); logError(4100, msg, rs); free(msg); fdb_future_destroy(f); return RES(0, 4100); } - if(outMore) { + if (outMore) { logError(4100, "verifying no more in range", rs); fdb_future_destroy(f); return RES(0, 4100); @@ -444,84 +481,84 @@ struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) { double end = getTime(); - return RES(GET_RANGE_COUNT/(end - start), 0); + return RES(GET_RANGE_COUNT / (end - start), 0); } uint32_t GET_KEY_COUNT = 2000; -const char *GET_KEY_KPI = "C get key throughput (local client)"; -struct RunResult getKey(struct ResultSet *rs, FDBTransaction *tr) { +const char* GET_KEY_KPI = "C get key throughput (local client)"; +struct RunResult getKey(struct ResultSet* rs, FDBTransaction* tr) { fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); double start = getTime(); fdb_bool_t present; - uint8_t const *outValue; + uint8_t const* outValue; int outValueLength; int i; - for(i = 0; i < GET_KEY_COUNT; i++) { + for (i = 0; i < GET_KEY_COUNT; i++) { int key = ((uint64_t)rand()) % numKeys; int offset = (((uint64_t)rand()) % 21) - 10; - FDBFuture *f = fdb_transaction_get_key(tr, keys[key], keySize, 1, offset, 0); + FDBFuture* f = fdb_transaction_get_key(tr, keys[key], keySize, 1, offset, 0); e = maybeLogError(fdb_future_block_until_ready(f), "waiting for get key", rs); - if(e) { + if (e) { fdb_future_destroy(f); return RES(0, e); } e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs); fdb_future_destroy(f); - if(e) { + if (e) { return RES(0, e); } } double end = getTime(); - return RES(GET_KEY_COUNT/(end - start), 0); + return RES(GET_KEY_COUNT / (end - start), 0); } uint32_t GET_SINGLE_KEY_RANGE_COUNT = 2000; -const char *GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)"; -struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) { +const char* GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)"; +struct RunResult getSingleKeyRange(struct ResultSet* rs, FDBTransaction* tr) { fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); double start = getTime(); - const FDBKeyValue *outKv; + const FDBKeyValue* outKv; int outCount; fdb_bool_t outMore; int i; - for(i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) { + for (i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) { int key = ((uint64_t)rand()) % (numKeys - 1); - FDBFuture *f = fdb_transaction_get_range(tr, - keys[key], keySize, 1, 0, - keys[key + 1], keySize, 1, 0, - 2, 0, - FDB_STREAMING_MODE_EXACT, 1, 0, 0); + FDBFuture* f = fdb_transaction_get_range( + tr, keys[key], keySize, 1, 0, keys[key + 1], keySize, 1, 0, 2, 0, FDB_STREAMING_MODE_EXACT, 1, 0, 0); e = maybeLogError(fdb_future_block_until_ready(f), "waiting for single key range", rs); - if(e) { + if (e) { fdb_future_destroy(f); return RES(0, e); } - e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs); - if(e) { + e = maybeLogError( + fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs); + if (e) { fdb_future_destroy(f); return RES(0, e); } - if(outCount != 1) { + if (outCount != 1) { logError(4100, "more than one key returned in single key range read", rs); fdb_future_destroy(f); return RES(0, 4100); } - if(outMore) { + if (outMore) { logError(4100, "more keys to read in single key range read", rs); fdb_future_destroy(f); return RES(0, 4100); @@ -532,33 +569,34 @@ struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) { double end = getTime(); - return RES(GET_SINGLE_KEY_RANGE_COUNT/(end - start), 0); + return RES(GET_SINGLE_KEY_RANGE_COUNT / (end - start), 0); } -struct RunResult singleKey(struct ResultSet *rs, FDBTransaction *tr) { +struct RunResult singleKey(struct ResultSet* rs, FDBTransaction* tr) { int k = ((uint64_t)rand()) % numKeys; fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize); return RES(0, 0); } uint32_t WRITE_TRANSACTION_COUNT = 1000; -const char *WRITE_TRANSACTION_KPI = "C write_transaction throughput (local client)"; -struct RunResult writeTransaction(struct ResultSet *rs, FDBDatabase *db) { +const char* WRITE_TRANSACTION_KPI = "C write_transaction throughput (local client)"; +struct RunResult writeTransaction(struct ResultSet* rs, FDBDatabase* db) { double start = getTime(); int i; - for(i = 0; i < WRITE_TRANSACTION_COUNT; i++) { + for (i = 0; i < WRITE_TRANSACTION_COUNT; i++) { struct RunResult res = run(rs, db, &singleKey); - if(res.e) return res; + if (res.e) + return res; } double end = getTime(); - return RES(WRITE_TRANSACTION_COUNT/(end - start), 0); + return RES(WRITE_TRANSACTION_COUNT / (end - start), 0); } -void runTests(struct ResultSet *rs) { - FDBDatabase *db = openDatabase(rs, &netThread); +void runTests(struct ResultSet* rs) { + FDBDatabase* db = openDatabase(rs, &netThread); printf("Loading database...\n"); insertData(rs, db); @@ -600,15 +638,15 @@ void runTests(struct ResultSet *rs) { fdb_stop_network(); } -int main(int argc, char **argv) { +int main(int argc, char** argv) { srand(time(NULL)); - struct ResultSet *rs = newResultSet(); + struct ResultSet* rs = newResultSet(); checkError(fdb_select_api_version(620), "select API version", rs); printf("Running performance test at client version: %s\n", fdb_get_client_version()); - valueStr = (uint8_t*)malloc((sizeof(uint8_t))*valueSize); + valueStr = (uint8_t*)malloc((sizeof(uint8_t)) * valueSize); int i; - for(i = 0; i < valueSize; i++) { + for (i = 0; i < valueSize; i++) { valueStr[i] = (uint8_t)'x'; } diff --git a/bindings/c/test/ryw_benchmark.c b/bindings/c/test/ryw_benchmark.c index 1777150894..e4d7e48e75 100644 --- a/bindings/c/test/ryw_benchmark.c +++ b/bindings/c/test/ryw_benchmark.c @@ -34,23 +34,26 @@ int numKeys = 10000; int keySize = 16; uint8_t** keys; -void insertData(FDBTransaction *tr) { +void insertData(FDBTransaction* tr) { fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1); - uint8_t *v = (uint8_t*)"foo"; + uint8_t* v = (uint8_t*)"foo"; uint32_t i; - for(i = 0; i <= numKeys; ++i) { + for (i = 0; i <= numKeys; ++i) { fdb_transaction_set(tr, keys[i], keySize, v, 3); } } -int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction *tr, struct ResultSet *rs, const char *kpiName) { +int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), + FDBTransaction* tr, + struct ResultSet* rs, + const char* kpiName) { int numRuns = 25; - int *results = malloc(sizeof(int)*numRuns); + int* results = malloc(sizeof(int) * numRuns); int i = 0; - for(; i < numRuns; ++i) { + for (; i < numRuns; ++i) { results[i] = testFxn(tr, rs); - if(results[i] < 0) { + if (results[i] < 0) { free(results); return -1; } @@ -64,17 +67,19 @@ int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction * return result; } -int getSingle(FDBTransaction *tr, struct ResultSet *rs) { +int getSingle(FDBTransaction* tr, struct ResultSet* rs) { int present; - uint8_t const *value; + uint8_t const* value; int length; int i; double start = getTime(); - for(i = 0; i < numKeys; ++i) { - FDBFuture *f = fdb_transaction_get(tr, keys[5001], keySize, 0); - if(getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs)) return -1; - if(getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs)) return -1; + for (i = 0; i < numKeys; ++i) { + FDBFuture* f = fdb_transaction_get(tr, keys[5001], keySize, 0); + if (getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs)) + return -1; + if (getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs)) + return -1; fdb_future_destroy(f); } double end = getTime(); @@ -82,17 +87,19 @@ int getSingle(FDBTransaction *tr, struct ResultSet *rs) { return numKeys / (end - start); } -int getManySequential(FDBTransaction *tr, struct ResultSet *rs) { +int getManySequential(FDBTransaction* tr, struct ResultSet* rs) { int present; - uint8_t const *value; + uint8_t const* value; int length; int i; double start = getTime(); - for(i = 0; i < numKeys; ++i) { - FDBFuture *f = fdb_transaction_get(tr, keys[i], keySize, 0); - if(getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs)) return -1; - if(getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs)) return -1; + for (i = 0; i < numKeys; ++i) { + FDBFuture* f = fdb_transaction_get(tr, keys[i], keySize, 0); + if (getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs)) + return -1; + if (getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs)) + return -1; fdb_future_destroy(f); } double end = getTime(); @@ -100,20 +107,30 @@ int getManySequential(FDBTransaction *tr, struct ResultSet *rs) { return numKeys / (end - start); } -int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) { +int getRangeBasic(FDBTransaction* tr, struct ResultSet* rs) { int count; - const FDBKeyValue *kvs; + const FDBKeyValue* kvs; int more; int i; double start = getTime(); - for(i = 0; i < 100; ++i) { - FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0); + for (i = 0; i < 100; ++i) { + FDBFuture* f = fdb_transaction_get_range(tr, + FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), + FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), + numKeys, + 0, + 0, + 1, + 0, + 0); - if(getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs)) return -1; - if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs)) return -1; + if (getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs)) + return -1; + if (getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs)) + return -1; - if(count != numKeys) { + if (count != numKeys) { fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys); addError(rs, "GetRangeBasic bad count"); return -1; @@ -124,26 +141,37 @@ int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) { return 100 * numKeys / (end - start); } -int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) { +int singleClearGetRange(FDBTransaction* tr, struct ResultSet* rs) { int count; - const FDBKeyValue *kvs; + const FDBKeyValue* kvs; int more; int i; - for(i = 0; i < numKeys; i+=2) { + for (i = 0; i < numKeys; i += 2) { fdb_transaction_clear(tr, keys[i], keySize); } double start = getTime(); - for(i = 0; i < 100; ++i) { - FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0); + for (i = 0; i < 100; ++i) { + FDBFuture* f = fdb_transaction_get_range(tr, + FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), + FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), + numKeys, + 0, + 0, + 1, + 0, + 0); - if(getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs)) return -1; - if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs)) return -1; + if (getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs)) + return -1; + if (getError( + fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs)) + return -1; fdb_future_destroy(f); - if(count != numKeys/2) { + if (count != numKeys / 2) { fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys); addError(rs, "SingleClearGetRange bad count"); return -1; @@ -155,27 +183,38 @@ int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) { return 100 * numKeys / 2 / (end - start); } -int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) { +int clearRangeGetRange(FDBTransaction* tr, struct ResultSet* rs) { int count; - const FDBKeyValue *kvs; + const FDBKeyValue* kvs; int more; int i; - for(i = 0; i < numKeys; i+=4) { - fdb_transaction_clear_range(tr, keys[i], keySize, keys[i+1], keySize); + for (i = 0; i < numKeys; i += 4) { + fdb_transaction_clear_range(tr, keys[i], keySize, keys[i + 1], keySize); } double start = getTime(); - for(i = 0; i < 100; ++i) { - FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0); + for (i = 0; i < 100; ++i) { + FDBFuture* f = fdb_transaction_get_range(tr, + FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), + FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), + numKeys, + 0, + 0, + 1, + 0, + 0); - if(getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs)) return -1; - if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs)) return -1; + if (getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs)) + return -1; + if (getError( + fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs)) + return -1; fdb_future_destroy(f); - if(count != numKeys*3/4) { - fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys*3/4); + if (count != numKeys * 3 / 4) { + fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys * 3 / 4); addError(rs, "ClearRangeGetRange bad count"); return -1; } @@ -186,13 +225,13 @@ int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) { return 100 * numKeys * 3 / 4 / (end - start); } -int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) { +int interleavedSetsGets(FDBTransaction* tr, struct ResultSet* rs) { int present; - uint8_t const *value; + uint8_t const* value; int length; int i; - uint8_t *k = (uint8_t*)"foo"; + uint8_t* k = (uint8_t*)"foo"; uint8_t v[10]; int num = 1; @@ -200,10 +239,12 @@ int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) { sprintf((char*)v, "%d", num); fdb_transaction_set(tr, k, 3, v, strlen((char*)v)); - for(i = 0; i < 10000; ++i) { - FDBFuture *f = fdb_transaction_get(tr, k, 3, 0); - if(getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs)) return -1; - if(getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs)) return -1; + for (i = 0; i < 10000; ++i) { + FDBFuture* f = fdb_transaction_get(tr, k, 3, 0); + if (getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs)) + return -1; + if (getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs)) + return -1; fdb_future_destroy(f); sprintf((char*)v, "%d", ++num); @@ -214,13 +255,13 @@ int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) { return 10000 / (end - start); } -void runTests(struct ResultSet *rs) { - FDBDatabase *db = openDatabase(rs, &netThread); +void runTests(struct ResultSet* rs) { + FDBDatabase* db = openDatabase(rs, &netThread); - FDBTransaction *tr; + FDBTransaction* tr; checkError(fdb_database_create_transaction(db, &tr), "create transaction", rs); - FDBFuture *f = fdb_transaction_get_read_version(tr); + FDBFuture* f = fdb_transaction_get_read_version(tr); checkError(fdb_future_block_until_ready(f), "block for read version", rs); int64_t version; @@ -241,9 +282,9 @@ void runTests(struct ResultSet *rs) { fdb_stop_network(); } -int main(int argc, char **argv) { +int main(int argc, char** argv) { srand(time(NULL)); - struct ResultSet *rs = newResultSet(); + struct ResultSet* rs = newResultSet(); checkError(fdb_select_api_version(620), "select API version", rs); printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version()); @@ -255,4 +296,3 @@ int main(int argc, char **argv) { return 0; } - diff --git a/bindings/c/test/test.h b/bindings/c/test/test.h index 895691c265..699ffb9d05 100644 --- a/bindings/c/test/test.h +++ b/bindings/c/test/test.h @@ -38,27 +38,27 @@ double getTime() { static struct timeval tv; gettimeofday(&tv, NULL); - return tv.tv_usec/1000000.0 + tv.tv_sec; + return tv.tv_usec / 1000000.0 + tv.tv_sec; } -void writeKey(uint8_t **dest, int key, int keySize) { - *dest = (uint8_t*)malloc((sizeof(uint8_t))*keySize); +void writeKey(uint8_t** dest, int key, int keySize) { + *dest = (uint8_t*)malloc((sizeof(uint8_t)) * keySize); sprintf((char*)*dest, "%0*d", keySize, key); } -uint8_t **generateKeys(int numKeys, int keySize) { - uint8_t **keys = (uint8_t**)malloc(sizeof(uint8_t*)*(numKeys+1)); +uint8_t** generateKeys(int numKeys, int keySize) { + uint8_t** keys = (uint8_t**)malloc(sizeof(uint8_t*) * (numKeys + 1)); uint32_t i; - for(i = 0; i <= numKeys; ++i) { + for (i = 0; i <= numKeys; ++i) { writeKey(keys + i, i, keySize); } return keys; } -void freeKeys(uint8_t **keys, int numKeys) { +void freeKeys(uint8_t** keys, int numKeys) { uint32_t i; - for(i = 0; i < numKeys; i++) { + for (i = 0; i < numKeys; i++) { free(keys[i]); } free(keys); @@ -68,38 +68,39 @@ int cmpfunc(const void* a, const void* b) { return (*(int*)a - *(int*)b); } -int median(int *values, int length) { +int median(int* values, int length) { qsort(values, length, sizeof(int), cmpfunc); - return values[length/2]; + return values[length / 2]; } struct RunResult { int res; fdb_error_t e; }; -#define RES(x, y) (struct RunResult) { x, y } +#define RES(x, y) \ + (struct RunResult) { x, y } struct Kpi { - const char *name; + const char* name; int value; - const char *units; + const char* units; - struct Kpi *next; + struct Kpi* next; }; struct Error { - char *message; + char* message; - struct Error *next; + struct Error* next; }; struct ResultSet { - struct Kpi *kpis; - struct Error *errors; + struct Kpi* kpis; + struct Error* errors; }; struct ResultSet* newResultSet() { - struct ResultSet *rs = malloc(sizeof(struct ResultSet)); + struct ResultSet* rs = malloc(sizeof(struct ResultSet)); rs->kpis = NULL; rs->errors = NULL; @@ -107,8 +108,8 @@ struct ResultSet* newResultSet() { return rs; } -void addKpi(struct ResultSet *rs, const char *name, int value, const char *units) { - struct Kpi *k = malloc(sizeof(struct Kpi)); +void addKpi(struct ResultSet* rs, const char* name, int value, const char* units) { + struct Kpi* k = malloc(sizeof(struct Kpi)); k->name = name; k->value = value; k->units = units; @@ -116,20 +117,20 @@ void addKpi(struct ResultSet *rs, const char *name, int value, const char *units rs->kpis = k; } -void addError(struct ResultSet *rs, const char *message) { - struct Error *e = malloc(sizeof(struct Error)); - e->message = (char*)malloc(strlen(message)+1); +void addError(struct ResultSet* rs, const char* message) { + struct Error* e = malloc(sizeof(struct Error)); + e->message = (char*)malloc(strlen(message) + 1); strcpy(e->message, message); e->next = rs->errors; rs->errors = e; } -void writeResultSet(struct ResultSet *rs) { +void writeResultSet(struct ResultSet* rs) { uint64_t id = ((uint64_t)rand() << 32) + rand(); char name[100]; sprintf(name, "fdb-c_result-%" SCNu64 ".json", id); - FILE *fp = fopen(name, "w"); - if(!fp) { + FILE* fp = fopen(name, "w"); + if (!fp) { fprintf(stderr, "Could not open results file %s\n", name); exit(1); } @@ -137,10 +138,10 @@ void writeResultSet(struct ResultSet *rs) { fprintf(fp, "{\n"); fprintf(fp, "\t\"kpis\": {\n"); - struct Kpi *k = rs->kpis; - while(k != NULL) { + struct Kpi* k = rs->kpis; + while (k != NULL) { fprintf(fp, "\t\t\"%s\": { \"units\": \"%s\", \"value\": %d }", k->name, k->units, k->value); - if(k->next != NULL) { + if (k->next != NULL) { fprintf(fp, ","); } fprintf(fp, "\n"); @@ -150,10 +151,10 @@ void writeResultSet(struct ResultSet *rs) { fprintf(fp, "\t},\n"); fprintf(fp, "\t\"errors\": [\n"); - struct Error *e = rs->errors; - while(e != NULL) { + struct Error* e = rs->errors; + while (e != NULL) { fprintf(fp, "\t\t\"%s\"", e->message); - if(e->next != NULL) { + if (e->next != NULL) { fprintf(fp, ","); } fprintf(fp, "\n"); @@ -166,17 +167,17 @@ void writeResultSet(struct ResultSet *rs) { fclose(fp); } -void freeResultSet(struct ResultSet *rs) { - struct Kpi *k = rs->kpis; - while(k != NULL) { - struct Kpi *next = k->next; +void freeResultSet(struct ResultSet* rs) { + struct Kpi* k = rs->kpis; + while (k != NULL) { + struct Kpi* next = k->next; free(k); k = next; } - struct Error *e = rs->errors; - while(e != NULL) { - struct Error *next = e->next; + struct Error* e = rs->errors; + while (e != NULL) { + struct Error* next = e->next; free(e->message); free(e); e = next; @@ -185,12 +186,12 @@ void freeResultSet(struct ResultSet *rs) { free(rs); } -fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet *rs) { - if(err) { - char *msg = (char*)malloc(strlen(context) + 100); +fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet* rs) { + if (err) { + char* msg = (char*)malloc(strlen(context) + 100); sprintf(msg, "Error in %s: %s", context, fdb_get_error(err)); fprintf(stderr, "%s\n", msg); - if(rs != NULL) { + if (rs != NULL) { addError(rs, msg); } @@ -200,9 +201,9 @@ fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet *rs) return err; } -void checkError(fdb_error_t err, const char* context, struct ResultSet *rs) { - if(getError(err, context, rs)) { - if(rs != NULL) { +void checkError(fdb_error_t err, const char* context, struct ResultSet* rs) { + if (getError(err, context, rs)) { + if (rs != NULL) { writeResultSet(rs); freeResultSet(rs); } @@ -210,11 +211,11 @@ void checkError(fdb_error_t err, const char* context, struct ResultSet *rs) { } } -fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet *rs) { - char *msg = (char*)malloc(strlen(context) + 100); +fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet* rs) { + char* msg = (char*)malloc(strlen(context) + 100); sprintf(msg, "Error in %s: %s", context, fdb_get_error(err)); fprintf(stderr, "%s\n", msg); - if(rs != NULL) { + if (rs != NULL) { addError(rs, msg); } @@ -222,8 +223,8 @@ fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet *rs) return err; } -fdb_error_t maybeLogError(fdb_error_t err, const char* context, struct ResultSet *rs) { - if(err && !fdb_error_predicate( FDB_ERROR_PREDICATE_RETRYABLE, err ) ) { +fdb_error_t maybeLogError(fdb_error_t err, const char* context, struct ResultSet* rs) { + if (err && !fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, err)) { return logError(err, context, rs); } return err; @@ -234,11 +235,11 @@ void* runNetwork() { return NULL; } -FDBDatabase* openDatabase(struct ResultSet *rs, pthread_t *netThread) { +FDBDatabase* openDatabase(struct ResultSet* rs, pthread_t* netThread) { checkError(fdb_setup_network(), "setup network", rs); pthread_create(netThread, NULL, &runNetwork, NULL); - FDBDatabase *db; + FDBDatabase* db; checkError(fdb_create_database(NULL, &db), "create database", rs); return db; diff --git a/bindings/c/test/txn_size_test.c b/bindings/c/test/txn_size_test.c index 73ee9a82e6..b49a748a98 100644 --- a/bindings/c/test/txn_size_test.c +++ b/bindings/c/test/txn_size_test.c @@ -31,14 +31,14 @@ pthread_t netThread; const int numKeys = 100; uint8_t** keys = NULL; -#define KEY_SIZE 16 +#define KEY_SIZE 16 #define VALUE_SIZE 100 uint8_t valueStr[VALUE_SIZE]; fdb_error_t getSize(struct ResultSet* rs, FDBTransaction* tr, int64_t* out_size) { fdb_error_t e; FDBFuture* future = fdb_transaction_get_approximate_size(tr); - + e = maybeLogError(fdb_future_block_until_ready(future), "waiting for get future", rs); if (e) { fdb_future_destroy(future); @@ -55,11 +55,11 @@ fdb_error_t getSize(struct ResultSet* rs, FDBTransaction* tr, int64_t* out_size) return 0; } -void runTests(struct ResultSet *rs) { +void runTests(struct ResultSet* rs) { int64_t sizes[numKeys]; int i = 0, j = 0; - FDBDatabase *db = openDatabase(rs, &netThread); - FDBTransaction *tr = NULL; + FDBDatabase* db = openDatabase(rs, &netThread); + FDBTransaction* tr = NULL; fdb_error_t e = fdb_database_create_transaction(db, &tr); checkError(e, "create transaction", rs); memset(sizes, 0, numKeys * sizeof(uint32_t)); @@ -82,7 +82,7 @@ void runTests(struct ResultSet *rs) { printf("size %d: %u\n", i, sizes[i]); i++; - fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i+1], KEY_SIZE); + fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i + 1], KEY_SIZE); e = getSize(rs, tr, sizes + i); checkError(e, "transaction get size", rs); printf("size %d: %u\n", i, sizes[i]); @@ -94,9 +94,9 @@ void runTests(struct ResultSet *rs) { printf("Test passed!\n"); } -int main(int argc, char **argv) { +int main(int argc, char** argv) { srand(time(NULL)); - struct ResultSet *rs = newResultSet(); + struct ResultSet* rs = newResultSet(); checkError(fdb_select_api_version(620), "select API version", rs); printf("Running performance test at client version: %s\n", fdb_get_client_version()); diff --git a/bindings/c/test/workloads/SimpleWorkload.cpp b/bindings/c/test/workloads/SimpleWorkload.cpp index 3d61f85cb9..bce14f4115 100644 --- a/bindings/c/test/workloads/SimpleWorkload.cpp +++ b/bindings/c/test/workloads/SimpleWorkload.cpp @@ -104,7 +104,10 @@ struct SimpleWorkload : FDBWorkload { unsigned long from, to, lastTx = 0; std::unordered_map callbacks; - PopulateActor(const Callback& promise, SimpleWorkload& self, FDBDatabase* db, unsigned long from, + PopulateActor(const Callback& promise, + SimpleWorkload& self, + FDBDatabase* db, + unsigned long from, unsigned long to) : ActorBase(promise, self, db), from(from), to(to) { error = fdb_database_create_transaction(db, &tx); @@ -130,8 +133,11 @@ struct SimpleWorkload : FDBWorkload { for (; from < to && ops < self.insertsPerTx; ++ops, ++from) { std::string value = std::to_string(from); std::string key = KEY_PREFIX + value; - fdb_transaction_set(tx, reinterpret_cast(key.c_str()), key.size(), - reinterpret_cast(value.c_str()), value.size()); + fdb_transaction_set(tx, + reinterpret_cast(key.c_str()), + key.size(), + reinterpret_cast(value.c_str()), + value.size()); } lastTx = ops; auto commit_future = fdb_transaction_commit(tx); @@ -154,7 +160,8 @@ struct SimpleWorkload : FDBWorkload { run(); }, [this](fdb_error_t error) { - self.context->trace(FDBSeverity::Error, "AssertionFailure", + self.context->trace(FDBSeverity::Error, + "AssertionFailure", { { "Reason", "tx.onError failed" }, { "Error", std::string(fdb_get_error(error)) } }); self.success = false; @@ -230,7 +237,8 @@ struct SimpleWorkload : FDBWorkload { get(); }, [this](fdb_error_t) { - self.context->trace(FDBSeverity::Error, "AssertionFailure", + self.context->trace(FDBSeverity::Error, + "AssertionFailure", { { "Reason", "tx.onError failed" }, { "Error", std::string(fdb_get_error(error)) } }); self.success = false; @@ -260,8 +268,8 @@ struct SimpleWorkload : FDBWorkload { runFor = context->getOption("runFor", 10.0); auto err = fdb_select_api_version(620); if (err) { - context->trace(FDBSeverity::Info, "SelectAPIVersionFailed", - { { "Error", std::string(fdb_get_error(err)) } }); + context->trace( + FDBSeverity::Info, "SelectAPIVersionFailed", { { "Error", std::string(fdb_get_error(err)) } }); } return true; } diff --git a/bindings/c/test/workloads/workloads.cpp b/bindings/c/test/workloads/workloads.cpp index 498d58f2e8..1f01054e84 100644 --- a/bindings/c/test/workloads/workloads.cpp +++ b/bindings/c/test/workloads/workloads.cpp @@ -23,19 +23,19 @@ FDBWorkloadFactoryImpl::~FDBWorkloadFactoryImpl() {} std::map& FDBWorkloadFactoryImpl::factories() { - static std::map _factories; - return _factories; + static std::map _factories; + return _factories; } -std::shared_ptr FDBWorkloadFactoryImpl::create(const std::string &name) { - auto res = factories().find(name); - if (res == factories().end()) { - return nullptr; - } - return res->second->create(); +std::shared_ptr FDBWorkloadFactoryImpl::create(const std::string& name) { + auto res = factories().find(name); + if (res == factories().end()) { + return nullptr; + } + return res->second->create(); } FDBWorkloadFactory* workloadFactory(FDBLogger*) { - static FDBWorkloadFactoryImpl impl; - return &impl; + static FDBWorkloadFactoryImpl impl; + return &impl; } diff --git a/bindings/c/test/workloads/workloads.h b/bindings/c/test/workloads/workloads.h index d058922e68..b9666d80c2 100644 --- a/bindings/c/test/workloads/workloads.h +++ b/bindings/c/test/workloads/workloads.h @@ -33,15 +33,11 @@ struct FDBWorkloadFactoryImpl : FDBWorkloadFactory { std::shared_ptr create(const std::string& name) override; }; -template +template struct FDBWorkloadFactoryT : IFDBWorkloadFactory { - explicit FDBWorkloadFactoryT(const std::string& name) { - FDBWorkloadFactoryImpl::factories()[name] = this; - } + explicit FDBWorkloadFactoryT(const std::string& name) { FDBWorkloadFactoryImpl::factories()[name] = this; } - std::shared_ptr create() override { - return std::make_shared(); - } + std::shared_ptr create() override { return std::make_shared(); } }; extern "C" DLLEXPORT FDBWorkloadFactory* workloadFactory(FDBLogger*); diff --git a/bindings/flow/DirectoryLayer.actor.cpp b/bindings/flow/DirectoryLayer.actor.cpp index 1b80249773..3ef201456f 100644 --- a/bindings/flow/DirectoryLayer.actor.cpp +++ b/bindings/flow/DirectoryLayer.actor.cpp @@ -22,486 +22,541 @@ #include "DirectoryPartition.h" namespace FDB { - const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = {1,0,0,0,0,0,0,0}; - const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca"); - const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer"); - const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version"); - const int64_t DirectoryLayer::SUB_DIR_KEY = 0; +const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = { 1, 0, 0, 0, 0, 0, 0, 0 }; +const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca"); +const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer"); +const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version"); +const int64_t DirectoryLayer::SUB_DIR_KEY = 0; - const uint32_t DirectoryLayer::VERSION[3] = {1, 0, 0}; +const uint32_t DirectoryLayer::VERSION[3] = { 1, 0, 0 }; - const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = LiteralStringRef("\xfe"); - const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX); - const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace(); - const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition"); +const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = LiteralStringRef("\xfe"); +const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX); +const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace(); +const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition"); - DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes) : - nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes), - rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY)) - { } +DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes) + : nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes), + rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY)) {} - Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const { - return nodeSubspace.get(prefix); - } - - template - Optional DirectoryLayer::nodeWithPrefix(Optional const& prefix) const { - if(!prefix.present()) { - return Optional(); - } - - return nodeWithPrefix(prefix.get()); - } - - ACTOR Future find(Reference dirLayer, Reference tr, IDirectory::Path path) { - state int pathIndex = 0; - state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path); - - for(; pathIndex != path.size(); ++pathIndex) { - ASSERT(node.subspace.present()); - Optional> val = wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key())); - - node.path.push_back(path[pathIndex]); - node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path); - - DirectoryLayer::Node _node = wait(node.loadMetadata(tr)); - node = _node; - - if(!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) { - return node; - } - } - - if(!node.loadedMetadata) { - DirectoryLayer::Node _node = wait(node.loadMetadata(tr)); - node = _node; - } - - return node; - } - - IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const { - Path path; - - path.reserve(this->path.size() + subpath.size()); - path.insert(path.end(), this->path.begin(), this->path.end()); - path.insert(path.end(), subpath.begin(), subpath.end()); - - return path; - } - - Reference DirectoryLayer::contentsOfNode(Subspace const& node, Path const& path, Standalone const& layer) { - Standalone prefix = nodeSubspace.unpack(node.key()).getString(0); - - if(layer == PARTITION_LAYER) { - return Reference(new DirectoryPartition(toAbsolutePath(path), prefix, Reference::addRef(this))); - } - else { - return Reference(new DirectorySubspace(toAbsolutePath(path), prefix, Reference::addRef(this), layer)); - } - } - - Reference DirectoryLayer::openInternal(Standalone const& layer, Node const& existingNode, bool allowOpen) { - if (!allowOpen) { - throw directory_already_exists(); - } - if(layer.size() > 0 && layer != existingNode.layer) { - throw mismatched_layer(); - } - - return existingNode.getContents(); - } - - Future> DirectoryLayer::open(Reference const& tr, Path const& path, Standalone const& layer) { - return createOrOpenInternal(tr, path, layer, Optional>(), false, true); - } - - void DirectoryLayer::initializeDirectory(Reference const& tr) const { - tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12)); - } - - ACTOR Future checkVersionInternal(const DirectoryLayer* dirLayer, Reference tr, bool writeAccess) { - Optional> versionBytes = wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY))); - - if(!versionBytes.present()) { - if(writeAccess) { - dirLayer->initializeDirectory(tr); - } - return Void(); - } - else { - if(versionBytes.get().size() != 12) { - throw invalid_directory_layer_metadata(); - } - if(((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) { - throw incompatible_directory_version(); - } - else if(((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) { - throw incompatible_directory_version(); - } - } - - return Void(); - } - - Future DirectoryLayer::checkVersion(Reference const& tr, bool writeAccess) const { - return checkVersionInternal(this, tr, writeAccess); - } - - ACTOR Future> getPrefix(Reference dirLayer, Reference tr, Optional> prefix) { - if(!prefix.present()) { - Standalone allocated = wait(dirLayer->allocator.allocate(tr)); - state Standalone finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key()); - - FDBStandalone result = wait(tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1)); - - if(result.size() > 0) { - throw directory_prefix_not_empty(); - } - - return finalPrefix; - } - - return prefix.get(); - } - - ACTOR Future> nodeContainingKey(Reference dirLayer, Reference tr, Standalone key, bool snapshot) { - if(key.startsWith(dirLayer->nodeSubspace.key())) { - return dirLayer->rootNode; - } - - KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key))); - FDBStandalone result = wait(tr->getRange(range, 1, snapshot, true)); - - if(result.size() > 0) { - Standalone prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0); - if(key.startsWith(prevPrefix)) { - return dirLayer->nodeWithPrefix(prevPrefix); - } - } +Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const { + return nodeSubspace.get(prefix); +} +template +Optional DirectoryLayer::nodeWithPrefix(Optional const& prefix) const { + if (!prefix.present()) { return Optional(); } - ACTOR Future isPrefixFree(Reference dirLayer, Reference tr, Standalone prefix, bool snapshot){ - if(!prefix.size()) { - return false; - } + return nodeWithPrefix(prefix.get()); +} - Optional node = wait(nodeContainingKey(dirLayer, tr, prefix, snapshot)); - if(node.present()) { - return false; - } +ACTOR Future find(Reference dirLayer, + Reference tr, + IDirectory::Path path) { + state int pathIndex = 0; + state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path); - FDBStandalone result = wait(tr->getRange(KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot)); - return !result.size(); + for (; pathIndex != path.size(); ++pathIndex) { + ASSERT(node.subspace.present()); + Optional> val = + wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key())); - } + node.path.push_back(path[pathIndex]); + node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path); - ACTOR Future getParentNode(Reference dirLayer, Reference tr, IDirectory::Path path) { - if(path.size() > 1) { - Reference parent = wait(dirLayer->createOrOpenInternal(tr, IDirectory::Path(path.begin(), path.end() - 1), StringRef(), Optional>(), true, true)); - return dirLayer->nodeWithPrefix(parent->key()); - } - else { - return dirLayer->rootNode; + DirectoryLayer::Node _node = wait(node.loadMetadata(tr)); + node = _node; + + if (!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) { + return node; } } - ACTOR Future> createInternal( - Reference dirLayer, Reference tr, IDirectory::Path path, - Standalone layer, Optional> prefix, bool allowCreate) - { - if(!allowCreate) { - throw directory_does_not_exist(); - } - - wait(dirLayer->checkVersion(tr, true)); - - state Standalone newPrefix = wait(getPrefix(dirLayer, tr, prefix)); - bool isFree = wait(isPrefixFree(dirLayer, tr, newPrefix, !prefix.present())); - - if(!isFree) { - throw directory_prefix_in_use(); - } - - Subspace parentNode = wait(getParentNode(dirLayer, tr, path)); - Subspace node = dirLayer->nodeWithPrefix(newPrefix); - - tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix); - tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer); - return dirLayer->contentsOfNode(node, path, layer); + if (!node.loadedMetadata) { + DirectoryLayer::Node _node = wait(node.loadMetadata(tr)); + node = _node; } - ACTOR Future> _createOrOpenInternal( - Reference dirLayer, Reference tr, IDirectory::Path path, - Standalone layer, Optional> prefix, bool allowCreate, bool allowOpen) - { - ASSERT(!prefix.present() || allowCreate); - wait(dirLayer->checkVersion(tr, false)); + return node; +} - if(prefix.present() && !dirLayer->allowManualPrefixes) { - if(!dirLayer->getPath().size()) { - throw manual_prefixes_not_enabled(); - } - else { - throw prefix_in_partition(); - } +IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const { + Path path; + + path.reserve(this->path.size() + subpath.size()); + path.insert(path.end(), this->path.begin(), this->path.end()); + path.insert(path.end(), subpath.begin(), subpath.end()); + + return path; +} + +Reference DirectoryLayer::contentsOfNode(Subspace const& node, + Path const& path, + Standalone const& layer) { + Standalone prefix = nodeSubspace.unpack(node.key()).getString(0); + + if (layer == PARTITION_LAYER) { + return Reference( + new DirectoryPartition(toAbsolutePath(path), prefix, Reference::addRef(this))); + } else { + return Reference( + new DirectorySubspace(toAbsolutePath(path), prefix, Reference::addRef(this), layer)); + } +} + +Reference DirectoryLayer::openInternal(Standalone const& layer, + Node const& existingNode, + bool allowOpen) { + if (!allowOpen) { + throw directory_already_exists(); + } + if (layer.size() > 0 && layer != existingNode.layer) { + throw mismatched_layer(); + } + + return existingNode.getContents(); +} + +Future> DirectoryLayer::open(Reference const& tr, + Path const& path, + Standalone const& layer) { + return createOrOpenInternal(tr, path, layer, Optional>(), false, true); +} + +void DirectoryLayer::initializeDirectory(Reference const& tr) const { + tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12)); +} + +ACTOR Future checkVersionInternal(const DirectoryLayer* dirLayer, Reference tr, bool writeAccess) { + Optional> versionBytes = + wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY))); + + if (!versionBytes.present()) { + if (writeAccess) { + dirLayer->initializeDirectory(tr); + } + return Void(); + } else { + if (versionBytes.get().size() != 12) { + throw invalid_directory_layer_metadata(); + } + if (((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) { + throw incompatible_directory_version(); + } else if (((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) { + throw incompatible_directory_version(); + } + } + + return Void(); +} + +Future DirectoryLayer::checkVersion(Reference const& tr, bool writeAccess) const { + return checkVersionInternal(this, tr, writeAccess); +} + +ACTOR Future> getPrefix(Reference dirLayer, + Reference tr, + Optional> prefix) { + if (!prefix.present()) { + Standalone allocated = wait(dirLayer->allocator.allocate(tr)); + state Standalone finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key()); + + FDBStandalone result = wait(tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1)); + + if (result.size() > 0) { + throw directory_prefix_not_empty(); } - if(!path.size()){ - throw cannot_open_root_directory(); - } + return finalPrefix; + } - state DirectoryLayer::Node existingNode = wait(find(dirLayer, tr, path)); - if(existingNode.exists()) { - if(existingNode.isInPartition()) { - IDirectory::Path subpath = existingNode.getPartitionSubpath(); - Reference dirSpace = wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(tr, subpath, layer, prefix, allowCreate, allowOpen)); - return dirSpace; - } - return dirLayer->openInternal(layer, existingNode, allowOpen); + return prefix.get(); +} + +ACTOR Future> nodeContainingKey(Reference dirLayer, + Reference tr, + Standalone key, + bool snapshot) { + if (key.startsWith(dirLayer->nodeSubspace.key())) { + return dirLayer->rootNode; + } + + KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key))); + FDBStandalone result = wait(tr->getRange(range, 1, snapshot, true)); + + if (result.size() > 0) { + Standalone prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0); + if (key.startsWith(prevPrefix)) { + return dirLayer->nodeWithPrefix(prevPrefix); } - else { - Reference dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate)); + } + + return Optional(); +} + +ACTOR Future isPrefixFree(Reference dirLayer, + Reference tr, + Standalone prefix, + bool snapshot) { + if (!prefix.size()) { + return false; + } + + Optional node = wait(nodeContainingKey(dirLayer, tr, prefix, snapshot)); + if (node.present()) { + return false; + } + + FDBStandalone result = wait(tr->getRange( + KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot)); + return !result.size(); +} + +ACTOR Future getParentNode(Reference dirLayer, + Reference tr, + IDirectory::Path path) { + if (path.size() > 1) { + Reference parent = + wait(dirLayer->createOrOpenInternal(tr, + IDirectory::Path(path.begin(), path.end() - 1), + StringRef(), + Optional>(), + true, + true)); + return dirLayer->nodeWithPrefix(parent->key()); + } else { + return dirLayer->rootNode; + } +} + +ACTOR Future> createInternal(Reference dirLayer, + Reference tr, + IDirectory::Path path, + Standalone layer, + Optional> prefix, + bool allowCreate) { + if (!allowCreate) { + throw directory_does_not_exist(); + } + + wait(dirLayer->checkVersion(tr, true)); + + state Standalone newPrefix = wait(getPrefix(dirLayer, tr, prefix)); + bool isFree = wait(isPrefixFree(dirLayer, tr, newPrefix, !prefix.present())); + + if (!isFree) { + throw directory_prefix_in_use(); + } + + Subspace parentNode = wait(getParentNode(dirLayer, tr, path)); + Subspace node = dirLayer->nodeWithPrefix(newPrefix); + + tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix); + tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer); + return dirLayer->contentsOfNode(node, path, layer); +} + +ACTOR Future> _createOrOpenInternal(Reference dirLayer, + Reference tr, + IDirectory::Path path, + Standalone layer, + Optional> prefix, + bool allowCreate, + bool allowOpen) { + ASSERT(!prefix.present() || allowCreate); + wait(dirLayer->checkVersion(tr, false)); + + if (prefix.present() && !dirLayer->allowManualPrefixes) { + if (!dirLayer->getPath().size()) { + throw manual_prefixes_not_enabled(); + } else { + throw prefix_in_partition(); + } + } + + if (!path.size()) { + throw cannot_open_root_directory(); + } + + state DirectoryLayer::Node existingNode = wait(find(dirLayer, tr, path)); + if (existingNode.exists()) { + if (existingNode.isInPartition()) { + IDirectory::Path subpath = existingNode.getPartitionSubpath(); + Reference dirSpace = + wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal( + tr, subpath, layer, prefix, allowCreate, allowOpen)); return dirSpace; } + return dirLayer->openInternal(layer, existingNode, allowOpen); + } else { + Reference dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate)); + return dirSpace; + } +} + +Future> DirectoryLayer::createOrOpenInternal(Reference const& tr, + Path const& path, + Standalone const& layer, + Optional> const& prefix, + bool allowCreate, + bool allowOpen) { + return _createOrOpenInternal( + Reference::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen); +} + +Future> DirectoryLayer::create(Reference const& tr, + Path const& path, + Standalone const& layer, + Optional> const& prefix) { + return createOrOpenInternal(tr, path, layer, prefix, true, false); +} + +Future> DirectoryLayer::createOrOpen(Reference const& tr, + Path const& path, + Standalone const& layer) { + return createOrOpenInternal(tr, path, layer, Optional>(), true, true); +} + +ACTOR Future>> listInternal(Reference dirLayer, + Reference tr, + IDirectory::Path path) { + wait(dirLayer->checkVersion(tr, false)); + + state DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); + + if (!node.exists()) { + throw directory_does_not_exist(); + } + if (node.isInPartition(true)) { + Standalone> partitionList = + wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath())); + return partitionList; } - Future> DirectoryLayer::createOrOpenInternal( - Reference const& tr, Path const& path, Standalone const& layer, - Optional> const& prefix, bool allowCreate, bool allowOpen) - { - return _createOrOpenInternal(Reference::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen); - } + state Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY); + state Key begin = subdir.range().begin; + state Standalone> subdirectories; - Future> DirectoryLayer::create( - Reference const& tr, Path const& path, Standalone const& layer, - Optional> const& prefix) - { - return createOrOpenInternal(tr, path, layer, prefix, true, false); - } + loop { + FDBStandalone subdirRange = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end))); - Future> DirectoryLayer::createOrOpen(Reference const& tr, Path const& path, Standalone const& layer) { - return createOrOpenInternal(tr, path, layer, Optional>(), true, true); - } - - ACTOR Future>> listInternal(Reference dirLayer, Reference tr, IDirectory::Path path) { - wait(dirLayer->checkVersion(tr, false)); - - state DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); - - if(!node.exists()) { - throw directory_does_not_exist(); - } - if(node.isInPartition(true)) { - Standalone> partitionList = wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath())); - return partitionList; + for (int i = 0; i < subdirRange.size(); ++i) { + subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0)); } - state Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY); - state Key begin = subdir.range().begin; - state Standalone> subdirectories; - - loop { - FDBStandalone subdirRange = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end))); - - for(int i = 0; i < subdirRange.size(); ++i) { - subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0)); - } - - if(!subdirRange.more) { - return subdirectories; - } - - begin = keyAfter(subdirRange.back().key); + if (!subdirRange.more) { + return subdirectories; } - } - Future>> DirectoryLayer::list(Reference const& tr, Path const& path) { - return listInternal(Reference::addRef(this), tr, path); + begin = keyAfter(subdirRange.back().key); } +} - bool pathsEqual(IDirectory::Path const& path1, IDirectory::Path const& path2, size_t maxElementsToCheck = std::numeric_limits::max()) { - if(std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) { +Future>> DirectoryLayer::list(Reference const& tr, Path const& path) { + return listInternal(Reference::addRef(this), tr, path); +} + +bool pathsEqual(IDirectory::Path const& path1, + IDirectory::Path const& path2, + size_t maxElementsToCheck = std::numeric_limits::max()) { + if (std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) { + return false; + } + for (int i = 0; i < path1.size() && i < maxElementsToCheck; ++i) { + if (path1[i] != path2[i]) { return false; } - for(int i = 0; i < path1.size() && i < maxElementsToCheck; ++i) { - if(path1[i] != path2[i]) { - return false; - } - } - - return true; } - ACTOR Future removeFromParent(Reference dirLayer, Reference tr, IDirectory::Path path) { - ASSERT(path.size() >= 1); - DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1))); - if(parentNode.subspace.present()) { - tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key()); - } + return true; +} - return Void(); +ACTOR Future removeFromParent(Reference dirLayer, + Reference tr, + IDirectory::Path path) { + ASSERT(path.size() >= 1); + DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1))); + if (parentNode.subspace.present()) { + tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key()); } - ACTOR Future> moveInternal(Reference dirLayer, Reference tr, IDirectory::Path oldPath, IDirectory::Path newPath) { - wait(dirLayer->checkVersion(tr, true)); + return Void(); +} - if(oldPath.size() <= newPath.size()) { - if(pathsEqual(oldPath, newPath, oldPath.size())) { - throw invalid_destination_directory(); - } +ACTOR Future> moveInternal(Reference dirLayer, + Reference tr, + IDirectory::Path oldPath, + IDirectory::Path newPath) { + wait(dirLayer->checkVersion(tr, true)); + + if (oldPath.size() <= newPath.size()) { + if (pathsEqual(oldPath, newPath, oldPath.size())) { + throw invalid_destination_directory(); } - - std::vector> futures; - futures.push_back(find(dirLayer, tr, oldPath)); - futures.push_back(find(dirLayer, tr, newPath)); - - std::vector nodes = wait(getAll(futures)); - - state DirectoryLayer::Node oldNode = nodes[0]; - state DirectoryLayer::Node newNode = nodes[1]; - - if(!oldNode.exists()) { - throw directory_does_not_exist(); - } - - if(oldNode.isInPartition() || newNode.isInPartition()) { - if(!oldNode.isInPartition() || !newNode.isInPartition() || !pathsEqual(oldNode.path, newNode.path)) { - throw cannot_move_directory_between_partitions(); - } - - Reference partitionMove = wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath())); - return partitionMove; - } - - if(newNode.exists() || newPath.empty()) { - throw directory_already_exists(); - } - - DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1))); - if(!parentNode.exists()) { - throw parent_directory_does_not_exist(); - } - - tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(), dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0)); - wait(removeFromParent(dirLayer, tr, oldPath)); - - return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer); } - Future> DirectoryLayer::move(Reference const& tr, Path const& oldPath, Path const& newPath) { - return moveInternal(Reference::addRef(this), tr, oldPath, newPath); + std::vector> futures; + futures.push_back(find(dirLayer, tr, oldPath)); + futures.push_back(find(dirLayer, tr, newPath)); + + std::vector nodes = wait(getAll(futures)); + + state DirectoryLayer::Node oldNode = nodes[0]; + state DirectoryLayer::Node newNode = nodes[1]; + + if (!oldNode.exists()) { + throw directory_does_not_exist(); } - Future> DirectoryLayer::moveTo(Reference const& tr, Path const& newAbsolutePath) { + if (oldNode.isInPartition() || newNode.isInPartition()) { + if (!oldNode.isInPartition() || !newNode.isInPartition() || !pathsEqual(oldNode.path, newNode.path)) { + throw cannot_move_directory_between_partitions(); + } + + Reference partitionMove = + wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath())); + return partitionMove; + } + + if (newNode.exists() || newPath.empty()) { + throw directory_already_exists(); + } + + DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1))); + if (!parentNode.exists()) { + throw parent_directory_does_not_exist(); + } + + tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(), + dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0)); + wait(removeFromParent(dirLayer, tr, oldPath)); + + return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer); +} + +Future> DirectoryLayer::move(Reference const& tr, + Path const& oldPath, + Path const& newPath) { + return moveInternal(Reference::addRef(this), tr, oldPath, newPath); +} + +Future> DirectoryLayer::moveTo(Reference const& tr, + Path const& newAbsolutePath) { + throw cannot_modify_root_directory(); +} + +Future removeRecursive(Reference const&, Reference const&, Subspace const&); +ACTOR Future removeRecursive(Reference dirLayer, Reference tr, Subspace nodeSub) { + state Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY); + state Key begin = subdir.range().begin; + state std::vector> futures; + + loop { + FDBStandalone range = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end))); + for (int i = 0; i < range.size(); ++i) { + Subspace subNode = dirLayer->nodeWithPrefix(range[i].value); + futures.push_back(removeRecursive(dirLayer, tr, subNode)); + } + + if (!range.more) { + break; + } + + begin = keyAfter(range.back().key); + } + + // waits are done concurrently + wait(waitForAll(futures)); + + Standalone nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0); + + tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix))); + tr->clear(nodeSub.range()); + + return Void(); +} + +Future removeInternal(Reference const&, + Reference const&, + IDirectory::Path const&, + bool const&); +ACTOR Future removeInternal(Reference dirLayer, + Reference tr, + IDirectory::Path path, + bool failOnNonexistent) { + wait(dirLayer->checkVersion(tr, true)); + + if (path.empty()) { throw cannot_modify_root_directory(); } - Future removeRecursive(Reference const&, Reference const&, Subspace const&); - ACTOR Future removeRecursive(Reference dirLayer, Reference tr, Subspace nodeSub) { - state Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY); - state Key begin = subdir.range().begin; - state std::vector> futures; + state DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); - loop { - FDBStandalone range = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end))); - for (int i = 0; i < range.size(); ++i) { - Subspace subNode = dirLayer->nodeWithPrefix(range[i].value); - futures.push_back(removeRecursive(dirLayer, tr, subNode)); - } - - if(!range.more) { - break; - } - - begin = keyAfter(range.back().key); - } - - // waits are done concurrently - wait(waitForAll(futures)); - - Standalone nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0); - - tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix))); - tr->clear(nodeSub.range()); - - return Void(); - } - - Future removeInternal(Reference const&, Reference const&, IDirectory::Path const&, bool const&); - ACTOR Future removeInternal(Reference dirLayer, Reference tr, IDirectory::Path path, bool failOnNonexistent) { - wait(dirLayer->checkVersion(tr, true)); - - if(path.empty()) { - throw cannot_modify_root_directory(); - } - - state DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); - - if(!node.exists()) { - if(failOnNonexistent) { - throw directory_does_not_exist(); - } - else { - return false; - } - } - - if(node.isInPartition()) { - bool recurse = wait(removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent)); - return recurse; - } - - - state std::vector> futures; - futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get())); - futures.push_back(removeFromParent(dirLayer, tr, path)); - - wait(waitForAll(futures)); - - return true; - } - - Future DirectoryLayer::remove(Reference const& tr, Path const& path) { - return success(removeInternal(Reference::addRef(this), tr, path, true)); - } - - Future DirectoryLayer::removeIfExists(Reference const& tr, Path const& path) { - return removeInternal(Reference::addRef(this), tr, path, false); - } - - ACTOR Future existsInternal(Reference dirLayer, Reference tr, IDirectory::Path path) { - wait(dirLayer->checkVersion(tr, false)); - - DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); - - if(!node.exists()) { + if (!node.exists()) { + if (failOnNonexistent) { + throw directory_does_not_exist(); + } else { return false; } - - if(node.isInPartition()) { - bool exists = wait(node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath())); - return exists; - } - - return true; } - Future DirectoryLayer::exists(Reference const& tr, Path const& path) { - return existsInternal(Reference::addRef(this), tr, path); + if (node.isInPartition()) { + bool recurse = wait( + removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent)); + return recurse; } - Reference DirectoryLayer::getDirectoryLayer() { - return Reference::addRef(this); - } + state std::vector> futures; + futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get())); + futures.push_back(removeFromParent(dirLayer, tr, path)); - const Standalone DirectoryLayer::getLayer() const { - return StringRef(); - } + wait(waitForAll(futures)); - const IDirectory::Path DirectoryLayer::getPath() const { - return path; - } + return true; } + +Future DirectoryLayer::remove(Reference const& tr, Path const& path) { + return success(removeInternal(Reference::addRef(this), tr, path, true)); +} + +Future DirectoryLayer::removeIfExists(Reference const& tr, Path const& path) { + return removeInternal(Reference::addRef(this), tr, path, false); +} + +ACTOR Future existsInternal(Reference dirLayer, + Reference tr, + IDirectory::Path path) { + wait(dirLayer->checkVersion(tr, false)); + + DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); + + if (!node.exists()) { + return false; + } + + if (node.isInPartition()) { + bool exists = wait(node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath())); + return exists; + } + + return true; +} + +Future DirectoryLayer::exists(Reference const& tr, Path const& path) { + return existsInternal(Reference::addRef(this), tr, path); +} + +Reference DirectoryLayer::getDirectoryLayer() { + return Reference::addRef(this); +} + +const Standalone DirectoryLayer::getLayer() const { + return StringRef(); +} + +const IDirectory::Path DirectoryLayer::getPath() const { + return path; +} +} // namespace FDB diff --git a/bindings/flow/DirectoryLayer.h b/bindings/flow/DirectoryLayer.h old mode 100755 new mode 100644 index 36ef61c9fb..f5b87de263 --- a/bindings/flow/DirectoryLayer.h +++ b/bindings/flow/DirectoryLayer.h @@ -28,84 +28,108 @@ #include "HighContentionAllocator.h" namespace FDB { - class DirectoryLayer : public IDirectory { - public: - DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE, Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE, bool allowManualPrefixes = false); +class DirectoryLayer : public IDirectory { +public: + DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE, + Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE, + bool allowManualPrefixes = false); - Future> create(Reference const& tr, Path const& path, Standalone const& layer = Standalone(), Optional> const& prefix = Optional>()); - Future> open(Reference const& tr, Path const& path, Standalone const& layer = Standalone()); - Future> createOrOpen(Reference const& tr, Path const& path, Standalone const& layer = Standalone()); + Future> create( + Reference const& tr, + Path const& path, + Standalone const& layer = Standalone(), + Optional> const& prefix = Optional>()); + Future> open(Reference const& tr, + Path const& path, + Standalone const& layer = Standalone()); + Future> createOrOpen(Reference const& tr, + Path const& path, + Standalone const& layer = Standalone()); - Future exists(Reference const& tr, Path const& path = Path()); - Future>> list(Reference const& tr, Path const& path = Path()); + Future exists(Reference const& tr, Path const& path = Path()); + Future>> list(Reference const& tr, Path const& path = Path()); - Future> move(Reference const& tr, Path const& oldPath, Path const& newPath); - Future> moveTo(Reference const& tr, Path const& newAbsolutePath); + Future> move(Reference const& tr, + Path const& oldPath, + Path const& newPath); + Future> moveTo(Reference const& tr, Path const& newAbsolutePath); - Future remove(Reference const& tr, Path const& path = Path()); - Future removeIfExists(Reference const& tr, Path const& path = Path()); + Future remove(Reference const& tr, Path const& path = Path()); + Future removeIfExists(Reference const& tr, Path const& path = Path()); - Reference getDirectoryLayer(); - const Standalone getLayer() const; - const Path getPath() const; + Reference getDirectoryLayer(); + const Standalone getLayer() const; + const Path getPath() const; - static const Subspace DEFAULT_NODE_SUBSPACE; - static const Subspace DEFAULT_CONTENT_SUBSPACE; - static const StringRef PARTITION_LAYER; + static const Subspace DEFAULT_NODE_SUBSPACE; + static const Subspace DEFAULT_CONTENT_SUBSPACE; + static const StringRef PARTITION_LAYER; - //private: - static const uint8_t LITTLE_ENDIAN_LONG_ONE[8]; - static const StringRef HIGH_CONTENTION_KEY; - static const StringRef LAYER_KEY; - static const StringRef VERSION_KEY; - static const int64_t SUB_DIR_KEY; - static const uint32_t VERSION[3]; - static const StringRef DEFAULT_NODE_SUBSPACE_PREFIX; + // private: + static const uint8_t LITTLE_ENDIAN_LONG_ONE[8]; + static const StringRef HIGH_CONTENTION_KEY; + static const StringRef LAYER_KEY; + static const StringRef VERSION_KEY; + static const int64_t SUB_DIR_KEY; + static const uint32_t VERSION[3]; + static const StringRef DEFAULT_NODE_SUBSPACE_PREFIX; - struct Node { - Node() {} - Node(Reference const& directoryLayer, Optional const& subspace, Path const& path, Path const& targetPath); + struct Node { + Node() {} + Node(Reference const& directoryLayer, + Optional const& subspace, + Path const& path, + Path const& targetPath); - bool exists() const; + bool exists() const; - Future loadMetadata(Reference tr); - void ensureMetadataLoaded() const; + Future loadMetadata(Reference tr); + void ensureMetadataLoaded() const; - bool isInPartition(bool includeEmptySubpath = false) const; - Path getPartitionSubpath() const; - Reference getContents() const; - - Reference directoryLayer; - Optional subspace; - Path path; - Path targetPath; - Standalone layer; - - bool loadedMetadata; - }; - - Reference openInternal(Standalone const& layer, Node const& existingNode, bool allowOpen); - Future> createOrOpenInternal(Reference const& tr, Path const& path, Standalone const& layer, Optional> const& prefix, bool allowCreate, bool allowOpen); - - void initializeDirectory(Reference const& tr) const; - Future checkVersion(Reference const& tr, bool writeAccess) const; - - template - Optional nodeWithPrefix(Optional const& prefix) const; - Subspace nodeWithPrefix(StringRef const& prefix) const; - - Reference contentsOfNode(Subspace const& node, Path const& path, Standalone const& layer); - - Path toAbsolutePath(Path const& subpath) const; - - Subspace rootNode; - Subspace nodeSubspace; - Subspace contentSubspace; - HighContentionAllocator allocator; - bool allowManualPrefixes; + bool isInPartition(bool includeEmptySubpath = false) const; + Path getPartitionSubpath() const; + Reference getContents() const; + Reference directoryLayer; + Optional subspace; Path path; + Path targetPath; + Standalone layer; + + bool loadedMetadata; }; -} + + Reference openInternal(Standalone const& layer, + Node const& existingNode, + bool allowOpen); + Future> createOrOpenInternal(Reference const& tr, + Path const& path, + Standalone const& layer, + Optional> const& prefix, + bool allowCreate, + bool allowOpen); + + void initializeDirectory(Reference const& tr) const; + Future checkVersion(Reference const& tr, bool writeAccess) const; + + template + Optional nodeWithPrefix(Optional const& prefix) const; + Subspace nodeWithPrefix(StringRef const& prefix) const; + + Reference contentsOfNode(Subspace const& node, + Path const& path, + Standalone const& layer); + + Path toAbsolutePath(Path const& subpath) const; + + Subspace rootNode; + Subspace nodeSubspace; + Subspace contentSubspace; + HighContentionAllocator allocator; + bool allowManualPrefixes; + + Path path; +}; +} // namespace FDB #endif \ No newline at end of file diff --git a/bindings/flow/DirectoryPartition.h b/bindings/flow/DirectoryPartition.h index e154806373..8ec80e3288 100644 --- a/bindings/flow/DirectoryPartition.h +++ b/bindings/flow/DirectoryPartition.h @@ -28,34 +28,38 @@ #include "DirectoryLayer.h" namespace FDB { - class DirectoryPartition : public DirectorySubspace { +class DirectoryPartition : public DirectorySubspace { - public: - DirectoryPartition(Path const& path, StringRef const& prefix, Reference parentDirectoryLayer) - : DirectorySubspace(path, prefix, Reference(new DirectoryLayer(Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)), Subspace(prefix))), DirectoryLayer::PARTITION_LAYER), - parentDirectoryLayer(parentDirectoryLayer) - { - this->directoryLayer->path = path; - } - virtual ~DirectoryPartition() {} +public: + DirectoryPartition(Path const& path, StringRef const& prefix, Reference parentDirectoryLayer) + : DirectorySubspace(path, + prefix, + Reference(new DirectoryLayer( + Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)), + Subspace(prefix))), + DirectoryLayer::PARTITION_LAYER), + parentDirectoryLayer(parentDirectoryLayer) { + this->directoryLayer->path = path; + } + virtual ~DirectoryPartition() {} - virtual Key key() const { throw cannot_use_partition_as_subspace(); } - virtual bool contains(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); } + virtual Key key() const { throw cannot_use_partition_as_subspace(); } + virtual bool contains(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); } - virtual Key pack(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); } - virtual Tuple unpack(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); } - virtual KeyRange range(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); } + virtual Key pack(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); } + virtual Tuple unpack(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); } + virtual KeyRange range(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); } - virtual Subspace subspace(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); } - virtual Subspace get(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); } + virtual Subspace subspace(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); } + virtual Subspace get(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); } - protected: - Reference parentDirectoryLayer; +protected: + Reference parentDirectoryLayer; - virtual Reference getDirectoryLayerForPath(Path const& path) const { - return path.empty() ? parentDirectoryLayer : directoryLayer; - } - }; -} + virtual Reference getDirectoryLayerForPath(Path const& path) const { + return path.empty() ? parentDirectoryLayer : directoryLayer; + } +}; +} // namespace FDB #endif \ No newline at end of file diff --git a/bindings/flow/DirectorySubspace.cpp b/bindings/flow/DirectorySubspace.cpp old mode 100755 new mode 100644 index 237d72285f..63b26b5765 --- a/bindings/flow/DirectorySubspace.cpp +++ b/bindings/flow/DirectorySubspace.cpp @@ -21,89 +21,100 @@ #include "DirectorySubspace.h" namespace FDB { - DirectorySubspace::DirectorySubspace(Path const& path, StringRef const& prefix, Reference directoryLayer, Standalone const& layer) - : Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) { } +DirectorySubspace::DirectorySubspace(Path const& path, + StringRef const& prefix, + Reference directoryLayer, + Standalone const& layer) + : Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) {} +Future> DirectorySubspace::create(Reference const& tr, + Path const& path, + Standalone const& layer, + Optional> const& prefix) { + return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix); +} - Future> DirectorySubspace::create(Reference const& tr, Path const& path, Standalone const& layer, - Optional> const& prefix) - { - return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix); +Future> DirectorySubspace::open(Reference const& tr, + Path const& path, + Standalone const& layer) { + return directoryLayer->open(tr, getPartitionSubpath(path), layer); +} + +Future> DirectorySubspace::createOrOpen(Reference const& tr, + Path const& path, + Standalone const& layer) { + return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer); +} + +Future DirectorySubspace::exists(Reference const& tr, Path const& path) { + Reference directoryLayer = getDirectoryLayerForPath(path); + return directoryLayer->exists(tr, getPartitionSubpath(path, directoryLayer)); +} + +Future>> DirectorySubspace::list(Reference const& tr, Path const& path) { + return directoryLayer->list(tr, getPartitionSubpath(path)); +} + +Future> DirectorySubspace::move(Reference const& tr, + Path const& oldPath, + Path const& newPath) { + return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath)); +} + +Future> DirectorySubspace::moveTo(Reference const& tr, + Path const& newAbsolutePath) { + Reference directoryLayer = getDirectoryLayerForPath(Path()); + Path directoryLayerPath = directoryLayer->getPath(); + + if (directoryLayerPath.size() > newAbsolutePath.size()) { + return cannot_move_directory_between_partitions(); } - Future> DirectorySubspace::open(Reference const& tr, Path const& path, Standalone const& layer) { - return directoryLayer->open(tr, getPartitionSubpath(path), layer); - } - - Future> DirectorySubspace::createOrOpen(Reference const& tr, Path const& path, Standalone const& layer) { - return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer); - } - - Future DirectorySubspace::exists(Reference const& tr, Path const& path) { - Reference directoryLayer = getDirectoryLayerForPath(path); - return directoryLayer->exists(tr, getPartitionSubpath(path, directoryLayer)); - } - - Future>> DirectorySubspace::list(Reference const& tr, Path const& path) { - return directoryLayer->list(tr, getPartitionSubpath(path)); - } - - Future> DirectorySubspace::move(Reference const& tr, Path const& oldPath, Path const& newPath) { - return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath)); - } - - Future> DirectorySubspace::moveTo(Reference const& tr, Path const& newAbsolutePath) { - Reference directoryLayer = getDirectoryLayerForPath(Path()); - Path directoryLayerPath = directoryLayer->getPath(); - - if(directoryLayerPath.size() > newAbsolutePath.size()) { + for (int i = 0; i < directoryLayerPath.size(); ++i) { + if (directoryLayerPath[i] != newAbsolutePath[i]) { return cannot_move_directory_between_partitions(); } - - for(int i = 0; i < directoryLayerPath.size(); ++i) { - if(directoryLayerPath[i] != newAbsolutePath[i]) { - return cannot_move_directory_between_partitions(); - } - } - - Path newRelativePath(newAbsolutePath.begin() + directoryLayerPath.size(), newAbsolutePath.end()); - return directoryLayer->move(tr, getPartitionSubpath(Path(), directoryLayer), newRelativePath); } - Future DirectorySubspace::remove(Reference const& tr, Path const& path) { - Reference directoryLayer = getDirectoryLayerForPath(path); - return directoryLayer->remove(tr, getPartitionSubpath(path, directoryLayer)); - } - - Future DirectorySubspace::removeIfExists(Reference const& tr, Path const& path) { - Reference directoryLayer = getDirectoryLayerForPath(path); - return directoryLayer->removeIfExists(tr, getPartitionSubpath(path, directoryLayer)); - } - - Reference DirectorySubspace::getDirectoryLayer() { - return directoryLayer; - } - - const Standalone DirectorySubspace::getLayer() const { - return layer; - } - - const IDirectory::Path DirectorySubspace::getPath() const { - return path; - } - - IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path, Reference directoryLayer) const { - if(!directoryLayer) { - directoryLayer = this->directoryLayer; - } - - Path newPath(this->path.begin() + directoryLayer->getPath().size(), this->path.end()); - newPath.insert(newPath.end(), path.begin(), path.end()); - - return newPath; - } - - Reference DirectorySubspace::getDirectoryLayerForPath(Path const& path) const { - return directoryLayer; - } + Path newRelativePath(newAbsolutePath.begin() + directoryLayerPath.size(), newAbsolutePath.end()); + return directoryLayer->move(tr, getPartitionSubpath(Path(), directoryLayer), newRelativePath); } + +Future DirectorySubspace::remove(Reference const& tr, Path const& path) { + Reference directoryLayer = getDirectoryLayerForPath(path); + return directoryLayer->remove(tr, getPartitionSubpath(path, directoryLayer)); +} + +Future DirectorySubspace::removeIfExists(Reference const& tr, Path const& path) { + Reference directoryLayer = getDirectoryLayerForPath(path); + return directoryLayer->removeIfExists(tr, getPartitionSubpath(path, directoryLayer)); +} + +Reference DirectorySubspace::getDirectoryLayer() { + return directoryLayer; +} + +const Standalone DirectorySubspace::getLayer() const { + return layer; +} + +const IDirectory::Path DirectorySubspace::getPath() const { + return path; +} + +IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path, + Reference directoryLayer) const { + if (!directoryLayer) { + directoryLayer = this->directoryLayer; + } + + Path newPath(this->path.begin() + directoryLayer->getPath().size(), this->path.end()); + newPath.insert(newPath.end(), path.begin(), path.end()); + + return newPath; +} + +Reference DirectorySubspace::getDirectoryLayerForPath(Path const& path) const { + return directoryLayer; +} +} // namespace FDB diff --git a/bindings/flow/DirectorySubspace.h b/bindings/flow/DirectorySubspace.h old mode 100755 new mode 100644 index 9a40cbf0b4..8c02fdd9e2 --- a/bindings/flow/DirectorySubspace.h +++ b/bindings/flow/DirectorySubspace.h @@ -28,39 +28,53 @@ #include "Subspace.h" namespace FDB { - class DirectorySubspace : public IDirectory, public Subspace { +class DirectorySubspace : public IDirectory, public Subspace { - public: - DirectorySubspace(Path const& path, StringRef const& prefix, Reference directorLayer, Standalone const& layer = Standalone()); - virtual ~DirectorySubspace() {} +public: + DirectorySubspace(Path const& path, + StringRef const& prefix, + Reference directorLayer, + Standalone const& layer = Standalone()); + virtual ~DirectorySubspace() {} - virtual Future> create(Reference const& tr, Path const& path, Standalone const& layer = Standalone(), - Optional> const& prefix = Optional>()); + virtual Future> create( + Reference const& tr, + Path const& path, + Standalone const& layer = Standalone(), + Optional> const& prefix = Optional>()); - virtual Future> open(Reference const& tr, Path const& path, Standalone const& layer = Standalone()); - virtual Future> createOrOpen(Reference const& tr, Path const& path, Standalone const& layer = Standalone()); + virtual Future> open(Reference const& tr, + Path const& path, + Standalone const& layer = Standalone()); + virtual Future> createOrOpen( + Reference const& tr, + Path const& path, + Standalone const& layer = Standalone()); - virtual Future exists(Reference const& tr, Path const& path = Path()); - virtual Future>> list(Reference const& tr, Path const& path = Path()); + virtual Future exists(Reference const& tr, Path const& path = Path()); + virtual Future>> list(Reference const& tr, Path const& path = Path()); - virtual Future> move(Reference const& tr, Path const& oldPath, Path const& newPath); - virtual Future> moveTo(Reference const& tr, Path const& newAbsolutePath); + virtual Future> move(Reference const& tr, + Path const& oldPath, + Path const& newPath); + virtual Future> moveTo(Reference const& tr, Path const& newAbsolutePath); - virtual Future remove(Reference const& tr, Path const& path = Path()); - virtual Future removeIfExists(Reference const& tr, Path const& path = Path()); + virtual Future remove(Reference const& tr, Path const& path = Path()); + virtual Future removeIfExists(Reference const& tr, Path const& path = Path()); - virtual Reference getDirectoryLayer(); - virtual const Standalone getLayer() const; - virtual const Path getPath() const; + virtual Reference getDirectoryLayer(); + virtual const Standalone getLayer() const; + virtual const Path getPath() const; - protected: - Reference directoryLayer; - Path path; - Standalone layer; +protected: + Reference directoryLayer; + Path path; + Standalone layer; - virtual Path getPartitionSubpath(Path const& path, Reference directoryLayer = Reference()) const; - virtual Reference getDirectoryLayerForPath(Path const& path) const; - }; -} + virtual Path getPartitionSubpath(Path const& path, + Reference directoryLayer = Reference()) const; + virtual Reference getDirectoryLayerForPath(Path const& path) const; +}; +} // namespace FDB #endif \ No newline at end of file diff --git a/bindings/flow/FDBLoanerTypes.h b/bindings/flow/FDBLoanerTypes.h old mode 100755 new mode 100644 index 3256b5339e..97e4394298 --- a/bindings/flow/FDBLoanerTypes.h +++ b/bindings/flow/FDBLoanerTypes.h @@ -22,292 +22,304 @@ #define FDB_FLOW_LOANER_TYPES_H namespace FDB { - typedef StringRef KeyRef; - typedef StringRef ValueRef; +typedef StringRef KeyRef; +typedef StringRef ValueRef; - typedef int64_t Version; +typedef int64_t Version; - typedef Standalone Key; - typedef Standalone Value; +typedef Standalone Key; +typedef Standalone Value; - inline Key keyAfter( const KeyRef& key ) { - if(key == LiteralStringRef("\xff\xff")) - return key; +inline Key keyAfter(const KeyRef& key) { + if (key == LiteralStringRef("\xff\xff")) + return key; - Standalone r; - uint8_t* s = new (r.arena()) uint8_t[ key.size() + 1 ]; - memcpy(s, key.begin(), key.size() ); - s[key.size()] = 0; - ((StringRef&) r) = StringRef( s, key.size() + 1 ); - return r; - } - - inline KeyRef keyAfter( const KeyRef& key, Arena& arena ) { - if(key == LiteralStringRef("\xff\xff")) - return key; - uint8_t* t = new ( arena ) uint8_t[ key.size()+1 ]; - memcpy(t, key.begin(), key.size() ); - t[key.size()] = 0; - return KeyRef(t,key.size()+1); - } - - struct KeySelectorRef { - KeyRef key; // Find the last item less than key - bool orEqual; // (or equal to key, if this is true) - int offset; // and then move forward this many items (or backward if negative) - KeySelectorRef() {} - KeySelectorRef( const KeyRef& key, bool orEqual, int offset ) : key(key), orEqual(orEqual), offset(offset) {} - - KeySelectorRef( Arena& arena, const KeySelectorRef& copyFrom ) : key(arena,copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {} - int expectedSize() const { return key.expectedSize(); } - - // std::string toString() const { - // if (offset > 0) { - // if (orEqual) return format("firstGreaterThan(%s)%+d", printable(key).c_str(), offset-1); - // else return format("firstGreaterOrEqual(%s)%+d", printable(key).c_str(), offset-1); - // } else { - // if (orEqual) return format("lastLessOrEqual(%s)%+d", printable(key).c_str(), offset); - // else return format("lastLessThan(%s)%+d", printable(key).c_str(), offset); - // } - // } - - bool isBackward() const { return !orEqual && offset<=0; } // True if the resolution of the KeySelector depends only on keys less than key - bool isFirstGreaterOrEqual() const { return !orEqual && offset==1; } - bool isFirstGreaterThan() const { return orEqual && offset==1; } - bool isLastLessOrEqual() const { return orEqual && offset==0; } - - // True iff, regardless of the contents of the database, lhs must resolve to a key > rhs - bool isDefinitelyGreater( KeyRef const& k ) { - return offset >= 1 && ( isFirstGreaterOrEqual() ? key > k : key >= k ); - } - // True iff, regardless of the contents of the database, lhs must resolve to a key < rhs - bool isDefinitelyLess( KeyRef const& k ) { - return offset <= 0 && ( isLastLessOrEqual() ? key < k : key <= k ); - } - - template - void serialize( Ar& ar ) { - serializer(ar, key, orEqual, offset); - } - }; - inline bool operator == (const KeySelectorRef& lhs, const KeySelectorRef& rhs) { return lhs.key == rhs.key && lhs.orEqual==rhs.orEqual && lhs.offset==rhs.offset; } - inline KeySelectorRef lastLessThan( const KeyRef& k ) { - return KeySelectorRef( k, false, 0 ); - } - inline KeySelectorRef lastLessOrEqual( const KeyRef& k ) { - return KeySelectorRef( k, true, 0 ); - } - inline KeySelectorRef firstGreaterThan( const KeyRef& k ) { - return KeySelectorRef( k, true, +1 ); - } - inline KeySelectorRef firstGreaterOrEqual( const KeyRef& k ) { - return KeySelectorRef( k, false, +1 ); - } - inline KeySelectorRef operator + (const KeySelectorRef& s, int off) { - return KeySelectorRef(s.key, s.orEqual, s.offset+off); - } - inline KeySelectorRef operator - (const KeySelectorRef& s, int off) { - return KeySelectorRef(s.key, s.orEqual, s.offset-off); - } - - typedef Standalone KeySelector; - - struct KeyValueRef { - KeyRef key; - ValueRef value; - KeyValueRef() {} - KeyValueRef( const KeyRef& key, const ValueRef& value ) : key(key), value(value) {} - KeyValueRef( Arena& a, const KeyValueRef& copyFrom ) : key(a, copyFrom.key), value(a, copyFrom.value) {} - bool operator == ( const KeyValueRef& r ) const { return key == r.key && value == r.value; } - - int expectedSize() const { return key.expectedSize() + value.expectedSize(); } - - template - force_inline void serialize(Ar& ar) { serializer(ar, key, value); } - - struct OrderByKey { - bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { - return a.key < b.key; - } - template - bool operator()(T const& a, KeyValueRef const& b) const { - return a < b.key; - } - template - bool operator()(KeyValueRef const& a, T const& b) const { - return a.key < b; - } - }; - - struct OrderByKeyBack { - bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { - return a.key > b.key; - } - template - bool operator()(T const& a, KeyValueRef const& b) const { - return a > b.key; - } - template - bool operator()(KeyValueRef const& a, T const& b) const { - return a.key > b; - } - }; - }; - - typedef Standalone KeyValue; - - struct RangeResultRef : VectorRef { - bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the limits requested) - // False implies that no such values remain - Optional readThrough; // Only present when 'more' is true. When present, this value represent the end (or beginning if reverse) of the range - // which was read to produce these results. This is guarenteed to be less than the requested range. - bool readToBegin; - bool readThroughEnd; - - RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {} - RangeResultRef( Arena& p, const RangeResultRef& toCopy ) : more( toCopy.more ), readToBegin( toCopy.readToBegin ), readThroughEnd( toCopy.readThroughEnd ), readThrough( toCopy.readThrough.present() ? KeyRef( p, toCopy.readThrough.get() ) : Optional() ), VectorRef( p, toCopy ) {} - RangeResultRef( const VectorRef& value, bool more, Optional readThrough = Optional() ) : VectorRef( value ), more( more ), readThrough( readThrough ), readToBegin( false ), readThroughEnd( false ) {} - RangeResultRef( bool readToBegin, bool readThroughEnd ) : more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) { } - - template - void serialize( Ar& ar ) { - serializer(ar, ((VectorRef&)*this), more, readThrough, readToBegin, readThroughEnd); - } - }; - - struct GetRangeLimits { - enum { ROW_LIMIT_UNLIMITED = -1, BYTE_LIMIT_UNLIMITED = -1 }; - - int rows; - int minRows; - int bytes; - - GetRangeLimits() : rows( ROW_LIMIT_UNLIMITED ), minRows(1), bytes( BYTE_LIMIT_UNLIMITED ) {} - explicit GetRangeLimits( int rowLimit ) : rows( rowLimit ), minRows(1), bytes( BYTE_LIMIT_UNLIMITED ) {} - GetRangeLimits( int rowLimit, int byteLimit ) : rows( rowLimit ), minRows(1), bytes( byteLimit ) {} - - void decrement( VectorRef const& data ); - void decrement( KeyValueRef const& data ); - - // True if either the row or byte limit has been reached - bool isReached(); - - // True if data would cause the row or byte limit to be reached - bool reachedBy( VectorRef const& data ); - - bool hasByteLimit(); - bool hasRowLimit(); - - bool hasSatisfiedMinRows(); - bool isValid() { return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED) - && (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED) - && minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED); } - }; - - struct KeyRangeRef { - const KeyRef begin, end; - KeyRangeRef() {} - KeyRangeRef( const KeyRef& begin, const KeyRef& end ) : begin(begin), end(end) { - if( begin > end ) { - throw inverted_range(); - } - } - KeyRangeRef( Arena& a, const KeyRangeRef& copyFrom ) : begin(a, copyFrom.begin), end(a, copyFrom.end) {} - bool operator == ( const KeyRangeRef& r ) const { return begin == r.begin && end == r.end; } - bool operator != ( const KeyRangeRef& r ) const { return begin != r.begin || end != r.end; } - bool contains( const KeyRef& key ) const { return begin <= key && key < end; } - bool contains( const KeyRangeRef& keys ) const { return begin <= keys.begin && keys.end <= end; } - bool intersects( const KeyRangeRef& keys ) const { return begin < keys.end && keys.begin < end; } - bool empty() const { return begin == end; } - - Standalone withPrefix( const StringRef& prefix ) const { - return KeyRangeRef( begin.withPrefix(prefix), end.withPrefix(prefix) ); - } - - const KeyRangeRef& operator = (const KeyRangeRef& rhs) { - const_cast(begin) = rhs.begin; - const_cast(end) = rhs.end; - return *this; - } - - int expectedSize() const { return begin.expectedSize() + end.expectedSize(); } - - template - force_inline void serialize(Ar& ar) { - serializer(ar, const_cast(begin), const_cast(end)); - if( begin > end ) { - throw inverted_range(); - }; - } - - struct ArbitraryOrder { - bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const { - if (a.begin < b.begin) return true; - if (a.begin > b.begin) return false; - return a.end < b.end; - } - }; - }; - - inline KeyRangeRef operator & (const KeyRangeRef& lhs, const KeyRangeRef& rhs) { - KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end); - if (e < b) - return KeyRangeRef(); - return KeyRangeRef(b,e); - } - - typedef Standalone KeyRange; - - template - static std::string describe(T const& item) { - return item.toString(); - } - template - static std::string describe(std::map const& items, int max_items = -1) { - if (!items.size()) - return "[no items]"; - - std::string s; - int count = 0; - for (auto it = items.begin(); it != items.end(); it++) { - if (++count > max_items && max_items >= 0) - break; - if (count > 1) s += ","; - s += describe(it->first) + "=>" + describe(it->second); - } - return s; - } - - template - static std::string describeList(T const& items, int max_items) { - if (!items.size()) - return "[no items]"; - - std::string s; - int count = 0; - for (auto const& item : items) { - if (++count > max_items && max_items >= 0) - break; - if (count > 1) s += ","; - s += describe(item); - } - return s; - } - - template - static std::string describe(std::vector const& items, int max_items = -1) { - return describeList(items, max_items); - } - - template - static std::string describe(std::set const& items, int max_items = -1) { - return describeList(items, max_items); - } - - template - static std::string describe(std::pair const& pair) { - return "first: " + describe(pair.first) + " second: " + describe(pair.second); - } + Standalone r; + uint8_t* s = new (r.arena()) uint8_t[key.size() + 1]; + memcpy(s, key.begin(), key.size()); + s[key.size()] = 0; + ((StringRef&)r) = StringRef(s, key.size() + 1); + return r; } +inline KeyRef keyAfter(const KeyRef& key, Arena& arena) { + if (key == LiteralStringRef("\xff\xff")) + return key; + uint8_t* t = new (arena) uint8_t[key.size() + 1]; + memcpy(t, key.begin(), key.size()); + t[key.size()] = 0; + return KeyRef(t, key.size() + 1); +} + +struct KeySelectorRef { + KeyRef key; // Find the last item less than key + bool orEqual; // (or equal to key, if this is true) + int offset; // and then move forward this many items (or backward if negative) + KeySelectorRef() {} + KeySelectorRef(const KeyRef& key, bool orEqual, int offset) : key(key), orEqual(orEqual), offset(offset) {} + + KeySelectorRef(Arena& arena, const KeySelectorRef& copyFrom) + : key(arena, copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {} + int expectedSize() const { return key.expectedSize(); } + + // std::string toString() const { + // if (offset > 0) { + // if (orEqual) return format("firstGreaterThan(%s)%+d", printable(key).c_str(), offset-1); + // else return format("firstGreaterOrEqual(%s)%+d", printable(key).c_str(), offset-1); + // } else { + // if (orEqual) return format("lastLessOrEqual(%s)%+d", printable(key).c_str(), offset); + // else return format("lastLessThan(%s)%+d", printable(key).c_str(), offset); + // } + // } + + bool isBackward() const { + return !orEqual && offset <= 0; + } // True if the resolution of the KeySelector depends only on keys less than key + bool isFirstGreaterOrEqual() const { return !orEqual && offset == 1; } + bool isFirstGreaterThan() const { return orEqual && offset == 1; } + bool isLastLessOrEqual() const { return orEqual && offset == 0; } + + // True iff, regardless of the contents of the database, lhs must resolve to a key > rhs + bool isDefinitelyGreater(KeyRef const& k) { return offset >= 1 && (isFirstGreaterOrEqual() ? key > k : key >= k); } + // True iff, regardless of the contents of the database, lhs must resolve to a key < rhs + bool isDefinitelyLess(KeyRef const& k) { return offset <= 0 && (isLastLessOrEqual() ? key < k : key <= k); } + + template + void serialize(Ar& ar) { + serializer(ar, key, orEqual, offset); + } +}; +inline bool operator==(const KeySelectorRef& lhs, const KeySelectorRef& rhs) { + return lhs.key == rhs.key && lhs.orEqual == rhs.orEqual && lhs.offset == rhs.offset; +} +inline KeySelectorRef lastLessThan(const KeyRef& k) { + return KeySelectorRef(k, false, 0); +} +inline KeySelectorRef lastLessOrEqual(const KeyRef& k) { + return KeySelectorRef(k, true, 0); +} +inline KeySelectorRef firstGreaterThan(const KeyRef& k) { + return KeySelectorRef(k, true, +1); +} +inline KeySelectorRef firstGreaterOrEqual(const KeyRef& k) { + return KeySelectorRef(k, false, +1); +} +inline KeySelectorRef operator+(const KeySelectorRef& s, int off) { + return KeySelectorRef(s.key, s.orEqual, s.offset + off); +} +inline KeySelectorRef operator-(const KeySelectorRef& s, int off) { + return KeySelectorRef(s.key, s.orEqual, s.offset - off); +} + +typedef Standalone KeySelector; + +struct KeyValueRef { + KeyRef key; + ValueRef value; + KeyValueRef() {} + KeyValueRef(const KeyRef& key, const ValueRef& value) : key(key), value(value) {} + KeyValueRef(Arena& a, const KeyValueRef& copyFrom) : key(a, copyFrom.key), value(a, copyFrom.value) {} + bool operator==(const KeyValueRef& r) const { return key == r.key && value == r.value; } + + int expectedSize() const { return key.expectedSize() + value.expectedSize(); } + + template + force_inline void serialize(Ar& ar) { + serializer(ar, key, value); + } + + struct OrderByKey { + bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key < b.key; } + template + bool operator()(T const& a, KeyValueRef const& b) const { + return a < b.key; + } + template + bool operator()(KeyValueRef const& a, T const& b) const { + return a.key < b; + } + }; + + struct OrderByKeyBack { + bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key > b.key; } + template + bool operator()(T const& a, KeyValueRef const& b) const { + return a > b.key; + } + template + bool operator()(KeyValueRef const& a, T const& b) const { + return a.key > b; + } + }; +}; + +typedef Standalone KeyValue; + +struct RangeResultRef : VectorRef { + bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the + // limits requested) + // False implies that no such values remain + Optional readThrough; // Only present when 'more' is true. When present, this value represent the end (or + // beginning if reverse) of the range + // which was read to produce these results. This is guarenteed to be less than the requested range. + bool readToBegin; + bool readThroughEnd; + + RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {} + RangeResultRef(Arena& p, const RangeResultRef& toCopy) + : more(toCopy.more), readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd), + readThrough(toCopy.readThrough.present() ? KeyRef(p, toCopy.readThrough.get()) : Optional()), + VectorRef(p, toCopy) {} + RangeResultRef(const VectorRef& value, bool more, Optional readThrough = Optional()) + : VectorRef(value), more(more), readThrough(readThrough), readToBegin(false), readThroughEnd(false) { + } + RangeResultRef(bool readToBegin, bool readThroughEnd) + : more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) {} + + template + void serialize(Ar& ar) { + serializer(ar, ((VectorRef&)*this), more, readThrough, readToBegin, readThroughEnd); + } +}; + +struct GetRangeLimits { + enum { ROW_LIMIT_UNLIMITED = -1, BYTE_LIMIT_UNLIMITED = -1 }; + + int rows; + int minRows; + int bytes; + + GetRangeLimits() : rows(ROW_LIMIT_UNLIMITED), minRows(1), bytes(BYTE_LIMIT_UNLIMITED) {} + explicit GetRangeLimits(int rowLimit) : rows(rowLimit), minRows(1), bytes(BYTE_LIMIT_UNLIMITED) {} + GetRangeLimits(int rowLimit, int byteLimit) : rows(rowLimit), minRows(1), bytes(byteLimit) {} + + void decrement(VectorRef const& data); + void decrement(KeyValueRef const& data); + + // True if either the row or byte limit has been reached + bool isReached(); + + // True if data would cause the row or byte limit to be reached + bool reachedBy(VectorRef const& data); + + bool hasByteLimit(); + bool hasRowLimit(); + + bool hasSatisfiedMinRows(); + bool isValid() { + return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED) && (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED) && + minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED); + } +}; + +struct KeyRangeRef { + const KeyRef begin, end; + KeyRangeRef() {} + KeyRangeRef(const KeyRef& begin, const KeyRef& end) : begin(begin), end(end) { + if (begin > end) { + throw inverted_range(); + } + } + KeyRangeRef(Arena& a, const KeyRangeRef& copyFrom) : begin(a, copyFrom.begin), end(a, copyFrom.end) {} + bool operator==(const KeyRangeRef& r) const { return begin == r.begin && end == r.end; } + bool operator!=(const KeyRangeRef& r) const { return begin != r.begin || end != r.end; } + bool contains(const KeyRef& key) const { return begin <= key && key < end; } + bool contains(const KeyRangeRef& keys) const { return begin <= keys.begin && keys.end <= end; } + bool intersects(const KeyRangeRef& keys) const { return begin < keys.end && keys.begin < end; } + bool empty() const { return begin == end; } + + Standalone withPrefix(const StringRef& prefix) const { + return KeyRangeRef(begin.withPrefix(prefix), end.withPrefix(prefix)); + } + + const KeyRangeRef& operator=(const KeyRangeRef& rhs) { + const_cast(begin) = rhs.begin; + const_cast(end) = rhs.end; + return *this; + } + + int expectedSize() const { return begin.expectedSize() + end.expectedSize(); } + + template + force_inline void serialize(Ar& ar) { + serializer(ar, const_cast(begin), const_cast(end)); + if (begin > end) { + throw inverted_range(); + }; + } + + struct ArbitraryOrder { + bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const { + if (a.begin < b.begin) + return true; + if (a.begin > b.begin) + return false; + return a.end < b.end; + } + }; +}; + +inline KeyRangeRef operator&(const KeyRangeRef& lhs, const KeyRangeRef& rhs) { + KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end); + if (e < b) + return KeyRangeRef(); + return KeyRangeRef(b, e); +} + +typedef Standalone KeyRange; + +template +static std::string describe(T const& item) { + return item.toString(); +} +template +static std::string describe(std::map const& items, int max_items = -1) { + if (!items.size()) + return "[no items]"; + + std::string s; + int count = 0; + for (auto it = items.begin(); it != items.end(); it++) { + if (++count > max_items && max_items >= 0) + break; + if (count > 1) + s += ","; + s += describe(it->first) + "=>" + describe(it->second); + } + return s; +} + +template +static std::string describeList(T const& items, int max_items) { + if (!items.size()) + return "[no items]"; + + std::string s; + int count = 0; + for (auto const& item : items) { + if (++count > max_items && max_items >= 0) + break; + if (count > 1) + s += ","; + s += describe(item); + } + return s; +} + +template +static std::string describe(std::vector const& items, int max_items = -1) { + return describeList(items, max_items); +} + +template +static std::string describe(std::set const& items, int max_items = -1) { + return describeList(items, max_items); +} + +template +static std::string describe(std::pair const& pair) { + return "first: " + describe(pair.first) + " second: " + describe(pair.second); +} +} // namespace FDB + #endif /* FDB_LOANER_TYPES_H */ diff --git a/bindings/flow/HighContentionAllocator.actor.cpp b/bindings/flow/HighContentionAllocator.actor.cpp index 79acedee8e..41d3481bcd 100644 --- a/bindings/flow/HighContentionAllocator.actor.cpp +++ b/bindings/flow/HighContentionAllocator.actor.cpp @@ -21,90 +21,90 @@ #include "HighContentionAllocator.h" namespace FDB { - ACTOR Future> _allocate(Reference tr, Subspace counters, Subspace recent){ - state int64_t start = 0; - state int64_t window = 0; +ACTOR Future> _allocate(Reference tr, Subspace counters, Subspace recent) { + state int64_t start = 0; + state int64_t window = 0; + + loop { + FDBStandalone range = wait(tr->getRange(counters.range(), 1, true, true)); + + if (range.size() > 0) { + start = counters.unpack(range[0].key).getInt(0); + } + + state bool windowAdvanced = false; + loop { + // if thread safety is needed, this should be locked { + if (windowAdvanced) { + tr->clear(KeyRangeRef(counters.key(), counters.get(start).key())); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); + tr->clear(KeyRangeRef(recent.key(), recent.get(start).key())); + } + + int64_t inc = 1; + tr->atomicOp(counters.get(start).key(), StringRef((uint8_t*)&inc, 8), FDB_MUTATION_TYPE_ADD); + Future>> countFuture = tr->get(counters.get(start).key(), true); + // } + + Optional> countValue = wait(countFuture); + + int64_t count = 0; + if (countValue.present()) { + if (countValue.get().size() != 8) { + throw invalid_directory_layer_metadata(); + } + count = *(int64_t*)countValue.get().begin(); + } + + window = HighContentionAllocator::windowSize(start); + if (count * 2 < window) { + break; + } + + start += window; + windowAdvanced = true; + } loop { - FDBStandalone range = wait(tr->getRange(counters.range(), 1, true, true)); + state int64_t candidate = deterministicRandom()->randomInt(start, start + window); - if(range.size() > 0) { - start = counters.unpack(range[0].key).getInt(0); + // if thread safety is needed, this should be locked { + state Future> latestCounter = tr->getRange(counters.range(), 1, true, true); + state Future>> candidateValue = tr->get(recent.get(candidate).key()); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); + tr->set(recent.get(candidate).key(), ValueRef()); + // } + + wait(success(latestCounter) && success(candidateValue)); + int64_t currentWindowStart = 0; + if (latestCounter.get().size() > 0) { + currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0); } - state bool windowAdvanced = false; - loop { - // if thread safety is needed, this should be locked { - if(windowAdvanced) { - tr->clear(KeyRangeRef(counters.key(), counters.get(start).key())); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); - tr->clear(KeyRangeRef(recent.key(), recent.get(start).key())); - } - - int64_t inc = 1; - tr->atomicOp(counters.get(start).key(), StringRef((uint8_t*)&inc, 8), FDB_MUTATION_TYPE_ADD); - Future>> countFuture = tr->get(counters.get(start).key(), true); - // } - - Optional> countValue = wait(countFuture); - - int64_t count = 0; - if(countValue.present()) { - if(countValue.get().size() != 8) { - throw invalid_directory_layer_metadata(); - } - count = *(int64_t*)countValue.get().begin(); - } - - window = HighContentionAllocator::windowSize(start); - if(count * 2 < window) { - break; - } - - start += window; - windowAdvanced = true; + if (currentWindowStart > start) { + break; } - loop { - state int64_t candidate = deterministicRandom()->randomInt(start, start + window); - - // if thread safety is needed, this should be locked { - state Future> latestCounter = tr->getRange(counters.range(), 1, true, true); - state Future>> candidateValue = tr->get(recent.get(candidate).key()); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); - tr->set(recent.get(candidate).key(), ValueRef()); - // } - - wait(success(latestCounter) && success(candidateValue)); - int64_t currentWindowStart = 0; - if(latestCounter.get().size() > 0) { - currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0); - } - - if(currentWindowStart > start) { - break; - } - - if(!candidateValue.get().present()) { - tr->addWriteConflictKey(recent.get(candidate).key()); - return Tuple().append(candidate).pack(); - } + if (!candidateValue.get().present()) { + tr->addWriteConflictKey(recent.get(candidate).key()); + return Tuple().append(candidate).pack(); } } } - - Future> HighContentionAllocator::allocate(Reference const& tr) const { - return _allocate(tr, counters, recent); - } - - int64_t HighContentionAllocator::windowSize(int64_t start) { - if (start < 255) { - return 64; - } - if (start < 65535) { - return 1024; - } - - return 8192; - } } + +Future> HighContentionAllocator::allocate(Reference const& tr) const { + return _allocate(tr, counters, recent); +} + +int64_t HighContentionAllocator::windowSize(int64_t start) { + if (start < 255) { + return 64; + } + if (start < 65535) { + return 1024; + } + + return 8192; +} +} // namespace FDB diff --git a/bindings/flow/HighContentionAllocator.h b/bindings/flow/HighContentionAllocator.h index 37e50a7897..bd4ed2cb97 100644 --- a/bindings/flow/HighContentionAllocator.h +++ b/bindings/flow/HighContentionAllocator.h @@ -26,16 +26,17 @@ #include "Subspace.h" namespace FDB { - class HighContentionAllocator { - public: - HighContentionAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {} - Future> allocate(Reference const& tr) const; +class HighContentionAllocator { +public: + HighContentionAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {} + Future> allocate(Reference const& tr) const; - static int64_t windowSize(int64_t start); - private: - Subspace counters; - Subspace recent; - }; -} + static int64_t windowSize(int64_t start); + +private: + Subspace counters; + Subspace recent; +}; +} // namespace FDB #endif diff --git a/bindings/flow/IDirectory.h b/bindings/flow/IDirectory.h old mode 100755 new mode 100644 index 39256b1661..e629b7cf82 --- a/bindings/flow/IDirectory.h +++ b/bindings/flow/IDirectory.h @@ -27,34 +27,46 @@ #include "bindings/flow/fdb_flow.h" namespace FDB { - class DirectoryLayer; - class DirectorySubspace; +class DirectoryLayer; +class DirectorySubspace; - class IDirectory : public ReferenceCounted { - public: - typedef std::vector> Path; +class IDirectory : public ReferenceCounted { +public: + typedef std::vector> Path; - virtual Future> create(Reference const& tr, Path const& path, Standalone const& layer = Standalone(), - Optional> const& prefix = Optional>()) = 0; + virtual Future> create( + Reference const& tr, + Path const& path, + Standalone const& layer = Standalone(), + Optional> const& prefix = Optional>()) = 0; - virtual Future> open(Reference const& tr, Path const& path, Standalone const& layer = Standalone()) = 0; - virtual Future> createOrOpen(Reference const& tr, Path const& path, Standalone const& layer = Standalone()) = 0; + virtual Future> open(Reference const& tr, + Path const& path, + Standalone const& layer = Standalone()) = 0; + virtual Future> createOrOpen( + Reference const& tr, + Path const& path, + Standalone const& layer = Standalone()) = 0; - virtual Future exists(Reference const& tr, Path const& path = Path()) = 0; - virtual Future>> list(Reference const& tr, Path const& path = Path()) = 0; + virtual Future exists(Reference const& tr, Path const& path = Path()) = 0; + virtual Future>> list(Reference const& tr, + Path const& path = Path()) = 0; - virtual Future> move(Reference const& tr, Path const& oldPath, Path const& newPath) = 0; - virtual Future> moveTo(Reference const& tr, Path const& newAbsolutePath) = 0; + virtual Future> move(Reference const& tr, + Path const& oldPath, + Path const& newPath) = 0; + virtual Future> moveTo(Reference const& tr, + Path const& newAbsolutePath) = 0; - virtual Future remove(Reference const& tr, Path const& path = Path()) = 0; - virtual Future removeIfExists(Reference const& tr, Path const& path = Path()) = 0; + virtual Future remove(Reference const& tr, Path const& path = Path()) = 0; + virtual Future removeIfExists(Reference const& tr, Path const& path = Path()) = 0; - virtual Reference getDirectoryLayer() = 0; - virtual const Standalone getLayer() const = 0; - virtual const Path getPath() const = 0; + virtual Reference getDirectoryLayer() = 0; + virtual const Standalone getLayer() const = 0; + virtual const Path getPath() const = 0; - virtual ~IDirectory() {}; - }; -} + virtual ~IDirectory(){}; +}; +} // namespace FDB #endif \ No newline at end of file diff --git a/bindings/flow/Node.actor.cpp b/bindings/flow/Node.actor.cpp old mode 100755 new mode 100644 index 27521b9524..901376aa68 --- a/bindings/flow/Node.actor.cpp +++ b/bindings/flow/Node.actor.cpp @@ -21,50 +21,49 @@ #include "DirectoryLayer.h" namespace FDB { - DirectoryLayer::Node::Node(Reference const& directoryLayer, Optional const& subspace, IDirectory::Path const& path, IDirectory::Path const& targetPath) - : directoryLayer(directoryLayer), - subspace(subspace), - path(path), - targetPath(targetPath), - loadedMetadata(false) - { } +DirectoryLayer::Node::Node(Reference const& directoryLayer, + Optional const& subspace, + IDirectory::Path const& path, + IDirectory::Path const& targetPath) + : directoryLayer(directoryLayer), subspace(subspace), path(path), targetPath(targetPath), loadedMetadata(false) {} - bool DirectoryLayer::Node::exists() const { - return subspace.present(); - } +bool DirectoryLayer::Node::exists() const { + return subspace.present(); +} - ACTOR Future loadMetadata(DirectoryLayer::Node *n, Reference tr) { - if(!n->exists()){ - n->loadedMetadata = true; - return *n; - } - - Optional> layer = wait(tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY))); - - n->layer = layer.present() ? layer.get() : Standalone(); +ACTOR Future loadMetadata(DirectoryLayer::Node* n, Reference tr) { + if (!n->exists()) { n->loadedMetadata = true; - return *n; } - //Calls to loadMetadata must keep the Node alive while the future is outstanding - Future DirectoryLayer::Node::loadMetadata(Reference tr) { - return FDB::loadMetadata(this, tr); - } + Optional> layer = wait(tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY))); - bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const { - ASSERT(loadedMetadata); - return exists() && layer == DirectoryLayer::PARTITION_LAYER && (includeEmptySubpath || targetPath.size() > path.size()); - } + n->layer = layer.present() ? layer.get() : Standalone(); + n->loadedMetadata = true; - IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const { - return Path(targetPath.begin() + path.size(), targetPath.end()); - } - - Reference DirectoryLayer::Node::getContents() const { - ASSERT(exists()); - ASSERT(loadedMetadata); - - return directoryLayer->contentsOfNode(subspace.get(), path, layer); + return *n; } + +// Calls to loadMetadata must keep the Node alive while the future is outstanding +Future DirectoryLayer::Node::loadMetadata(Reference tr) { + return FDB::loadMetadata(this, tr); } + +bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const { + ASSERT(loadedMetadata); + return exists() && layer == DirectoryLayer::PARTITION_LAYER && + (includeEmptySubpath || targetPath.size() > path.size()); +} + +IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const { + return Path(targetPath.begin() + path.size(), targetPath.end()); +} + +Reference DirectoryLayer::Node::getContents() const { + ASSERT(exists()); + ASSERT(loadedMetadata); + + return directoryLayer->contentsOfNode(subspace.get(), path, layer); +} +} // namespace FDB diff --git a/bindings/flow/Subspace.cpp b/bindings/flow/Subspace.cpp old mode 100755 new mode 100644 index a86ffb25e3..896e67f551 --- a/bindings/flow/Subspace.cpp +++ b/bindings/flow/Subspace.cpp @@ -21,71 +21,72 @@ #include "Subspace.h" namespace FDB { - Subspace::Subspace(Tuple const& tuple, StringRef const& rawPrefix){ - StringRef packed = tuple.pack(); +Subspace::Subspace(Tuple const& tuple, StringRef const& rawPrefix) { + StringRef packed = tuple.pack(); - this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + packed.size()); - this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); - this->rawPrefix.append(this->rawPrefix.arena(), packed.begin(), packed.size()); + this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + packed.size()); + this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); + this->rawPrefix.append(this->rawPrefix.arena(), packed.begin(), packed.size()); +} + +Subspace::Subspace(Tuple const& tuple, Standalone> const& rawPrefix) { + this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + tuple.pack().size()); + this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); + this->rawPrefix.append(this->rawPrefix.arena(), tuple.pack().begin(), tuple.pack().size()); +} + +Subspace::Subspace(StringRef const& rawPrefix) { + this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); +} + +Subspace::~Subspace() {} + +Key Subspace::key() const { + return StringRef(rawPrefix.begin(), rawPrefix.size()); +} + +Key Subspace::pack(const Tuple& tuple) const { + return tuple.pack().withPrefix(StringRef(rawPrefix.begin(), rawPrefix.size())); +} + +Tuple Subspace::unpack(StringRef const& key) const { + if (!contains(key)) { + throw key_not_in_subspace(); } + return Tuple::unpack(key.substr(rawPrefix.size())); +} - Subspace::Subspace(Tuple const& tuple, Standalone> const& rawPrefix) { - this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + tuple.pack().size()); - this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); - this->rawPrefix.append(this->rawPrefix.arena(), tuple.pack().begin(), tuple.pack().size()); - } +KeyRange Subspace::range(Tuple const& tuple) const { + VectorRef begin; + VectorRef end; - Subspace::Subspace(StringRef const& rawPrefix){ - this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); - } + KeyRange keyRange; - Subspace::~Subspace() { } + begin.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1); + begin.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size()); + begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); + begin.push_back(keyRange.arena(), uint8_t('\x00')); - Key Subspace::key() const { - return StringRef(rawPrefix.begin(), rawPrefix.size()); - } + end.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1); + end.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size()); + end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); + end.push_back(keyRange.arena(), uint8_t('\xff')); - Key Subspace::pack(const Tuple& tuple) const { - return tuple.pack().withPrefix(StringRef(rawPrefix.begin(), rawPrefix.size())); - } + // FIXME: test that this uses the keyRange arena and doesn't create another one + keyRange.KeyRangeRef::operator=( + KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size()))); + return keyRange; +} - Tuple Subspace::unpack(StringRef const& key) const { - if (!contains(key)) { - throw key_not_in_subspace(); - } - return Tuple::unpack(key.substr(rawPrefix.size())); - } +bool Subspace::contains(KeyRef const& key) const { + return key.startsWith(StringRef(rawPrefix.begin(), rawPrefix.size())); +} - KeyRange Subspace::range(Tuple const& tuple) const { - VectorRef begin; - VectorRef end; +Subspace Subspace::subspace(Tuple const& tuple) const { + return Subspace(tuple, rawPrefix); +} - KeyRange keyRange; - - begin.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1); - begin.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size()); - begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); - begin.push_back(keyRange.arena(), uint8_t('\x00')); - - end.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1); - end.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size()); - end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); - end.push_back(keyRange.arena(), uint8_t('\xff')); - - // FIXME: test that this uses the keyRange arena and doesn't create another one - keyRange.KeyRangeRef::operator=(KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size()))); - return keyRange; - } - - bool Subspace::contains(KeyRef const& key) const { - return key.startsWith(StringRef(rawPrefix.begin(), rawPrefix.size())); - } - - Subspace Subspace::subspace(Tuple const& tuple) const { - return Subspace(tuple, rawPrefix); - } - - Subspace Subspace::get(Tuple const& tuple) const { - return subspace(tuple); - } -} \ No newline at end of file +Subspace Subspace::get(Tuple const& tuple) const { + return subspace(tuple); +} +} // namespace FDB \ No newline at end of file diff --git a/bindings/flow/Subspace.h b/bindings/flow/Subspace.h old mode 100755 new mode 100644 index ffd22ac02c..bdbf0e4d04 --- a/bindings/flow/Subspace.h +++ b/bindings/flow/Subspace.h @@ -28,65 +28,65 @@ #include "Tuple.h" namespace FDB { - class Subspace { - public: - Subspace(Tuple const& tuple = Tuple(), StringRef const& rawPrefix = StringRef()); - Subspace(StringRef const& rawPrefix); +class Subspace { +public: + Subspace(Tuple const& tuple = Tuple(), StringRef const& rawPrefix = StringRef()); + Subspace(StringRef const& rawPrefix); - virtual ~Subspace(); + virtual ~Subspace(); - virtual Key key() const; - virtual bool contains(KeyRef const& key) const; + virtual Key key() const; + virtual bool contains(KeyRef const& key) const; - virtual Key pack(Tuple const& tuple = Tuple()) const; - virtual Tuple unpack(KeyRef const& key) const; - virtual KeyRange range(Tuple const& tuple = Tuple()) const; + virtual Key pack(Tuple const& tuple = Tuple()) const; + virtual Tuple unpack(KeyRef const& key) const; + virtual KeyRange range(Tuple const& tuple = Tuple()) const; - template - Key pack(T const& item) const { - Tuple t; - t.append(item); - return pack(t); - } + template + Key pack(T const& item) const { + Tuple t; + t.append(item); + return pack(t); + } - Key packNested(Tuple const& item) const { - Tuple t; - t.appendNested(item); - return pack(t); - } + Key packNested(Tuple const& item) const { + Tuple t; + t.appendNested(item); + return pack(t); + } - Key pack(StringRef const& item, bool utf8=false) const { - Tuple t; - t.append(item, utf8); - return pack(t); - } + Key pack(StringRef const& item, bool utf8 = false) const { + Tuple t; + t.append(item, utf8); + return pack(t); + } - virtual Subspace subspace(Tuple const& tuple) const; - virtual Subspace get(Tuple const& tuple) const; + virtual Subspace subspace(Tuple const& tuple) const; + virtual Subspace get(Tuple const& tuple) const; - template - Subspace get(T const& item) const { - Tuple t; - t.append(item); - return get(t); - } + template + Subspace get(T const& item) const { + Tuple t; + t.append(item); + return get(t); + } - Subspace getNested(Tuple const& item) const { - Tuple t; - t.appendNested(item); - return get(t); - } + Subspace getNested(Tuple const& item) const { + Tuple t; + t.appendNested(item); + return get(t); + } - Subspace get(StringRef const& item, bool utf8=false) const { - Tuple t; - t.append(item, utf8); - return get(t); - } + Subspace get(StringRef const& item, bool utf8 = false) const { + Tuple t; + t.append(item, utf8); + return get(t); + } - private: - Subspace(Tuple const& tuple, Standalone> const& rawPrefix); - Standalone> rawPrefix; - }; -} +private: + Subspace(Tuple const& tuple, Standalone> const& rawPrefix); + Standalone> rawPrefix; +}; +} // namespace FDB #endif \ No newline at end of file diff --git a/bindings/flow/Tuple.cpp b/bindings/flow/Tuple.cpp old mode 100755 new mode 100644 index 8ce411dce4..760f44ad9f --- a/bindings/flow/Tuple.cpp +++ b/bindings/flow/Tuple.cpp @@ -22,623 +22,602 @@ #include namespace FDB { - // The floating point operations depend on this using the IEEE 754 standard. - BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); - BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); +// The floating point operations depend on this using the IEEE 754 standard. +BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); +BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); - const size_t Uuid::SIZE = 16; +const size_t Uuid::SIZE = 16; - const uint8_t Tuple::NULL_CODE = 0x00; - const uint8_t Tuple::BYTES_CODE = 0x01; - const uint8_t Tuple::STRING_CODE = 0x02; - const uint8_t Tuple::NESTED_CODE = 0x05; - const uint8_t Tuple::INT_ZERO_CODE = 0x14; - const uint8_t Tuple::POS_INT_END = 0x1c; - const uint8_t Tuple::NEG_INT_START = 0x0c; - const uint8_t Tuple::FLOAT_CODE = 0x20; - const uint8_t Tuple::DOUBLE_CODE = 0x21; - const uint8_t Tuple::FALSE_CODE = 0x26; - const uint8_t Tuple::TRUE_CODE = 0x27; - const uint8_t Tuple::UUID_CODE = 0x30; +const uint8_t Tuple::NULL_CODE = 0x00; +const uint8_t Tuple::BYTES_CODE = 0x01; +const uint8_t Tuple::STRING_CODE = 0x02; +const uint8_t Tuple::NESTED_CODE = 0x05; +const uint8_t Tuple::INT_ZERO_CODE = 0x14; +const uint8_t Tuple::POS_INT_END = 0x1c; +const uint8_t Tuple::NEG_INT_START = 0x0c; +const uint8_t Tuple::FLOAT_CODE = 0x20; +const uint8_t Tuple::DOUBLE_CODE = 0x21; +const uint8_t Tuple::FALSE_CODE = 0x26; +const uint8_t Tuple::TRUE_CODE = 0x27; +const uint8_t Tuple::UUID_CODE = 0x30; - static float bigEndianFloat(float orig) { - int32_t big = *(int32_t*)&orig; - big = bigEndian32(big); - return *(float*)&big; +static float bigEndianFloat(float orig) { + int32_t big = *(int32_t*)&orig; + big = bigEndian32(big); + return *(float*)&big; +} + +static double bigEndianDouble(double orig) { + int64_t big = *(int64_t*)&orig; + big = bigEndian64(big); + return *(double*)&big; +} + +static size_t find_string_terminator(const StringRef data, size_t offset) { + size_t i = offset; + while (i < data.size() - 1 && !(data[i] == (uint8_t)'\x00' && data[i + 1] != (uint8_t)'\xff')) { + i += (data[i] == '\x00' ? 2 : 1); } - static double bigEndianDouble(double orig) { - int64_t big = *(int64_t*)&orig; - big = bigEndian64(big); - return *(double*)&big; + return i; +} + +static size_t find_string_terminator(const Standalone> data, size_t offset) { + size_t i = offset; + while (i < data.size() - 1 && !(data[i] == '\x00' && data[i + 1] != (uint8_t)'\xff')) { + i += (data[i] == '\x00' ? 2 : 1); } - static size_t find_string_terminator(const StringRef data, size_t offset) { - size_t i = offset; - while (i < data.size() - 1 && !(data[i] == (uint8_t)'\x00' && data[i+1] != (uint8_t)'\xff')) { - i += (data[i] == '\x00' ? 2 : 1); + return i; +} + +// If encoding and the sign bit is 1 (the number is negative), flip all the bits. +// If decoding and the sign bit is 0 (the number is negative), flip all the bits. +// Otherwise, the number is positive, so flip the sign bit. +static void adjust_floating_point(uint8_t* bytes, size_t size, bool encode) { + if ((encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x00)) || + (!encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x80))) { + for (size_t i = 0; i < size; i++) { + bytes[i] ^= (uint8_t)0xff; } - - return i; + } else { + bytes[0] ^= (uint8_t)0x80; } +} - static size_t find_string_terminator(const Standalone > data, size_t offset ) { - size_t i = offset; - while (i < data.size() - 1 && !(data[i] == '\x00' && data[i+1] != (uint8_t)'\xff')) { - i += (data[i] == '\x00' ? 2 : 1); - } +Tuple::Tuple(StringRef const& str) { + data.append(data.arena(), str.begin(), str.size()); - return i; - } + size_t i = 0; + int depth = 0; + while (i < data.size()) { + if (depth == 0) + offsets.push_back(i); - // If encoding and the sign bit is 1 (the number is negative), flip all the bits. - // If decoding and the sign bit is 0 (the number is negative), flip all the bits. - // Otherwise, the number is positive, so flip the sign bit. - static void adjust_floating_point(uint8_t *bytes, size_t size, bool encode) { - if((encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x00)) || (!encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x80))) { - for(size_t i = 0; i < size; i++) { - bytes[i] ^= (uint8_t)0xff; + if (depth > 0 && data[i] == NULL_CODE) { + if (i + 1 < data.size() && data[i + 1] == 0xff) { + // NULL value. + i += 2; + } else { + // Nested terminator. + i += 1; + depth -= 1; } + } else if (data[i] == BYTES_CODE || data[i] == STRING_CODE) { + i = find_string_terminator(str, i + 1) + 1; + } else if (data[i] >= NEG_INT_START && data[i] <= POS_INT_END) { + i += abs(data[i] - INT_ZERO_CODE) + 1; + } else if (data[i] == NULL_CODE || data[i] == TRUE_CODE || data[i] == FALSE_CODE) { + i += 1; + } else if (data[i] == UUID_CODE) { + i += Uuid::SIZE + 1; + } else if (data[i] == FLOAT_CODE) { + i += sizeof(float) + 1; + } else if (data[i] == DOUBLE_CODE) { + i += sizeof(double) + 1; + } else if (data[i] == NESTED_CODE) { + i += 1; + depth += 1; } else { - bytes[0] ^= (uint8_t)0x80; + throw invalid_tuple_data_type(); } } - Tuple::Tuple(StringRef const& str) { - data.append(data.arena(), str.begin(), str.size()); + if (depth != 0) { + throw invalid_tuple_data_type(); + } +} - size_t i = 0; - int depth = 0; - while(i < data.size()) { - if(depth == 0) offsets.push_back(i); +// Note: this is destructive of the original offsets, so should only +// be used once we are done. +Tuple::Tuple(Standalone> data, std::vector offsets) { + this->data = data; + this->offsets = std::move(offsets); +} - if(depth > 0 && data[i] == NULL_CODE) { - if(i + 1 < data.size() && data[i+1] == 0xff) { - // NULL value. +Tuple Tuple::unpack(StringRef const& str) { + return Tuple(str); +} + +Tuple& Tuple::append(Tuple const& tuple) { + for (size_t offset : tuple.offsets) { + offsets.push_back(offset + data.size()); + } + + data.append(data.arena(), tuple.data.begin(), tuple.data.size()); + + return *this; +} + +Tuple& Tuple::append(StringRef const& str, bool utf8) { + offsets.push_back(data.size()); + + const uint8_t utfChar = utf8 ? STRING_CODE : BYTES_CODE; + data.append(data.arena(), &utfChar, 1); + + size_t lastPos = 0; + for (size_t pos = 0; pos < str.size(); ++pos) { + if (str[pos] == '\x00') { + data.append(data.arena(), str.begin() + lastPos, pos - lastPos); + data.push_back(data.arena(), (uint8_t)'\x00'); + data.push_back(data.arena(), (uint8_t)'\xff'); + lastPos = pos + 1; + } + } + + data.append(data.arena(), str.begin() + lastPos, str.size() - lastPos); + data.push_back(data.arena(), (uint8_t)'\x00'); + + return *this; +} + +Tuple& Tuple::append(int32_t value) { + return append((int64_t)value); +} + +Tuple& Tuple::append(int64_t value) { + uint64_t swap = value; + bool neg = false; + + offsets.push_back(data.size()); + + if (value < 0) { + value = ~(-value); + neg = true; + } + + swap = bigEndian64(value); + + for (int i = 0; i < 8; i++) { + if (((uint8_t*)&swap)[i] != (neg ? 255 : 0)) { + data.push_back(data.arena(), (uint8_t)(INT_ZERO_CODE + (8 - i) * (neg ? -1 : 1))); + data.append(data.arena(), ((const uint8_t*)&swap) + i, 8 - i); + return *this; + } + } + + data.push_back(data.arena(), INT_ZERO_CODE); + return *this; +} + +Tuple& Tuple::append(bool value) { + offsets.push_back(data.size()); + if (value) { + data.push_back(data.arena(), TRUE_CODE); + } else { + data.push_back(data.arena(), FALSE_CODE); + } + return *this; +} + +Tuple& Tuple::append(float value) { + offsets.push_back(data.size()); + float swap = bigEndianFloat(value); + uint8_t* bytes = (uint8_t*)&swap; + adjust_floating_point(bytes, sizeof(float), true); + + data.push_back(data.arena(), FLOAT_CODE); + data.append(data.arena(), bytes, sizeof(float)); + return *this; +} + +Tuple& Tuple::append(double value) { + offsets.push_back(data.size()); + double swap = value; + swap = bigEndianDouble(swap); + uint8_t* bytes = (uint8_t*)&swap; + adjust_floating_point(bytes, sizeof(double), true); + + data.push_back(data.arena(), DOUBLE_CODE); + data.append(data.arena(), bytes, sizeof(double)); + return *this; +} + +Tuple& Tuple::append(Uuid value) { + offsets.push_back(data.size()); + data.push_back(data.arena(), UUID_CODE); + data.append(data.arena(), value.getData().begin(), Uuid::SIZE); + return *this; +} + +Tuple& Tuple::appendNested(Tuple const& value) { + offsets.push_back(data.size()); + data.push_back(data.arena(), NESTED_CODE); + + for (size_t i = 0; i < value.size(); i++) { + size_t offset = value.offsets[i]; + size_t next_offset = (i + 1 < value.offsets.size() ? value.offsets[i + 1] : value.data.size()); + ASSERT(offset < value.data.size()); + ASSERT(next_offset <= value.data.size()); + uint8_t code = value.data[offset]; + if (code == NULL_CODE) { + data.push_back(data.arena(), NULL_CODE); + data.push_back(data.arena(), 0xff); + } else { + data.append(data.arena(), value.data.begin() + offset, next_offset - offset); + } + } + + data.push_back(data.arena(), (uint8_t)'\x00'); + + return *this; +} + +Tuple& Tuple::appendNull() { + offsets.push_back(data.size()); + data.push_back(data.arena(), NULL_CODE); + return *this; +} + +Tuple::ElementType Tuple::getType(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + + uint8_t code = data[offsets[index]]; + + if (code == NULL_CODE) { + return ElementType::NULL_TYPE; + } else if (code == BYTES_CODE) { + return ElementType::BYTES; + } else if (code == STRING_CODE) { + return ElementType::UTF8; + } else if (code == NESTED_CODE) { + return ElementType::NESTED; + } else if (code >= NEG_INT_START && code <= POS_INT_END) { + return ElementType::INT; + } else if (code == FLOAT_CODE) { + return ElementType::FLOAT; + } else if (code == DOUBLE_CODE) { + return ElementType::DOUBLE; + } else if (code == FALSE_CODE || code == TRUE_CODE) { + return ElementType::BOOL; + } else if (code == UUID_CODE) { + return ElementType::UUID; + } else { + throw invalid_tuple_data_type(); + } +} + +Standalone Tuple::getString(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + + uint8_t code = data[offsets[index]]; + if (code != BYTES_CODE && code != STRING_CODE) { + throw invalid_tuple_data_type(); + } + + size_t b = offsets[index] + 1; + size_t e; + if (offsets.size() > index + 1) { + e = offsets[index + 1]; + } else { + e = data.size(); + } + + Standalone result; + VectorRef staging; + + for (size_t i = b; i < e; ++i) { + if (data[i] == '\x00') { + staging.append(result.arena(), data.begin() + b, i - b); + ++i; + b = i + 1; + + if (i < e) { + staging.push_back(result.arena(), '\x00'); + } + } + } + + if (b < e) { + staging.append(result.arena(), data.begin() + b, e - b); + } + + result.StringRef::operator=(StringRef(staging.begin(), staging.size())); + return result; +} + +int64_t Tuple::getInt(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + + int64_t swap; + bool neg = false; + + ASSERT(offsets[index] < data.size()); + uint8_t code = data[offsets[index]]; + if (code < NEG_INT_START || code > POS_INT_END) { + throw invalid_tuple_data_type(); + } + + int8_t len = code - INT_ZERO_CODE; + + if (len < 0) { + len = -len; + neg = true; + } + + memset(&swap, neg ? '\xff' : 0, 8 - len); + memcpy(((uint8_t*)&swap) + 8 - len, data.begin() + offsets[index] + 1, len); + + swap = bigEndian64(swap); + + if (neg) { + swap = -(~swap); + } + + return swap; +} + +bool Tuple::getBool(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + ASSERT(offsets[index] < data.size()); + uint8_t code = data[offsets[index]]; + if (code == FALSE_CODE) { + return false; + } else if (code == TRUE_CODE) { + return true; + } else { + throw invalid_tuple_data_type(); + } +} + +float Tuple::getFloat(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + ASSERT(offsets[index] < data.size()); + uint8_t code = data[offsets[index]]; + if (code != FLOAT_CODE) { + throw invalid_tuple_data_type(); + } + + float swap; + uint8_t* bytes = (uint8_t*)&swap; + ASSERT(offsets[index] + 1 + sizeof(float) <= data.size()); + swap = *(float*)(data.begin() + offsets[index] + 1); + adjust_floating_point(bytes, sizeof(float), false); + + return bigEndianFloat(swap); +} + +double Tuple::getDouble(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + ASSERT(offsets[index] < data.size()); + uint8_t code = data[offsets[index]]; + if (code != DOUBLE_CODE) { + throw invalid_tuple_data_type(); + } + + double swap; + uint8_t* bytes = (uint8_t*)&swap; + ASSERT(offsets[index] + 1 + sizeof(double) <= data.size()); + swap = *(double*)(data.begin() + offsets[index] + 1); + adjust_floating_point(bytes, sizeof(double), false); + + return bigEndianDouble(swap); +} + +Uuid Tuple::getUuid(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + size_t offset = offsets[index]; + ASSERT(offset < data.size()); + uint8_t code = data[offset]; + if (code != UUID_CODE) { + throw invalid_tuple_data_type(); + } + ASSERT(offset + Uuid::SIZE + 1 <= data.size()); + StringRef uuidData(data.begin() + offset + 1, Uuid::SIZE); + return Uuid(uuidData); +} + +Tuple Tuple::getNested(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + size_t offset = offsets[index]; + ASSERT(offset < data.size()); + uint8_t code = data[offset]; + if (code != NESTED_CODE) { + throw invalid_tuple_data_type(); + } + + size_t next_offset = (index + 1 < offsets.size() ? offsets[index + 1] : data.size()); + ASSERT(next_offset <= data.size()); + ASSERT(data[next_offset - 1] == (uint8_t)0x00); + Standalone> dest; + dest.reserve(dest.arena(), next_offset - offset); + std::vector dest_offsets; + + size_t i = offset + 1; + int depth = 0; + while (i < next_offset - 1) { + if (depth == 0) + dest_offsets.push_back(dest.size()); + uint8_t code = data[i]; + dest.push_back(dest.arena(), code); // Copy over the type code. + if (code == NULL_CODE) { + if (depth > 0) { + if (i + 1 < next_offset - 1 && data[i + 1] == 0xff) { + // Null with a tuple nested in the nested tuple. + dest.push_back(dest.arena(), 0xff); i += 2; } else { // Nested terminator. - i += 1; depth -= 1; + i += 1; } - } - else if(data[i] == BYTES_CODE || data[i] == STRING_CODE) { - i = find_string_terminator(str, i+1) + 1; - } - else if(data[i] >= NEG_INT_START && data[i] <= POS_INT_END) { - i += abs(data[i] - INT_ZERO_CODE) + 1; - } - else if(data[i] == NULL_CODE || data[i] == TRUE_CODE || data[i] == FALSE_CODE) { - i += 1; - } - else if(data[i] == UUID_CODE) { - i += Uuid::SIZE + 1; - } - else if(data[i] == FLOAT_CODE) { - i += sizeof(float) + 1; - } - else if(data[i] == DOUBLE_CODE) { - i += sizeof(double) + 1; - } - else if(data[i] == NESTED_CODE) { - i += 1; - depth += 1; - } - else { - throw invalid_tuple_data_type(); - } - } - - if(depth != 0) { - throw invalid_tuple_data_type(); - } - } - - // Note: this is destructive of the original offsets, so should only - // be used once we are done. - Tuple::Tuple(Standalone> data, std::vector offsets) { - this->data = data; - this->offsets = std::move(offsets); - } - - Tuple Tuple::unpack(StringRef const& str) { - return Tuple(str); - } - - Tuple& Tuple::append(Tuple const& tuple) { - for(size_t offset : tuple.offsets) { - offsets.push_back(offset + data.size()); - } - - data.append(data.arena(), tuple.data.begin(), tuple.data.size()); - - return *this; - } - - Tuple& Tuple::append(StringRef const& str, bool utf8) { - offsets.push_back(data.size()); - - const uint8_t utfChar = utf8 ? STRING_CODE : BYTES_CODE; - data.append(data.arena(), &utfChar, 1); - - size_t lastPos = 0; - for(size_t pos = 0; pos < str.size(); ++pos) { - if(str[pos] == '\x00') { - data.append(data.arena(), str.begin() + lastPos, pos - lastPos); - data.push_back(data.arena(), (uint8_t)'\x00'); - data.push_back(data.arena(), (uint8_t)'\xff'); - lastPos = pos + 1; - } - } - - data.append(data.arena(), str.begin() + lastPos, str.size() - lastPos); - data.push_back(data.arena(), (uint8_t)'\x00'); - - return *this; - } - - Tuple& Tuple::append( int32_t value ) { - return append((int64_t)value); - } - - Tuple& Tuple::append( int64_t value ) { - uint64_t swap = value; - bool neg = false; - - offsets.push_back( data.size() ); - - if ( value < 0 ) { - value = ~(-value); - neg = true; - } - - swap = bigEndian64(value); - - for ( int i = 0; i < 8; i++ ) { - if ( ((uint8_t*)&swap)[i] != (neg ? 255 : 0) ) { - data.push_back( data.arena(), (uint8_t)(INT_ZERO_CODE + (8-i) * (neg ? -1 : 1)) ); - data.append( data.arena(), ((const uint8_t *)&swap) + i, 8 - i ); - return *this; - } - } - - data.push_back( data.arena(), INT_ZERO_CODE ); - return *this; - } - - Tuple& Tuple::append( bool value ) { - offsets.push_back( data.size() ); - if(value) { - data.push_back( data.arena(), TRUE_CODE ); - } else { - data.push_back( data.arena(), FALSE_CODE ); - } - return *this; - } - - Tuple& Tuple::append( float value ) { - offsets.push_back( data.size() ); - float swap = bigEndianFloat(value); - uint8_t *bytes = (uint8_t*)&swap; - adjust_floating_point(bytes, sizeof(float), true); - - data.push_back( data.arena(), FLOAT_CODE ); - data.append( data.arena(), bytes, sizeof(float) ); - return *this; - } - - Tuple& Tuple::append( double value ) { - offsets.push_back( data.size() ); - double swap = value; - swap = bigEndianDouble(swap); - uint8_t *bytes = (uint8_t*)&swap; - adjust_floating_point(bytes, sizeof(double), true); - - data.push_back( data.arena(), DOUBLE_CODE ); - data.append( data.arena(), bytes, sizeof(double) ); - return *this; - } - - Tuple& Tuple::append( Uuid value ) { - offsets.push_back( data.size() ); - data.push_back( data.arena(), UUID_CODE ); - data.append( data.arena(), value.getData().begin(), Uuid::SIZE ); - return *this; - } - - Tuple& Tuple::appendNested( Tuple const& value ) { - offsets.push_back( data.size() ); - data.push_back( data.arena(), NESTED_CODE ); - - for(size_t i = 0; i < value.size(); i++) { - size_t offset = value.offsets[i]; - size_t next_offset = (i+1 < value.offsets.size() ? value.offsets[i+1] : value.data.size()); - ASSERT(offset < value.data.size()); - ASSERT(next_offset <= value.data.size()); - uint8_t code = value.data[offset]; - if(code == NULL_CODE) { - data.push_back( data.arena(), NULL_CODE ); - data.push_back( data.arena(), 0xff ); } else { - data.append( data.arena(), value.data.begin() + offset, next_offset - offset); - } - } - - data.push_back( data.arena(), (uint8_t)'\x00'); - - return *this; - } - - Tuple& Tuple::appendNull() { - offsets.push_back(data.size()); - data.push_back(data.arena(), NULL_CODE); - return *this; - } - - Tuple::ElementType Tuple::getType(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - - uint8_t code = data[offsets[index]]; - - if(code == NULL_CODE) { - return ElementType::NULL_TYPE; - } - else if(code == BYTES_CODE) { - return ElementType::BYTES; - } - else if(code == STRING_CODE) { - return ElementType::UTF8; - } - else if(code == NESTED_CODE) { - return ElementType::NESTED; - } - else if(code >= NEG_INT_START && code <= POS_INT_END) { - return ElementType::INT; - } - else if(code == FLOAT_CODE) { - return ElementType::FLOAT; - } - else if(code == DOUBLE_CODE) { - return ElementType::DOUBLE; - } - else if(code == FALSE_CODE || code == TRUE_CODE) { - return ElementType::BOOL; - } - else if(code == UUID_CODE) { - return ElementType::UUID; - } - else { - throw invalid_tuple_data_type(); - } - } - - Standalone Tuple::getString(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - - uint8_t code = data[offsets[index]]; - if(code != BYTES_CODE && code != STRING_CODE) { - throw invalid_tuple_data_type(); - } - - size_t b = offsets[index] + 1; - size_t e; - if (offsets.size() > index + 1) { - e = offsets[index+1]; - } else { - e = data.size(); - } - - Standalone result; - VectorRef staging; - - for (size_t i = b; i < e; ++i) { - if(data[i] == '\x00') { - staging.append(result.arena(), data.begin() + b, i - b); - ++i; - b = i + 1; - - if(i < e) { - staging.push_back(result.arena(), '\x00'); - } - } - } - - if(b < e) { - staging.append(result.arena(), data.begin() + b, e - b); - } - - result.StringRef::operator=(StringRef(staging.begin(), staging.size())); - return result; - } - - int64_t Tuple::getInt(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - - int64_t swap; - bool neg = false; - - ASSERT(offsets[index] < data.size()); - uint8_t code = data[offsets[index]]; - if(code < NEG_INT_START || code > POS_INT_END) { - throw invalid_tuple_data_type(); - } - - int8_t len = code - INT_ZERO_CODE; - - if ( len < 0 ) { - len = -len; - neg = true; - } - - memset( &swap, neg ? '\xff' : 0, 8 - len ); - memcpy( ((uint8_t*)&swap) + 8 - len, data.begin() + offsets[index] + 1, len ); - - swap = bigEndian64( swap ); - - if ( neg ) { - swap = -(~swap); - } - - return swap; - } - - bool Tuple::getBool(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - ASSERT(offsets[index] < data.size()); - uint8_t code = data[offsets[index]]; - if(code == FALSE_CODE) { - return false; - } else if(code == TRUE_CODE) { - return true; - } else { - throw invalid_tuple_data_type(); - } - } - - float Tuple::getFloat(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - ASSERT(offsets[index] < data.size()); - uint8_t code = data[offsets[index]]; - if(code != FLOAT_CODE) { - throw invalid_tuple_data_type(); - } - - float swap; - uint8_t* bytes = (uint8_t*)&swap; - ASSERT(offsets[index] + 1 + sizeof(float) <= data.size()); - swap = *(float*)(data.begin() + offsets[index] + 1); - adjust_floating_point( bytes, sizeof(float), false ); - - return bigEndianFloat(swap); - } - - double Tuple::getDouble(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - ASSERT(offsets[index] < data.size()); - uint8_t code = data[offsets[index]]; - if(code != DOUBLE_CODE) { - throw invalid_tuple_data_type(); - } - - double swap; - uint8_t* bytes = (uint8_t*)&swap; - ASSERT(offsets[index] + 1 + sizeof(double) <= data.size()); - swap = *(double*)(data.begin() + offsets[index] + 1); - adjust_floating_point( bytes, sizeof(double), false ); - - return bigEndianDouble(swap); - } - - Uuid Tuple::getUuid(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - size_t offset = offsets[index]; - ASSERT(offset < data.size()); - uint8_t code = data[offset]; - if(code != UUID_CODE) { - throw invalid_tuple_data_type(); - } - ASSERT(offset + Uuid::SIZE + 1 <= data.size()); - StringRef uuidData(data.begin() + offset + 1, Uuid::SIZE); - return Uuid(uuidData); - } - - Tuple Tuple::getNested(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - size_t offset = offsets[index]; - ASSERT(offset < data.size()); - uint8_t code = data[offset]; - if(code != NESTED_CODE) { - throw invalid_tuple_data_type(); - } - - size_t next_offset = (index + 1 < offsets.size() ? offsets[index+1] : data.size()); - ASSERT(next_offset <= data.size()); - ASSERT(data[next_offset - 1] == (uint8_t)0x00); - Standalone> dest; - dest.reserve(dest.arena(), next_offset - offset); - std::vector dest_offsets; - - size_t i = offset + 1; - int depth = 0; - while(i < next_offset - 1) { - if (depth == 0) dest_offsets.push_back(dest.size()); - uint8_t code = data[i]; - dest.push_back(dest.arena(), code); // Copy over the type code. - if(code == NULL_CODE) { - if(depth > 0) { - if(i + 1 < next_offset - 1 && data[i+1] == 0xff) { - // Null with a tuple nested in the nested tuple. - dest.push_back(dest.arena(), 0xff); - i += 2; - } else { - // Nested terminator. - depth -= 1; - i += 1; - } - } else { - // A null object within the nested tuple. - ASSERT(i + 1 < next_offset - 1); - ASSERT(data[i+1] == 0xff); - i += 2; - } - } - else if(code == BYTES_CODE || code == STRING_CODE) { - size_t next_i = find_string_terminator(data, i+1) + 1; - ASSERT(next_i <= next_offset - 1); - size_t length = next_i - i - 1; - dest.append(dest.arena(), data.begin() + i + 1, length); - i = next_i; - } - else if(code >= NEG_INT_START && code <= POS_INT_END) { - size_t int_size = abs(code - INT_ZERO_CODE); - ASSERT(i + int_size <= next_offset - 1); - dest.append(dest.arena(), data.begin() + i + 1, int_size); - i += int_size + 1; - } - else if(code == TRUE_CODE || code == FALSE_CODE) { - i += 1; - } - else if(code == UUID_CODE) { - ASSERT(i + 1 + Uuid::SIZE <= next_offset - 1); - dest.append(dest.arena(), data.begin() + i + 1, Uuid::SIZE); - i += Uuid::SIZE + 1; - } - else if(code == FLOAT_CODE) { - ASSERT(i + 1 + sizeof(float) <= next_offset - 1); - dest.append(dest.arena(), data.begin() + i + 1, sizeof(float)); - i += sizeof(float) + 1; - } - else if(code == DOUBLE_CODE) { - ASSERT(i + 1 + sizeof(double) <= next_offset - 1); - dest.append(dest.arena(), data.begin() + i + 1, sizeof(double)); - i += sizeof(double) + 1; - } - else if(code == NESTED_CODE) { - i += 1; - depth += 1; - } - else { - throw invalid_tuple_data_type(); - } - } - - // The item may shrink because of escaped nulls that are unespaced. - return Tuple(dest, dest_offsets); - } - - KeyRange Tuple::range(Tuple const& tuple) const { - VectorRef begin; - VectorRef end; - - KeyRange keyRange; - - begin.reserve(keyRange.arena(), data.size() + tuple.pack().size() + 1); - begin.append(keyRange.arena(), data.begin(), data.size()); - begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); - begin.push_back(keyRange.arena(), uint8_t('\x00')); - - end.reserve(keyRange.arena(), data.size() + tuple.pack().size() + 1); - end.append(keyRange.arena(), data.begin(), data.size()); - end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); - end.push_back(keyRange.arena(), uint8_t('\xff')); - - keyRange.KeyRangeRef::operator=(KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size()))); - return keyRange; - } - - Tuple Tuple::subTuple(size_t start, size_t end) const { - if(start >= offsets.size() || end <= start) { - return Tuple(); - } - - size_t endPos = end < offsets.size() ? offsets[end] : data.size(); - return Tuple(StringRef(data.begin() + offsets[start], endPos - offsets[start])); - } - - // Comparisons - int compare(Standalone > const& v1, Standalone > const& v2) { - size_t i = 0; - while(i < v1.size() && i < v2.size()) { - if(v1[i] < v2[i]) { - return -1; - } else if(v1[i] > v2[i]) { - return 1; + // A null object within the nested tuple. + ASSERT(i + 1 < next_offset - 1); + ASSERT(data[i + 1] == 0xff); + i += 2; } + } else if (code == BYTES_CODE || code == STRING_CODE) { + size_t next_i = find_string_terminator(data, i + 1) + 1; + ASSERT(next_i <= next_offset - 1); + size_t length = next_i - i - 1; + dest.append(dest.arena(), data.begin() + i + 1, length); + i = next_i; + } else if (code >= NEG_INT_START && code <= POS_INT_END) { + size_t int_size = abs(code - INT_ZERO_CODE); + ASSERT(i + int_size <= next_offset - 1); + dest.append(dest.arena(), data.begin() + i + 1, int_size); + i += int_size + 1; + } else if (code == TRUE_CODE || code == FALSE_CODE) { i += 1; + } else if (code == UUID_CODE) { + ASSERT(i + 1 + Uuid::SIZE <= next_offset - 1); + dest.append(dest.arena(), data.begin() + i + 1, Uuid::SIZE); + i += Uuid::SIZE + 1; + } else if (code == FLOAT_CODE) { + ASSERT(i + 1 + sizeof(float) <= next_offset - 1); + dest.append(dest.arena(), data.begin() + i + 1, sizeof(float)); + i += sizeof(float) + 1; + } else if (code == DOUBLE_CODE) { + ASSERT(i + 1 + sizeof(double) <= next_offset - 1); + dest.append(dest.arena(), data.begin() + i + 1, sizeof(double)); + i += sizeof(double) + 1; + } else if (code == NESTED_CODE) { + i += 1; + depth += 1; + } else { + throw invalid_tuple_data_type(); } + } - if(i < v1.size()) { + // The item may shrink because of escaped nulls that are unespaced. + return Tuple(dest, dest_offsets); +} + +KeyRange Tuple::range(Tuple const& tuple) const { + VectorRef begin; + VectorRef end; + + KeyRange keyRange; + + begin.reserve(keyRange.arena(), data.size() + tuple.pack().size() + 1); + begin.append(keyRange.arena(), data.begin(), data.size()); + begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); + begin.push_back(keyRange.arena(), uint8_t('\x00')); + + end.reserve(keyRange.arena(), data.size() + tuple.pack().size() + 1); + end.append(keyRange.arena(), data.begin(), data.size()); + end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); + end.push_back(keyRange.arena(), uint8_t('\xff')); + + keyRange.KeyRangeRef::operator=( + KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size()))); + return keyRange; +} + +Tuple Tuple::subTuple(size_t start, size_t end) const { + if (start >= offsets.size() || end <= start) { + return Tuple(); + } + + size_t endPos = end < offsets.size() ? offsets[end] : data.size(); + return Tuple(StringRef(data.begin() + offsets[start], endPos - offsets[start])); +} + +// Comparisons +int compare(Standalone> const& v1, Standalone> const& v2) { + size_t i = 0; + while (i < v1.size() && i < v2.size()) { + if (v1[i] < v2[i]) { + return -1; + } else if (v1[i] > v2[i]) { return 1; } - if(i < v2.size()) { - return -1; - } - return 0; + i += 1; } - bool Tuple::operator==(Tuple const& other) const { - return compare(data, other.data) == 0; + if (i < v1.size()) { + return 1; } - bool Tuple::operator!=(Tuple const& other) const { - return compare(data, other.data) != 0; - } - bool Tuple::operator<(Tuple const& other) const { - return compare(data, other.data) < 0; - } - bool Tuple::operator<=(Tuple const& other) const { - return compare(data, other.data) <= 0; - } - bool Tuple::operator>(Tuple const& other) const { - return compare(data, other.data) > 0; - } - bool Tuple::operator>=(Tuple const& other) const { - return compare(data, other.data) >= 0; - } - - // UUID implementation - Uuid::Uuid(const StringRef& data) { - if (data.size() != Uuid::SIZE) { - throw invalid_uuid_size(); - } - this->data = data; - } - - StringRef Uuid::getData() const { - return data; - } - - bool Uuid::operator==(Uuid const& other) const { - return data == other.data; - } - bool Uuid::operator!=(Uuid const& other) const { - return data != other.data; - } - bool Uuid::operator<(Uuid const& other) const { - return data < other.data; - } - bool Uuid::operator<=(Uuid const& other) const { - return data <= other.data; - } - bool Uuid::operator>(Uuid const& other) const { - return data > other.data; - } - bool Uuid::operator>=(Uuid const& other) const { - return data >= other.data; + if (i < v2.size()) { + return -1; } + return 0; } + +bool Tuple::operator==(Tuple const& other) const { + return compare(data, other.data) == 0; +} +bool Tuple::operator!=(Tuple const& other) const { + return compare(data, other.data) != 0; +} +bool Tuple::operator<(Tuple const& other) const { + return compare(data, other.data) < 0; +} +bool Tuple::operator<=(Tuple const& other) const { + return compare(data, other.data) <= 0; +} +bool Tuple::operator>(Tuple const& other) const { + return compare(data, other.data) > 0; +} +bool Tuple::operator>=(Tuple const& other) const { + return compare(data, other.data) >= 0; +} + +// UUID implementation +Uuid::Uuid(const StringRef& data) { + if (data.size() != Uuid::SIZE) { + throw invalid_uuid_size(); + } + this->data = data; +} + +StringRef Uuid::getData() const { + return data; +} + +bool Uuid::operator==(Uuid const& other) const { + return data == other.data; +} +bool Uuid::operator!=(Uuid const& other) const { + return data != other.data; +} +bool Uuid::operator<(Uuid const& other) const { + return data < other.data; +} +bool Uuid::operator<=(Uuid const& other) const { + return data <= other.data; +} +bool Uuid::operator>(Uuid const& other) const { + return data > other.data; +} +bool Uuid::operator>=(Uuid const& other) const { + return data >= other.data; +} +} // namespace FDB diff --git a/bindings/flow/Tuple.h b/bindings/flow/Tuple.h old mode 100755 new mode 100644 index 25867ef71f..e26b489c61 --- a/bindings/flow/Tuple.h +++ b/bindings/flow/Tuple.h @@ -26,92 +26,93 @@ #include "bindings/flow/fdb_flow.h" namespace FDB { - struct Uuid { - const static size_t SIZE; +struct Uuid { + const static size_t SIZE; - Uuid(StringRef const& data); + Uuid(StringRef const& data); - StringRef getData() const; + StringRef getData() const; - // Comparisons - bool operator==(Uuid const& other) const; - bool operator!=(Uuid const& other) const; - bool operator<(Uuid const& other) const; - bool operator<=(Uuid const& other) const; - bool operator>(Uuid const& other) const; - bool operator>=(Uuid const& other) const; - private: - Standalone data; - }; + // Comparisons + bool operator==(Uuid const& other) const; + bool operator!=(Uuid const& other) const; + bool operator<(Uuid const& other) const; + bool operator<=(Uuid const& other) const; + bool operator>(Uuid const& other) const; + bool operator>=(Uuid const& other) const; - struct Tuple { - Tuple() {} +private: + Standalone data; +}; - static Tuple unpack(StringRef const& str); +struct Tuple { + Tuple() {} - Tuple& append(Tuple const& tuple); - Tuple& append(StringRef const& str, bool utf8=false); - Tuple& append(int32_t); - Tuple& append(int64_t); - Tuple& append(bool); - Tuple& append(float); - Tuple& append(double); - Tuple& append(Uuid); - Tuple& appendNested(Tuple const&); - Tuple& appendNull(); + static Tuple unpack(StringRef const& str); - StringRef pack() const { return StringRef(data.begin(), data.size()); } + Tuple& append(Tuple const& tuple); + Tuple& append(StringRef const& str, bool utf8 = false); + Tuple& append(int32_t); + Tuple& append(int64_t); + Tuple& append(bool); + Tuple& append(float); + Tuple& append(double); + Tuple& append(Uuid); + Tuple& appendNested(Tuple const&); + Tuple& appendNull(); - template - Tuple& operator<<(T const& t) { - return append(t); - } + StringRef pack() const { return StringRef(data.begin(), data.size()); } - enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED }; + template + Tuple& operator<<(T const& t) { + return append(t); + } - // this is number of elements, not length of data - size_t size() const { return offsets.size(); } + enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED }; - ElementType getType(size_t index) const; - Standalone getString(size_t index) const; - int64_t getInt(size_t index) const; - bool getBool(size_t index) const; - float getFloat(size_t index) const; - double getDouble(size_t index) const; - Uuid getUuid(size_t index) const; - Tuple getNested(size_t index) const; + // this is number of elements, not length of data + size_t size() const { return offsets.size(); } - KeyRange range(Tuple const& tuple = Tuple()) const; + ElementType getType(size_t index) const; + Standalone getString(size_t index) const; + int64_t getInt(size_t index) const; + bool getBool(size_t index) const; + float getFloat(size_t index) const; + double getDouble(size_t index) const; + Uuid getUuid(size_t index) const; + Tuple getNested(size_t index) const; - Tuple subTuple(size_t beginIndex, size_t endIndex = std::numeric_limits::max()) const; + KeyRange range(Tuple const& tuple = Tuple()) const; - // Comparisons - bool operator==(Tuple const& other) const; - bool operator!=(Tuple const& other) const; - bool operator<(Tuple const& other) const; - bool operator<=(Tuple const& other) const; - bool operator>(Tuple const& other) const; - bool operator>=(Tuple const& other) const; + Tuple subTuple(size_t beginIndex, size_t endIndex = std::numeric_limits::max()) const; - private: - static const uint8_t NULL_CODE; - static const uint8_t BYTES_CODE; - static const uint8_t STRING_CODE; - static const uint8_t NESTED_CODE; - static const uint8_t INT_ZERO_CODE; - static const uint8_t POS_INT_END; - static const uint8_t NEG_INT_START; - static const uint8_t FLOAT_CODE; - static const uint8_t DOUBLE_CODE; - static const uint8_t FALSE_CODE; - static const uint8_t TRUE_CODE; - static const uint8_t UUID_CODE; + // Comparisons + bool operator==(Tuple const& other) const; + bool operator!=(Tuple const& other) const; + bool operator<(Tuple const& other) const; + bool operator<=(Tuple const& other) const; + bool operator>(Tuple const& other) const; + bool operator>=(Tuple const& other) const; - Tuple(const StringRef& data); - Tuple(Standalone> data, std::vector offsets); - Standalone> data; - std::vector offsets; - }; -} +private: + static const uint8_t NULL_CODE; + static const uint8_t BYTES_CODE; + static const uint8_t STRING_CODE; + static const uint8_t NESTED_CODE; + static const uint8_t INT_ZERO_CODE; + static const uint8_t POS_INT_END; + static const uint8_t NEG_INT_START; + static const uint8_t FLOAT_CODE; + static const uint8_t DOUBLE_CODE; + static const uint8_t FALSE_CODE; + static const uint8_t TRUE_CODE; + static const uint8_t UUID_CODE; + + Tuple(const StringRef& data); + Tuple(Standalone> data, std::vector offsets); + Standalone> data; + std::vector offsets; +}; +} // namespace FDB #endif /* _FDB_TUPLE_H_ */ diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.actor.cpp index a6ddadba8a..c5738de2da 100644 --- a/bindings/flow/fdb_flow.actor.cpp +++ b/bindings/flow/fdb_flow.actor.cpp @@ -36,41 +36,42 @@ THREAD_FUNC networkThread(void* fdb) { } ACTOR Future _test() { - API *fdb = FDB::API::selectAPIVersion(620); + API* fdb = FDB::API::selectAPIVersion(620); auto db = fdb->createDatabase(); state Reference tr = db->createTransaction(); // tr->setVersion(1); - Version ver = wait( tr->getReadVersion() ); + Version ver = wait(tr->getReadVersion()); printf("%" PRId64 "\n", ver); - state std::vector< Future > versions; + state std::vector> versions; state double starttime = timer_monotonic(); state int i; // for (i = 0; i < 100000; i++) { // Version v = wait( tr->getReadVersion() ); // } - for ( i = 0; i < 100000; i++ ) { - versions.push_back( tr->getReadVersion() ); + for (i = 0; i < 100000; i++) { + versions.push_back(tr->getReadVersion()); } - for ( i = 0; i < 100000; i++ ) { - Version v = wait( versions[i] ); + for (i = 0; i < 100000; i++) { + Version v = wait(versions[i]); } // wait( waitForAllReady( versions ) ); - printf("Elapsed: %lf\n", timer_monotonic() - starttime ); + printf("Elapsed: %lf\n", timer_monotonic() - starttime); - tr->set( LiteralStringRef("foo"), LiteralStringRef("bar") ); + tr->set(LiteralStringRef("foo"), LiteralStringRef("bar")); - Optional< FDBStandalone > v = wait( tr->get( LiteralStringRef("foo") ) ); - if ( v.present() ) { - printf("%s\n", v.get().toString().c_str() ); + Optional> v = wait(tr->get(LiteralStringRef("foo"))); + if (v.present()) { + printf("%s\n", v.get().toString().c_str()); } - FDBStandalone r = wait( tr->getRange( KeyRangeRef( LiteralStringRef("a"), LiteralStringRef("z") ), 100 ) ); + FDBStandalone r = + wait(tr->getRange(KeyRangeRef(LiteralStringRef("a"), LiteralStringRef("z")), 100)); - for ( auto kv : r ) { + for (auto kv : r) { printf("%s is %s\n", kv.key.toString().c_str(), kv.value.toString().c_str()); } @@ -79,7 +80,7 @@ ACTOR Future _test() { } void fdb_flow_test() { - API *fdb = FDB::API::selectAPIVersion(620); + API* fdb = FDB::API::selectAPIVersion(620); fdb->setupNetwork(); startThread(networkThread, fdb); @@ -95,343 +96,372 @@ void fdb_flow_test() { } namespace FDB { - class DatabaseImpl : public Database, NonCopyable { - public: - virtual ~DatabaseImpl() { fdb_database_destroy(db); } +class DatabaseImpl : public Database, NonCopyable { +public: + virtual ~DatabaseImpl() { fdb_database_destroy(db); } - Reference createTransaction() override; - void setDatabaseOption(FDBDatabaseOption option, Optional value = Optional()) override; + Reference createTransaction() override; + void setDatabaseOption(FDBDatabaseOption option, Optional value = Optional()) override; - private: - FDBDatabase* db; - explicit DatabaseImpl(FDBDatabase* db) : db(db) {} +private: + FDBDatabase* db; + explicit DatabaseImpl(FDBDatabase* db) : db(db) {} - friend class API; - }; + friend class API; +}; - class TransactionImpl : public Transaction, private NonCopyable, public FastAllocated { - friend class DatabaseImpl; +class TransactionImpl : public Transaction, private NonCopyable, public FastAllocated { + friend class DatabaseImpl; - public: - virtual ~TransactionImpl() { - if (tr) { - fdb_transaction_destroy(tr); - } +public: + virtual ~TransactionImpl() { + if (tr) { + fdb_transaction_destroy(tr); } - - void setReadVersion(Version v) override; - Future getReadVersion() override; - - Future>> get(const Key& key, bool snapshot = false) override; - Future> getKey(const KeySelector& key, bool snapshot = false) override; - - Future watch(const Key& key) override; - - using Transaction::getRange; - Future> getRange(const KeySelector& begin, const KeySelector& end, - GetRangeLimits limits = GetRangeLimits(), bool snapshot = false, - bool reverse = false, - FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override; - - void addReadConflictRange(KeyRangeRef const& keys) override; - void addReadConflictKey(KeyRef const& key) override; - void addWriteConflictRange(KeyRangeRef const& keys) override; - void addWriteConflictKey(KeyRef const& key) override; - - void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) override; - void set(const KeyRef& key, const ValueRef& value) override; - void clear(const KeyRangeRef& range) override; - void clear(const KeyRef& key) override; - - Future commit() override; - Version getCommittedVersion() override; - Future> getVersionstamp() override; - - void setOption(FDBTransactionOption option, Optional value = Optional()) override; - - Future getApproximateSize() override; - Future onError(Error const& e) override; - - void cancel() override; - void reset() override; - - TransactionImpl() : tr(NULL) {} - TransactionImpl(TransactionImpl&& r) BOOST_NOEXCEPT { - tr = r.tr; - r.tr = NULL; - } - TransactionImpl& operator=(TransactionImpl&& r) BOOST_NOEXCEPT { - tr = r.tr; - r.tr = NULL; - return *this; - } - - private: - FDBTransaction* tr; - - explicit TransactionImpl(FDBDatabase* db); - }; - - static inline void throw_on_error( fdb_error_t e ) { - if (e) - throw Error(e); } - void CFuture::blockUntilReady() { - throw_on_error( fdb_future_block_until_ready( f ) ); + void setReadVersion(Version v) override; + Future getReadVersion() override; + + Future>> get(const Key& key, bool snapshot = false) override; + Future> getKey(const KeySelector& key, bool snapshot = false) override; + + Future watch(const Key& key) override; + + using Transaction::getRange; + Future> getRange(const KeySelector& begin, + const KeySelector& end, + GetRangeLimits limits = GetRangeLimits(), + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override; + + void addReadConflictRange(KeyRangeRef const& keys) override; + void addReadConflictKey(KeyRef const& key) override; + void addWriteConflictRange(KeyRangeRef const& keys) override; + void addWriteConflictKey(KeyRef const& key) override; + + void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) override; + void set(const KeyRef& key, const ValueRef& value) override; + void clear(const KeyRangeRef& range) override; + void clear(const KeyRef& key) override; + + Future commit() override; + Version getCommittedVersion() override; + Future> getVersionstamp() override; + + void setOption(FDBTransactionOption option, Optional value = Optional()) override; + + Future getApproximateSize() override; + Future onError(Error const& e) override; + + void cancel() override; + void reset() override; + + TransactionImpl() : tr(NULL) {} + TransactionImpl(TransactionImpl&& r) BOOST_NOEXCEPT { + tr = r.tr; + r.tr = NULL; + } + TransactionImpl& operator=(TransactionImpl&& r) BOOST_NOEXCEPT { + tr = r.tr; + r.tr = NULL; + return *this; } - void backToFutureCallback( FDBFuture* f, void* data ) { - g_network->onMainThread( Promise((SAV*)data), TaskPriority::DefaultOnMainThread ); // SOMEDAY: think about this priority - } +private: + FDBTransaction* tr; - // backToFuture( FDBFuture*, (FDBFuture* -> Type) ) -> Future - // Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread) - // and converts it into a Future (with callbacks working on this thread, cancellation etc). - // You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type - ACTOR template static Future backToFuture( FDBFuture* _f, Function convertValue ) { - state Reference f( new CFuture(_f) ); + explicit TransactionImpl(FDBDatabase* db); +}; - Promise ready; - Future onReady = ready.getFuture(); +static inline void throw_on_error(fdb_error_t e) { + if (e) + throw Error(e); +} - throw_on_error( fdb_future_set_callback( f->f, backToFutureCallback, ready.extractRawPointer() ) ); - wait( onReady ); +void CFuture::blockUntilReady() { + throw_on_error(fdb_future_block_until_ready(f)); +} - return convertValue( f ); - } +void backToFutureCallback(FDBFuture* f, void* data) { + g_network->onMainThread(Promise((SAV*)data), + TaskPriority::DefaultOnMainThread); // SOMEDAY: think about this priority +} - void API::setNetworkOption( FDBNetworkOption option, Optional value ) { - if ( value.present() ) - throw_on_error( fdb_network_set_option( option, value.get().begin(), value.get().size() ) ); - else - throw_on_error( fdb_network_set_option( option, NULL, 0 ) ); - } +// backToFuture( FDBFuture*, (FDBFuture* -> Type) ) -> Future +// Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread) +// and converts it into a Future (with callbacks working on this thread, cancellation etc). +// You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type +ACTOR template +static Future backToFuture(FDBFuture* _f, Function convertValue) { + state Reference f(new CFuture(_f)); - API* API::instance = NULL; - API::API(int version) : version(version) {} + Promise ready; + Future onReady = ready.getFuture(); - API* API::selectAPIVersion(int apiVersion) { - if(API::instance) { - if(apiVersion != API::instance->version) { - throw api_version_already_set(); - } - else { - return API::instance; - } - } + throw_on_error(fdb_future_set_callback(f->f, backToFutureCallback, ready.extractRawPointer())); + wait(onReady); - if(apiVersion < 500 || apiVersion > FDB_API_VERSION) { - throw api_version_not_supported(); - } + return convertValue(f); +} - throw_on_error( fdb_select_api_version_impl(apiVersion, FDB_API_VERSION) ); +void API::setNetworkOption(FDBNetworkOption option, Optional value) { + if (value.present()) + throw_on_error(fdb_network_set_option(option, value.get().begin(), value.get().size())); + else + throw_on_error(fdb_network_set_option(option, NULL, 0)); +} - API::instance = new API(apiVersion); - return API::instance; - } +API* API::instance = NULL; +API::API(int version) : version(version) {} - bool API::isAPIVersionSelected() { - return API::instance != NULL; - } - - API* API::getInstance() { - if(API::instance == NULL) { - throw api_version_unset(); - } - else { +API* API::selectAPIVersion(int apiVersion) { + if (API::instance) { + if (apiVersion != API::instance->version) { + throw api_version_already_set(); + } else { return API::instance; } } - void API::setupNetwork() { - throw_on_error( fdb_setup_network() ); + if (apiVersion < 500 || apiVersion > FDB_API_VERSION) { + throw api_version_not_supported(); } - void API::runNetwork() { - throw_on_error( fdb_run_network() ); + throw_on_error(fdb_select_api_version_impl(apiVersion, FDB_API_VERSION)); + + API::instance = new API(apiVersion); + return API::instance; +} + +bool API::isAPIVersionSelected() { + return API::instance != NULL; +} + +API* API::getInstance() { + if (API::instance == NULL) { + throw api_version_unset(); + } else { + return API::instance; } +} - void API::stopNetwork() { - throw_on_error( fdb_stop_network() ); +void API::setupNetwork() { + throw_on_error(fdb_setup_network()); +} + +void API::runNetwork() { + throw_on_error(fdb_run_network()); +} + +void API::stopNetwork() { + throw_on_error(fdb_stop_network()); +} + +bool API::evaluatePredicate(FDBErrorPredicate pred, Error const& e) { + return fdb_error_predicate(pred, e.code()); +} + +Reference API::createDatabase(std::string const& connFilename) { + FDBDatabase* db; + throw_on_error(fdb_create_database(connFilename.c_str(), &db)); + return Reference(new DatabaseImpl(db)); +} + +int API::getAPIVersion() const { + return version; +} + +Reference DatabaseImpl::createTransaction() { + return Reference(new TransactionImpl(db)); +} + +void DatabaseImpl::setDatabaseOption(FDBDatabaseOption option, Optional value) { + if (value.present()) + throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size())); + else + throw_on_error(fdb_database_set_option(db, option, NULL, 0)); +} + +TransactionImpl::TransactionImpl(FDBDatabase* db) { + throw_on_error(fdb_database_create_transaction(db, &tr)); +} + +void TransactionImpl::setReadVersion(Version v) { + fdb_transaction_set_read_version(tr, v); +} + +Future TransactionImpl::getReadVersion() { + return backToFuture(fdb_transaction_get_read_version(tr), [](Reference f) { + Version value; + + throw_on_error(fdb_future_get_int64(f->f, &value)); + + return value; + }); +} + +Future>> TransactionImpl::get(const Key& key, bool snapshot) { + return backToFuture>>( + fdb_transaction_get(tr, key.begin(), key.size(), snapshot), [](Reference f) { + fdb_bool_t present; + uint8_t const* value; + int value_length; + + throw_on_error(fdb_future_get_value(f->f, &present, &value, &value_length)); + + if (present) { + return Optional>(FDBStandalone(f, ValueRef(value, value_length))); + } else { + return Optional>(); + } + }); +} + +Future TransactionImpl::watch(const Key& key) { + return backToFuture(fdb_transaction_watch(tr, key.begin(), key.size()), [](Reference f) { + throw_on_error(fdb_future_get_error(f->f)); + return Void(); + }); +} + +Future> TransactionImpl::getKey(const KeySelector& key, bool snapshot) { + return backToFuture>( + fdb_transaction_get_key(tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot), + [](Reference f) { + uint8_t const* key; + int key_length; + + throw_on_error(fdb_future_get_key(f->f, &key, &key_length)); + + return FDBStandalone(f, KeyRef(key, key_length)); + }); +} + +Future> TransactionImpl::getRange(const KeySelector& begin, + const KeySelector& end, + GetRangeLimits limits, + bool snapshot, + bool reverse, + FDBStreamingMode streamingMode) { + // FIXME: iteration + return backToFuture>( + fdb_transaction_get_range(tr, + begin.key.begin(), + begin.key.size(), + begin.orEqual, + begin.offset, + end.key.begin(), + end.key.size(), + end.orEqual, + end.offset, + limits.rows, + limits.bytes, + streamingMode, + 1, + snapshot, + reverse), + [](Reference f) { + FDBKeyValue const* kv; + int count; + fdb_bool_t more; + + throw_on_error(fdb_future_get_keyvalue_array(f->f, &kv, &count, &more)); + + return FDBStandalone(f, + RangeResultRef(VectorRef((KeyValueRef*)kv, count), more)); + }); +} + +void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) { + throw_on_error(fdb_transaction_add_conflict_range( + tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ)); +} + +void TransactionImpl::addReadConflictKey(KeyRef const& key) { + return addReadConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key)))); +} + +void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) { + throw_on_error(fdb_transaction_add_conflict_range( + tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE)); +} + +void TransactionImpl::addWriteConflictKey(KeyRef const& key) { + return addWriteConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key)))); +} + +void TransactionImpl::atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) { + fdb_transaction_atomic_op(tr, key.begin(), key.size(), operand.begin(), operand.size(), operationType); +} + +void TransactionImpl::set(const KeyRef& key, const ValueRef& value) { + fdb_transaction_set(tr, key.begin(), key.size(), value.begin(), value.size()); +} + +void TransactionImpl::clear(const KeyRangeRef& range) { + fdb_transaction_clear_range(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size()); +} + +void TransactionImpl::clear(const KeyRef& key) { + fdb_transaction_clear(tr, key.begin(), key.size()); +} + +Future TransactionImpl::commit() { + return backToFuture(fdb_transaction_commit(tr), [](Reference f) { + throw_on_error(fdb_future_get_error(f->f)); + return Void(); + }); +} + +Version TransactionImpl::getCommittedVersion() { + Version v; + + throw_on_error(fdb_transaction_get_committed_version(tr, &v)); + return v; +} + +Future> TransactionImpl::getVersionstamp() { + return backToFuture>(fdb_transaction_get_versionstamp(tr), [](Reference f) { + uint8_t const* key; + int key_length; + + throw_on_error(fdb_future_get_key(f->f, &key, &key_length)); + + return FDBStandalone(f, StringRef(key, key_length)); + }); +} + +void TransactionImpl::setOption(FDBTransactionOption option, Optional value) { + if (value.present()) { + throw_on_error(fdb_transaction_set_option(tr, option, value.get().begin(), value.get().size())); + } else { + throw_on_error(fdb_transaction_set_option(tr, option, NULL, 0)); } +} - bool API::evaluatePredicate(FDBErrorPredicate pred, Error const& e) { - return fdb_error_predicate( pred, e.code() ); - } +Future TransactionImpl::getApproximateSize() { + return backToFuture(fdb_transaction_get_approximate_size(tr), [](Reference f) { + int64_t size = 0; + throw_on_error(fdb_future_get_int64(f->f, &size)); + return size; + }); +} - Reference API::createDatabase(std::string const& connFilename) { - FDBDatabase *db; - throw_on_error(fdb_create_database(connFilename.c_str(), &db)); - return Reference(new DatabaseImpl(db)); - } +Future TransactionImpl::onError(Error const& e) { + return backToFuture(fdb_transaction_on_error(tr, e.code()), [](Reference f) { + throw_on_error(fdb_future_get_error(f->f)); + return Void(); + }); +} - int API::getAPIVersion() const { - return version; - } +void TransactionImpl::cancel() { + fdb_transaction_cancel(tr); +} - Reference DatabaseImpl::createTransaction() { - return Reference(new TransactionImpl(db)); - } +void TransactionImpl::reset() { + fdb_transaction_reset(tr); +} - void DatabaseImpl::setDatabaseOption(FDBDatabaseOption option, Optional value) { - if (value.present()) - throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size())); - else - throw_on_error(fdb_database_set_option(db, option, NULL, 0)); - } - - TransactionImpl::TransactionImpl(FDBDatabase* db) { - throw_on_error(fdb_database_create_transaction(db, &tr)); - } - - void TransactionImpl::setReadVersion(Version v) { - fdb_transaction_set_read_version( tr, v ); - } - - Future TransactionImpl::getReadVersion() { - return backToFuture( fdb_transaction_get_read_version( tr ), [](Reference f){ - Version value; - - throw_on_error( fdb_future_get_int64( f->f, &value ) ); - - return value; - } ); - } - - Future>> TransactionImpl::get(const Key& key, bool snapshot) { - return backToFuture< Optional> >( fdb_transaction_get( tr, key.begin(), key.size(), snapshot ), [](Reference f) { - fdb_bool_t present; - uint8_t const* value; - int value_length; - - throw_on_error( fdb_future_get_value( f->f, &present, &value, &value_length ) ); - - if ( present ) { - return Optional>( FDBStandalone( f, ValueRef( value, value_length ) ) ); - } else { - return Optional>(); - } - } ); - } - - Future TransactionImpl::watch(const Key& key) { - return backToFuture< Void >( fdb_transaction_watch( tr, key.begin(), key.size() ), [](Reference f) { - throw_on_error( fdb_future_get_error( f->f ) ); - return Void(); - } ); - } - - Future> TransactionImpl::getKey(const KeySelector& key, bool snapshot) { - return backToFuture< FDBStandalone >( fdb_transaction_get_key( tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot ), [](Reference f) { - uint8_t const* key; - int key_length; - - throw_on_error( fdb_future_get_key( f->f, &key, &key_length ) ); - - return FDBStandalone( f, KeyRef( key, key_length ) ); - } ); - } - - Future> TransactionImpl::getRange(const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, bool snapshot, bool reverse, FDBStreamingMode streamingMode) { - // FIXME: iteration - return backToFuture< FDBStandalone >( fdb_transaction_get_range( tr, begin.key.begin(), begin.key.size(), begin.orEqual, begin.offset, end.key.begin(), end.key.size(), end.orEqual, end.offset, limits.rows, limits.bytes, streamingMode, 1, snapshot, reverse ), [](Reference f) { - FDBKeyValue const* kv; - int count; - fdb_bool_t more; - - throw_on_error( fdb_future_get_keyvalue_array( f->f, &kv, &count, &more ) ); - - return FDBStandalone( f, RangeResultRef( VectorRef( (KeyValueRef*)kv, count ), more ) ); - } ); - } - - void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) { - throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ ) ); - } - - void TransactionImpl::addReadConflictKey(KeyRef const& key) { - return addReadConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key)))); - } - - void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) { - throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE ) ); - } - - void TransactionImpl::addWriteConflictKey(KeyRef const& key) { - return addWriteConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key)))); - } - - void TransactionImpl::atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) { - fdb_transaction_atomic_op( tr, key.begin(), key.size(), operand.begin(), operand.size(), operationType ); - } - - void TransactionImpl::set(const KeyRef& key, const ValueRef& value) { - fdb_transaction_set( tr, key.begin(), key.size(), value.begin(), value.size() ); - } - - void TransactionImpl::clear(const KeyRangeRef& range) { - fdb_transaction_clear_range( tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size() ); - } - - void TransactionImpl::clear(const KeyRef& key) { - fdb_transaction_clear( tr, key.begin(), key.size() ); - } - - Future TransactionImpl::commit() { - return backToFuture< Void >( fdb_transaction_commit( tr ), [](Reference f) { - throw_on_error( fdb_future_get_error( f->f ) ); - return Void(); - } ); - } - - Version TransactionImpl::getCommittedVersion() { - Version v; - - throw_on_error( fdb_transaction_get_committed_version( tr, &v ) ); - return v; - } - - Future> TransactionImpl::getVersionstamp() { - return backToFuture>(fdb_transaction_get_versionstamp(tr), [](Reference f) { - uint8_t const* key; - int key_length; - - throw_on_error( fdb_future_get_key( f->f, &key, &key_length ) ); - - return FDBStandalone( f, StringRef( key, key_length ) ); - }); - } - - void TransactionImpl::setOption(FDBTransactionOption option, Optional value) { - if ( value.present() ) { - throw_on_error( fdb_transaction_set_option( tr, option, value.get().begin(), value.get().size() ) ); - } else { - throw_on_error( fdb_transaction_set_option( tr, option, NULL, 0 ) ); - } - } - - Future TransactionImpl::getApproximateSize() { - return backToFuture(fdb_transaction_get_approximate_size(tr), [](Reference f) { - int64_t size = 0; - throw_on_error(fdb_future_get_int64(f->f, &size)); - return size; - }); - } - - Future TransactionImpl::onError(Error const& e) { - return backToFuture< Void >( fdb_transaction_on_error( tr, e.code() ), [](Reference f) { - throw_on_error( fdb_future_get_error( f->f ) ); - return Void(); - } ); - } - - void TransactionImpl::cancel() { - fdb_transaction_cancel( tr ); - } - - void TransactionImpl::reset() { - fdb_transaction_reset( tr ); - } - -} // namespace FDB +} // namespace FDB diff --git a/bindings/flow/fdb_flow.h b/bindings/flow/fdb_flow.h index 90f77e67cc..5f86219605 100644 --- a/bindings/flow/fdb_flow.h +++ b/bindings/flow/fdb_flow.h @@ -30,121 +30,137 @@ #include "FDBLoanerTypes.h" namespace FDB { - struct CFuture : NonCopyable, ReferenceCounted, FastAllocated { - CFuture() : f(NULL) {} - explicit CFuture(FDBFuture* f) : f(f) {} - ~CFuture() { - if (f) { - fdb_future_destroy(f); - } +struct CFuture : NonCopyable, ReferenceCounted, FastAllocated { + CFuture() : f(NULL) {} + explicit CFuture(FDBFuture* f) : f(f) {} + ~CFuture() { + if (f) { + fdb_future_destroy(f); } + } - void blockUntilReady(); + void blockUntilReady(); - FDBFuture* f; - }; + FDBFuture* f; +}; - template - class FDBStandalone : public T { - public: - FDBStandalone() {} - FDBStandalone(Reference f, T const& t) : T(t), f(f) {} - FDBStandalone(FDBStandalone const& o) : T((T const&)o), f(o.f) {} +template +class FDBStandalone : public T { +public: + FDBStandalone() {} + FDBStandalone(Reference f, T const& t) : T(t), f(f) {} + FDBStandalone(FDBStandalone const& o) : T((T const&)o), f(o.f) {} - private: - Reference f; - }; +private: + Reference f; +}; - class ReadTransaction : public ReferenceCounted { - public: - virtual ~ReadTransaction(){}; - virtual void setReadVersion(Version v) = 0; - virtual Future getReadVersion() = 0; +class ReadTransaction : public ReferenceCounted { +public: + virtual ~ReadTransaction(){}; + virtual void setReadVersion(Version v) = 0; + virtual Future getReadVersion() = 0; - virtual Future>> get(const Key& key, bool snapshot = false) = 0; - virtual Future> getKey(const KeySelector& key, bool snapshot = false) = 0; - virtual Future watch(const Key& key) = 0; + virtual Future>> get(const Key& key, bool snapshot = false) = 0; + virtual Future> getKey(const KeySelector& key, bool snapshot = false) = 0; + virtual Future watch(const Key& key) = 0; - virtual Future> getRange( - const KeySelector& begin, const KeySelector& end, GetRangeLimits limits = GetRangeLimits(), - bool snapshot = false, bool reverse = false, - FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0; - virtual Future> getRange( - const KeySelector& begin, const KeySelector& end, int limit, bool snapshot = false, bool reverse = false, - FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { - return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode); - } - virtual Future> getRange( - const KeyRange& keys, int limit, bool snapshot = false, bool reverse = false, - FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { - return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), - KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limit, snapshot, reverse, - streamingMode); - } - virtual Future> getRange( - const KeyRange& keys, GetRangeLimits limits = GetRangeLimits(), bool snapshot = false, bool reverse = false, - FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { - return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), - KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limits, snapshot, reverse, - streamingMode); - } + virtual Future> getRange( + const KeySelector& begin, + const KeySelector& end, + GetRangeLimits limits = GetRangeLimits(), + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0; + virtual Future> getRange(const KeySelector& begin, + const KeySelector& end, + int limit, + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { + return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode); + } + virtual Future> getRange(const KeyRange& keys, + int limit, + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { + return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), + KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), + limit, + snapshot, + reverse, + streamingMode); + } + virtual Future> getRange(const KeyRange& keys, + GetRangeLimits limits = GetRangeLimits(), + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { + return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), + KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), + limits, + snapshot, + reverse, + streamingMode); + } - virtual void addReadConflictRange(KeyRangeRef const& keys) = 0; - virtual void addReadConflictKey(KeyRef const& key) = 0; + virtual void addReadConflictRange(KeyRangeRef const& keys) = 0; + virtual void addReadConflictKey(KeyRef const& key) = 0; - virtual void setOption(FDBTransactionOption option, Optional value = Optional()) = 0; + virtual void setOption(FDBTransactionOption option, Optional value = Optional()) = 0; - virtual Future onError(Error const& e) = 0; + virtual Future onError(Error const& e) = 0; - virtual void cancel() = 0; - virtual void reset() = 0; - }; + virtual void cancel() = 0; + virtual void reset() = 0; +}; - class Transaction : public ReadTransaction { - public: - virtual void addWriteConflictRange(KeyRangeRef const& keys) = 0; - virtual void addWriteConflictKey(KeyRef const& key) = 0; +class Transaction : public ReadTransaction { +public: + virtual void addWriteConflictRange(KeyRangeRef const& keys) = 0; + virtual void addWriteConflictKey(KeyRef const& key) = 0; - virtual void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) = 0; - virtual void set(const KeyRef& key, const ValueRef& value) = 0; - virtual void clear(const KeyRangeRef& range) = 0; - virtual void clear(const KeyRef& key) = 0; + virtual void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) = 0; + virtual void set(const KeyRef& key, const ValueRef& value) = 0; + virtual void clear(const KeyRangeRef& range) = 0; + virtual void clear(const KeyRef& key) = 0; - virtual Future commit() = 0; - virtual Version getCommittedVersion() = 0; - virtual Future getApproximateSize() = 0; - virtual Future> getVersionstamp() = 0; - }; + virtual Future commit() = 0; + virtual Version getCommittedVersion() = 0; + virtual Future getApproximateSize() = 0; + virtual Future> getVersionstamp() = 0; +}; - class Database : public ReferenceCounted { - public: - virtual ~Database(){}; - virtual Reference createTransaction() = 0; - virtual void setDatabaseOption(FDBDatabaseOption option, Optional value = Optional()) = 0; - }; +class Database : public ReferenceCounted { +public: + virtual ~Database(){}; + virtual Reference createTransaction() = 0; + virtual void setDatabaseOption(FDBDatabaseOption option, Optional value = Optional()) = 0; +}; - class API { - public: - static API* selectAPIVersion(int apiVersion); - static API* getInstance(); - static bool isAPIVersionSelected(); +class API { +public: + static API* selectAPIVersion(int apiVersion); + static API* getInstance(); + static bool isAPIVersionSelected(); - void setNetworkOption(FDBNetworkOption option, Optional value = Optional()); + void setNetworkOption(FDBNetworkOption option, Optional value = Optional()); - void setupNetwork(); - void runNetwork(); - void stopNetwork(); + void setupNetwork(); + void runNetwork(); + void stopNetwork(); - Reference createDatabase(std::string const& connFilename = ""); + Reference createDatabase(std::string const& connFilename = ""); - bool evaluatePredicate(FDBErrorPredicate pred, Error const& e); - int getAPIVersion() const; + bool evaluatePredicate(FDBErrorPredicate pred, Error const& e); + int getAPIVersion() const; - private: - static API* instance; +private: + static API* instance; - API(int version); - int version; - }; - } // namespace FDB + API(int version); + int version; +}; +} // namespace FDB #endif // FDB_FLOW_FDB_FLOW_H diff --git a/bindings/flow/tester/DirectoryTester.actor.cpp b/bindings/flow/tester/DirectoryTester.actor.cpp index a04317792b..35b95dabd6 100644 --- a/bindings/flow/tester/DirectoryTester.actor.cpp +++ b/bindings/flow/tester/DirectoryTester.actor.cpp @@ -19,14 +19,14 @@ */ #include "Tester.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. +#include "flow/actorcompiler.h" // This must be the last #include. using namespace FDB; ACTOR Future> popTuples(Reference data, int count = 1) { state std::vector tuples; - while(tuples.size() < count) { + while (tuples.size() < count) { Standalone sizeStr = wait(data->stack.pop()[0].value); int size = Tuple::unpack(sizeStr).getInt(0); @@ -34,7 +34,7 @@ ACTOR Future> popTuples(Reference data, int c state Tuple tuple; state int index; - for(index = 0; index < tupleItems.size(); ++index) { + for (index = 0; index < tupleItems.size(); ++index) { Standalone itemStr = wait(tupleItems[index].value); tuple.append(Tuple::unpack(itemStr)); } @@ -54,9 +54,9 @@ ACTOR Future> popPaths(Reference d std::vector tuples = wait(popTuples(data, count)); std::vector paths; - for(auto &tuple : tuples) { + for (auto& tuple : tuples) { IDirectory::Path path; - for(int i = 0; i < tuple.size(); ++i) { + for (int i = 0; i < tuple.size(); ++i) { path.push_back(tuple.getString(i)); } @@ -74,9 +74,9 @@ ACTOR Future popPath(Reference data) { std::string pathToString(IDirectory::Path const& path) { std::string str; str += "["; - for(int i = 0; i < path.size(); ++i) { + for (int i = 0; i < path.size(); ++i) { str += path[i].toString(); - if(i < path.size() - 1) { + if (i < path.size() - 1) { str += ", "; } } @@ -86,21 +86,21 @@ std::string pathToString(IDirectory::Path const& path) { IDirectory::Path combinePaths(IDirectory::Path const& path1, IDirectory::Path const& path2) { IDirectory::Path outPath(path1.begin(), path1.end()); - for(auto p : path2) { + for (auto p : path2) { outPath.push_back(p); } return outPath; } -void logOp(std::string message, bool force=false) { - if(LOG_OPS || force) { +void logOp(std::string message, bool force = false) { + if (LOG_OPS || force) { printf("%s\n", message.c_str()); fflush(stdout); } } -//DIRECTORY_CREATE_SUBSPACE +// DIRECTORY_CREATE_SUBSPACE struct DirectoryCreateSubspaceFunc : InstructionFunc { static const char* name; @@ -108,7 +108,8 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc { state Tuple path = wait(popTuple(data)); Tuple rawPrefix = wait(data->stack.waitAndPop()); - logOp(format("Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str())); + logOp(format( + "Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str())); data->directoryData.push(new Subspace(path, rawPrefix.getString(0))); return Void(); } @@ -116,7 +117,7 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc { const char* DirectoryCreateSubspaceFunc::name = "DIRECTORY_CREATE_SUBSPACE"; REGISTER_INSTRUCTION_FUNC(DirectoryCreateSubspaceFunc); -//DIRECTORY_CREATE_LAYER +// DIRECTORY_CREATE_LAYER struct DirectoryCreateLayerFunc : InstructionFunc { static const char* name; @@ -127,15 +128,21 @@ struct DirectoryCreateLayerFunc : InstructionFunc { int index2 = args[1].getInt(0); bool allowManualPrefixes = args[2].getInt(0) != 0; - if(!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) { + if (!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) { logOp("Create directory layer: None"); data->directoryData.push(); - } - else { + } else { Subspace* nodeSubspace = data->directoryData.directoryList[index1].subspace.get(); Subspace* contentSubspace = data->directoryData.directoryList[index2].subspace.get(); - logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, allow_manual_prefixes = %d", index1, nodeSubspace->key().printable().c_str(), index2, nodeSubspace->key().printable().c_str(), allowManualPrefixes)); - data->directoryData.push(Reference(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes))); + logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, " + "allow_manual_prefixes = %d", + index1, + nodeSubspace->key().printable().c_str(), + index2, + nodeSubspace->key().printable().c_str(), + allowManualPrefixes)); + data->directoryData.push( + Reference(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes))); } return Void(); @@ -144,7 +151,7 @@ struct DirectoryCreateLayerFunc : InstructionFunc { const char* DirectoryCreateLayerFunc::name = "DIRECTORY_CREATE_LAYER"; REGISTER_INSTRUCTION_FUNC(DirectoryCreateLayerFunc); -//DIRECTORY_CHANGE +// DIRECTORY_CHANGE struct DirectoryChangeFunc : InstructionFunc { static const char* name; @@ -153,13 +160,17 @@ struct DirectoryChangeFunc : InstructionFunc { data->directoryData.directoryListIndex = index.getInt(0); ASSERT(data->directoryData.directoryListIndex < data->directoryData.directoryList.size()); - if(!data->directoryData.directoryList[data->directoryData.directoryListIndex].valid()) { + if (!data->directoryData.directoryList[data->directoryData.directoryListIndex].valid()) { data->directoryData.directoryListIndex = data->directoryData.directoryErrorIndex; } - if(LOG_DIRS) { + if (LOG_DIRS) { DirectoryOrSubspace d = data->directoryData.directoryList[data->directoryData.directoryListIndex]; - printf("Changed directory to %d (%s @\'%s\')\n", data->directoryData.directoryListIndex, d.typeString().c_str(), d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str() : d.subspace.get()->key().printable().c_str()); + printf("Changed directory to %d (%s @\'%s\')\n", + data->directoryData.directoryListIndex, + d.typeString().c_str(), + d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str() + : d.subspace.get()->key().printable().c_str()); fflush(stdout); } @@ -169,7 +180,7 @@ struct DirectoryChangeFunc : InstructionFunc { const char* DirectoryChangeFunc::name = "DIRECTORY_CHANGE"; REGISTER_INSTRUCTION_FUNC(DirectoryChangeFunc); -//DIRECTORY_SET_ERROR_INDEX +// DIRECTORY_SET_ERROR_INDEX struct DirectorySetErrorIndexFunc : InstructionFunc { static const char* name; @@ -183,7 +194,7 @@ struct DirectorySetErrorIndexFunc : InstructionFunc { const char* DirectorySetErrorIndexFunc::name = "DIRECTORY_SET_ERROR_INDEX"; REGISTER_INSTRUCTION_FUNC(DirectorySetErrorIndexFunc); -//DIRECTORY_CREATE_OR_OPEN +// DIRECTORY_CREATE_OR_OPEN struct DirectoryCreateOrOpenFunc : InstructionFunc { static const char* name; @@ -193,11 +204,12 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc { Standalone layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0); Reference directory = data->directoryData.directory(); - logOp(format("create_or_open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str())); + logOp(format("create_or_open %s: layer=%s", + pathToString(combinePaths(directory->getPath(), path)).c_str(), + layer.printable().c_str())); - Reference dirSubspace = wait(executeMutation(instruction, [this, directory, layer] () { - return directory->createOrOpen(instruction->tr, path, layer); - })); + Reference dirSubspace = wait(executeMutation( + instruction, [this, directory, layer]() { return directory->createOrOpen(instruction->tr, path, layer); })); data->directoryData.push(dirSubspace); @@ -207,7 +219,7 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc { const char* DirectoryCreateOrOpenFunc::name = "DIRECTORY_CREATE_OR_OPEN"; REGISTER_INSTRUCTION_FUNC(DirectoryCreateOrOpenFunc); -//DIRECTORY_CREATE +// DIRECTORY_CREATE struct DirectoryCreateFunc : InstructionFunc { static const char* name; @@ -215,14 +227,19 @@ struct DirectoryCreateFunc : InstructionFunc { state IDirectory::Path path = wait(popPath(data)); std::vector args = wait(data->stack.waitAndPop(2)); Standalone layer = args[0].getType(0) == Tuple::NULL_TYPE ? StringRef() : args[0].getString(0); - Optional> prefix = args[1].getType(0) == Tuple::NULL_TYPE ? Optional>() : args[1].getString(0); + Optional> prefix = + args[1].getType(0) == Tuple::NULL_TYPE ? Optional>() : args[1].getString(0); Reference directory = data->directoryData.directory(); - logOp(format("create %s: layer=%s, prefix=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str(), prefix.present() ? prefix.get().printable().c_str() : "")); + logOp(format("create %s: layer=%s, prefix=%s", + pathToString(combinePaths(directory->getPath(), path)).c_str(), + layer.printable().c_str(), + prefix.present() ? prefix.get().printable().c_str() : "")); - Reference dirSubspace = wait(executeMutation(instruction, [this, directory, layer, prefix] () { - return directory->create(instruction->tr, path, layer, prefix); - })); + Reference dirSubspace = + wait(executeMutation(instruction, [this, directory, layer, prefix]() { + return directory->create(instruction->tr, path, layer, prefix); + })); data->directoryData.push(dirSubspace); @@ -232,7 +249,7 @@ struct DirectoryCreateFunc : InstructionFunc { const char* DirectoryCreateFunc::name = "DIRECTORY_CREATE"; REGISTER_INSTRUCTION_FUNC(DirectoryCreateFunc); -//DIRECTORY_OPEN +// DIRECTORY_OPEN struct DirectoryOpenFunc : InstructionFunc { static const char* name; @@ -242,7 +259,9 @@ struct DirectoryOpenFunc : InstructionFunc { Standalone layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0); Reference directory = data->directoryData.directory(); - logOp(format("open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str())); + logOp(format("open %s: layer=%s", + pathToString(combinePaths(directory->getPath(), path)).c_str(), + layer.printable().c_str())); Reference dirSubspace = wait(directory->open(instruction->tr, path, layer)); data->directoryData.push(dirSubspace); @@ -252,7 +271,7 @@ struct DirectoryOpenFunc : InstructionFunc { const char* DirectoryOpenFunc::name = "DIRECTORY_OPEN"; REGISTER_INSTRUCTION_FUNC(DirectoryOpenFunc); -//DIRECTORY_MOVE +// DIRECTORY_MOVE struct DirectoryMoveFunc : InstructionFunc { static const char* name; @@ -260,11 +279,12 @@ struct DirectoryMoveFunc : InstructionFunc { std::vector paths = wait(popPaths(data, 2)); Reference directory = data->directoryData.directory(); - logOp(format("move %s to %s", pathToString(combinePaths(directory->getPath(), paths[0])).c_str(), pathToString(combinePaths(directory->getPath(), paths[1])).c_str())); + logOp(format("move %s to %s", + pathToString(combinePaths(directory->getPath(), paths[0])).c_str(), + pathToString(combinePaths(directory->getPath(), paths[1])).c_str())); - Reference dirSubspace = wait(executeMutation(instruction, [this, directory, paths] () { - return directory->move(instruction->tr, paths[0], paths[1]); - })); + Reference dirSubspace = wait(executeMutation( + instruction, [this, directory, paths]() { return directory->move(instruction->tr, paths[0], paths[1]); })); data->directoryData.push(dirSubspace); @@ -274,7 +294,7 @@ struct DirectoryMoveFunc : InstructionFunc { const char* DirectoryMoveFunc::name = "DIRECTORY_MOVE"; REGISTER_INSTRUCTION_FUNC(DirectoryMoveFunc); -//DIRECTORY_MOVE_TO +// DIRECTORY_MOVE_TO struct DirectoryMoveToFunc : InstructionFunc { static const char* name; @@ -284,9 +304,8 @@ struct DirectoryMoveToFunc : InstructionFunc { Reference directory = data->directoryData.directory(); logOp(format("move %s to %s", pathToString(directory->getPath()).c_str(), pathToString(path).c_str())); - Reference dirSubspace = wait(executeMutation(instruction, [this, directory, path] () { - return directory->moveTo(instruction->tr, path); - })); + Reference dirSubspace = wait(executeMutation( + instruction, [this, directory, path]() { return directory->moveTo(instruction->tr, path); })); data->directoryData.push(dirSubspace); @@ -296,27 +315,22 @@ struct DirectoryMoveToFunc : InstructionFunc { const char* DirectoryMoveToFunc::name = "DIRECTORY_MOVE_TO"; REGISTER_INSTRUCTION_FUNC(DirectoryMoveToFunc); -//DIRECTORY_REMOVE +// DIRECTORY_REMOVE struct DirectoryRemoveFunc : InstructionFunc { static const char* name; ACTOR static Future call(Reference data, Reference instruction) { Tuple count = wait(data->stack.waitAndPop()); state Reference directory = data->directoryData.directory(); - if(count.getInt(0) == 0) { + if (count.getInt(0) == 0) { logOp(format("remove %s", pathToString(directory->getPath()).c_str())); - wait(executeMutation(instruction, [this] () { - return directory->remove(instruction->tr); - })); - } - else { + wait(executeMutation(instruction, [this]() { return directory->remove(instruction->tr); })); + } else { IDirectory::Path path = wait(popPath(data)); logOp(format("remove %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); - wait(executeMutation(instruction, [this, path] () { - return directory->remove(instruction->tr, path); - })); + wait(executeMutation(instruction, [this, path]() { return directory->remove(instruction->tr, path); })); } return Void(); @@ -325,27 +339,24 @@ struct DirectoryRemoveFunc : InstructionFunc { const char* DirectoryRemoveFunc::name = "DIRECTORY_REMOVE"; REGISTER_INSTRUCTION_FUNC(DirectoryRemoveFunc); -//DIRECTORY_REMOVE_IF_EXISTS +// DIRECTORY_REMOVE_IF_EXISTS struct DirectoryRemoveIfExistsFunc : InstructionFunc { static const char* name; ACTOR static Future call(Reference data, Reference instruction) { Tuple count = wait(data->stack.waitAndPop()); state Reference directory = data->directoryData.directory(); - if(count.getInt(0) == 0) { + if (count.getInt(0) == 0) { logOp(format("remove_if_exists %s", pathToString(directory->getPath()).c_str())); - wait(success(executeMutation(instruction, [this] () { - return directory->removeIfExists(instruction->tr); - }))); - } - else { + wait( + success(executeMutation(instruction, [this]() { return directory->removeIfExists(instruction->tr); }))); + } else { IDirectory::Path path = wait(popPath(data)); logOp(format("remove_if_exists %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); - wait(success(executeMutation(instruction, [this, path] () { - return directory->removeIfExists(instruction->tr, path); - }))); + wait(success(executeMutation(instruction, + [this, path]() { return directory->removeIfExists(instruction->tr, path); }))); } return Void(); @@ -354,7 +365,7 @@ struct DirectoryRemoveIfExistsFunc : InstructionFunc { const char* DirectoryRemoveIfExistsFunc::name = "DIRECTORY_REMOVE_IF_EXISTS"; REGISTER_INSTRUCTION_FUNC(DirectoryRemoveIfExistsFunc); -//DIRECTORY_LIST +// DIRECTORY_LIST struct DirectoryListFunc : InstructionFunc { static const char* name; @@ -362,12 +373,11 @@ struct DirectoryListFunc : InstructionFunc { Tuple count = wait(data->stack.waitAndPop()); state Reference directory = data->directoryData.directory(); state Standalone> subdirs; - if(count.getInt(0) == 0) { + if (count.getInt(0) == 0) { logOp(format("list %s", pathToString(directory->getPath()).c_str())); Standalone> _subdirs = wait(directory->list(instruction->tr)); subdirs = _subdirs; - } - else { + } else { IDirectory::Path path = wait(popPath(data)); logOp(format("list %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); Standalone> _subdirs = wait(directory->list(instruction->tr, path)); @@ -375,7 +385,7 @@ struct DirectoryListFunc : InstructionFunc { } Tuple subdirTuple; - for(auto &sd : subdirs) { + for (auto& sd : subdirs) { subdirTuple.append(sd, true); } @@ -386,7 +396,7 @@ struct DirectoryListFunc : InstructionFunc { const char* DirectoryListFunc::name = "DIRECTORY_LIST"; REGISTER_INSTRUCTION_FUNC(DirectoryListFunc); -//DIRECTORY_EXISTS +// DIRECTORY_EXISTS struct DirectoryExistsFunc : InstructionFunc { static const char* name; @@ -394,12 +404,11 @@ struct DirectoryExistsFunc : InstructionFunc { Tuple count = wait(data->stack.waitAndPop()); state Reference directory = data->directoryData.directory(); state bool result; - if(count.getInt(0) == 0) { + if (count.getInt(0) == 0) { bool _result = wait(directory->exists(instruction->tr)); result = _result; logOp(format("exists %s: %d", pathToString(directory->getPath()).c_str(), result)); - } - else { + } else { state IDirectory::Path path = wait(popPath(data)); bool _result = wait(directory->exists(instruction->tr, path)); result = _result; @@ -413,7 +422,7 @@ struct DirectoryExistsFunc : InstructionFunc { const char* DirectoryExistsFunc::name = "DIRECTORY_EXISTS"; REGISTER_INSTRUCTION_FUNC(DirectoryExistsFunc); -//DIRECTORY_PACK_KEY +// DIRECTORY_PACK_KEY struct DirectoryPackKeyFunc : InstructionFunc { static const char* name; @@ -427,17 +436,19 @@ struct DirectoryPackKeyFunc : InstructionFunc { const char* DirectoryPackKeyFunc::name = "DIRECTORY_PACK_KEY"; REGISTER_INSTRUCTION_FUNC(DirectoryPackKeyFunc); -//DIRECTORY_UNPACK_KEY +// DIRECTORY_UNPACK_KEY struct DirectoryUnpackKeyFunc : InstructionFunc { static const char* name; ACTOR static Future call(Reference data, Reference instruction) { Tuple key = wait(data->stack.waitAndPop()); - Subspace *subspace = data->directoryData.subspace(); - logOp(format("Unpack %s in subspace with prefix %s", key.getString(0).printable().c_str(), subspace->key().printable().c_str())); + Subspace* subspace = data->directoryData.subspace(); + logOp(format("Unpack %s in subspace with prefix %s", + key.getString(0).printable().c_str(), + subspace->key().printable().c_str())); Tuple tuple = subspace->unpack(key.getString(0)); - for(int i = 0; i < tuple.size(); ++i) { - data->stack.push(tuple.subTuple(i, i+1).pack()); + for (int i = 0; i < tuple.size(); ++i) { + data->stack.push(tuple.subTuple(i, i + 1).pack()); } return Void(); @@ -446,7 +457,7 @@ struct DirectoryUnpackKeyFunc : InstructionFunc { const char* DirectoryUnpackKeyFunc::name = "DIRECTORY_UNPACK_KEY"; REGISTER_INSTRUCTION_FUNC(DirectoryUnpackKeyFunc); -//DIRECTORY_RANGE +// DIRECTORY_RANGE struct DirectoryRangeFunc : InstructionFunc { static const char* name; @@ -462,7 +473,7 @@ struct DirectoryRangeFunc : InstructionFunc { const char* DirectoryRangeFunc::name = "DIRECTORY_RANGE"; REGISTER_INSTRUCTION_FUNC(DirectoryRangeFunc); -//DIRECTORY_CONTAINS +// DIRECTORY_CONTAINS struct DirectoryContainsFunc : InstructionFunc { static const char* name; @@ -477,15 +488,15 @@ struct DirectoryContainsFunc : InstructionFunc { const char* DirectoryContainsFunc::name = "DIRECTORY_CONTAINS"; REGISTER_INSTRUCTION_FUNC(DirectoryContainsFunc); -//DIRECTORY_OPEN_SUBSPACE +// DIRECTORY_OPEN_SUBSPACE struct DirectoryOpenSubspaceFunc : InstructionFunc { static const char* name; ACTOR static Future call(Reference data, Reference instruction) { Tuple tuple = wait(popTuple(data)); - Subspace *subspace = data->directoryData.subspace(); + Subspace* subspace = data->directoryData.subspace(); logOp(format("open_subspace %s (at %s)", tupleToString(tuple).c_str(), subspace->key().printable().c_str())); - Subspace *child = new Subspace(subspace->subspace(tuple)); + Subspace* child = new Subspace(subspace->subspace(tuple)); data->directoryData.push(child); return Void(); @@ -494,7 +505,7 @@ struct DirectoryOpenSubspaceFunc : InstructionFunc { const char* DirectoryOpenSubspaceFunc::name = "DIRECTORY_OPEN_SUBSPACE"; REGISTER_INSTRUCTION_FUNC(DirectoryOpenSubspaceFunc); -//DIRECTORY_LOG_SUBSPACE +// DIRECTORY_LOG_SUBSPACE struct DirectoryLogSubspaceFunc : InstructionFunc { static const char* name; @@ -510,7 +521,7 @@ struct DirectoryLogSubspaceFunc : InstructionFunc { const char* DirectoryLogSubspaceFunc::name = "DIRECTORY_LOG_SUBSPACE"; REGISTER_INSTRUCTION_FUNC(DirectoryLogSubspaceFunc); -//DIRECTORY_LOG_DIRECTORY +// DIRECTORY_LOG_DIRECTORY struct DirectoryLogDirectoryFunc : InstructionFunc { static const char* name; @@ -520,22 +531,23 @@ struct DirectoryLogDirectoryFunc : InstructionFunc { state bool exists = wait(directory->exists(instruction->tr)); state Tuple childrenTuple; - if(exists) { + if (exists) { Standalone> children = wait(directory->list(instruction->tr)); - for(auto &c : children) { + for (auto& c : children) { childrenTuple.append(c, true); } - } + } Subspace logSubspace(Tuple().append(data->directoryData.directoryListIndex), prefix.getString(0)); Tuple pathTuple; - for(auto &p : directory->getPath()) { + for (auto& p : directory->getPath()) { pathTuple.append(p, true); } instruction->tr->set(logSubspace.pack(LiteralStringRef("path"), true), pathTuple.pack()); - instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true), Tuple().append(directory->getLayer()).pack()); + instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true), + Tuple().append(directory->getLayer()).pack()); instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append(exists ? 1 : 0).pack()); instruction->tr->set(logSubspace.pack(LiteralStringRef("children"), true), childrenTuple.pack()); @@ -545,13 +557,13 @@ struct DirectoryLogDirectoryFunc : InstructionFunc { const char* DirectoryLogDirectoryFunc::name = "DIRECTORY_LOG_DIRECTORY"; REGISTER_INSTRUCTION_FUNC(DirectoryLogDirectoryFunc); -//DIRECTORY_STRIP_PREFIX +// DIRECTORY_STRIP_PREFIX struct DirectoryStripPrefixFunc : InstructionFunc { static const char* name; ACTOR static Future call(Reference data, Reference instruction) { Tuple str = wait(data->stack.waitAndPop()); - Subspace *subspace = data->directoryData.subspace(); + Subspace* subspace = data->directoryData.subspace(); ASSERT(str.getString(0).startsWith(subspace->key())); data->stack.pushTuple(str.getString(0).substr(subspace->key().size())); return Void(); @@ -559,4 +571,3 @@ struct DirectoryStripPrefixFunc : InstructionFunc { }; const char* DirectoryStripPrefixFunc::name = "DIRECTORY_STRIP_PREFIX"; REGISTER_INSTRUCTION_FUNC(DirectoryStripPrefixFunc); - diff --git a/bindings/flow/tester/Tester.actor.cpp b/bindings/flow/tester/Tester.actor.cpp index 10ca75d404..b46e91c3bb 100644 --- a/bindings/flow/tester/Tester.actor.cpp +++ b/bindings/flow/tester/Tester.actor.cpp @@ -20,7 +20,7 @@ #include "Tester.actor.h" #include -#ifdef __linux__ +#ifdef __linux__ #include #endif @@ -42,33 +42,32 @@ std::map, Reference> trMap; // NOTE: This was taken from within fdb_c.cpp (where it is defined as a static within the get_range function). // If that changes, this will also have to be changed. const int ITERATION_PROGRESSION[] = { 256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000 }; -const int MAX_ITERATION = sizeof(ITERATION_PROGRESSION)/sizeof(int); +const int MAX_ITERATION = sizeof(ITERATION_PROGRESSION) / sizeof(int); -static Future runTest(Reference const& data, Reference const& db, +static Future runTest(Reference const& data, + Reference const& db, StringRef const& prefix); -THREAD_FUNC networkThread( void* api ) { +THREAD_FUNC networkThread(void* api) { // This is the fdb_flow network we're running on a thread ((API*)api)->runNetwork(); THREAD_RETURN; } -bool hasEnding(std::string const &fullString, std::string const &ending) -{ +bool hasEnding(std::string const& fullString, std::string const& ending) { if (fullString.length() >= ending.length()) { return (0 == fullString.compare(fullString.length() - ending.length(), ending.length(), ending)); - } - else { + } else { return false; } } -ACTOR Future> waitAndPop(FlowTesterStack *self, int count) { +ACTOR Future> waitAndPop(FlowTesterStack* self, int count) { state std::vector tuples; state std::vector items = self->pop(count); state int index; - for(index = 0; index < items.size(); ++index) { + for (index = 0; index < items.size(); ++index) { Standalone itemStr = wait(items[index].value); tuples.push_back(Tuple::unpack(itemStr)); } @@ -80,7 +79,7 @@ Future> FlowTesterStack::waitAndPop(int count) { return ::waitAndPop(this, count); } -ACTOR Future waitAndPop(FlowTesterStack *self) { +ACTOR Future waitAndPop(FlowTesterStack* self) { std::vector tuples = wait(waitAndPop(self, 1)); return tuples[0]; } @@ -91,41 +90,33 @@ Future FlowTesterStack::waitAndPop() { std::string tupleToString(Tuple const& tuple) { std::string str = "("; - for(int i = 0; i < tuple.size(); ++i) { + for (int i = 0; i < tuple.size(); ++i) { Tuple::ElementType type = tuple.getType(i); - if(type == Tuple::NULL_TYPE) { + if (type == Tuple::NULL_TYPE) { str += "NULL"; - } - else if(type == Tuple::BYTES || type == Tuple::UTF8) { - if(type == Tuple::UTF8) { + } else if (type == Tuple::BYTES || type == Tuple::UTF8) { + if (type == Tuple::UTF8) { str += "u"; } str += "\'" + tuple.getString(i).printable() + "\'"; - } - else if(type == Tuple::INT) { + } else if (type == Tuple::INT) { str += format("%ld", tuple.getInt(i)); - } - else if(type == Tuple::FLOAT) { + } else if (type == Tuple::FLOAT) { str += format("%f", tuple.getFloat(i)); - } - else if(type == Tuple::DOUBLE) { + } else if (type == Tuple::DOUBLE) { str += format("%f", tuple.getDouble(i)); - } - else if(type == Tuple::BOOL) { + } else if (type == Tuple::BOOL) { str += tuple.getBool(i) ? "true" : "false"; - } - else if(type == Tuple::UUID) { + } else if (type == Tuple::UUID) { Uuid u = tuple.getUuid(i); str += format("%016llx%016llx", *(uint64_t*)u.getData().begin(), *(uint64_t*)(u.getData().begin() + 8)); - } - else if(type == Tuple::NESTED) { + } else if (type == Tuple::NESTED) { str += tupleToString(tuple.getNested(i)); - } - else { + } else { ASSERT(false); } - if(i < tuple.size() - 1) { + if (i < tuple.size() - 1) { str += ", "; } } @@ -134,27 +125,41 @@ std::string tupleToString(Tuple const& tuple) { return str; } -ACTOR Future< Standalone > getRange(Reference tr, KeySelectorRef begin, KeySelectorRef end, int limits = 0, bool snapshot = false, bool reverse = false, FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { +ACTOR Future> getRange(Reference tr, + KeySelectorRef begin, + KeySelectorRef end, + int limits = 0, + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { state KeySelector ks_begin(begin); state KeySelector ks_end(end); state Standalone results; state int iteration = 1; - loop{ - // printf("=====DB: begin:%s, end:%s, limits:%d\n", printable(begin.key).c_str(), printable(end.key).c_str(), limits); + loop { + // printf("=====DB: begin:%s, end:%s, limits:%d\n", printable(begin.key).c_str(), printable(end.key).c_str(), + // limits); state FDBStandalone r; if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) { int effective_iteration = std::min(iteration, MAX_ITERATION); int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1]; - FDBStandalone rTemp = wait(tr->getRange(ks_begin, ks_end, GetRangeLimits(limits, bytes_limit), snapshot, reverse, (FDBStreamingMode)FDB_STREAMING_MODE_EXACT)); + FDBStandalone rTemp = wait(tr->getRange(ks_begin, + ks_end, + GetRangeLimits(limits, bytes_limit), + snapshot, + reverse, + (FDBStreamingMode)FDB_STREAMING_MODE_EXACT)); r = rTemp; } else { - FDBStandalone rTemp = wait(tr->getRange(ks_begin, ks_end, limits, snapshot, reverse, streamingMode)); + FDBStandalone rTemp = + wait(tr->getRange(ks_begin, ks_end, limits, snapshot, reverse, streamingMode)); r = rTemp; } iteration += 1; // printf("=====DB: count:%d\n", r.size()); - for (auto & s : r) { - // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); + for (auto& s : r) { + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); results.push_back_deep(results.arena(), s); if (reverse) @@ -169,34 +174,45 @@ ACTOR Future< Standalone > getRange(Reference tr, K return results; } - if(limits > 0) { + if (limits > 0) { limits -= r.size(); } } } -ACTOR Future< Standalone > getRange(Reference tr, KeyRange keys, int limits = 0, bool snapshot = false, bool reverse = false, FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { +ACTOR Future> getRange(Reference tr, + KeyRange keys, + int limits = 0, + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { state Key begin(keys.begin); state Key end(keys.end); state Standalone results; state int iteration = 1; - loop{ + loop { // printf("=====DB: begin:%s, limits:%d\n", printable(begin).c_str(), limits); KeyRange keyRange(KeyRangeRef(begin, end > begin ? end : begin)); state FDBStandalone r; if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) { int effective_iteration = std::min(iteration, MAX_ITERATION); int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1]; - FDBStandalone rTemp = wait(tr->getRange(keyRange, GetRangeLimits(limits, bytes_limit), snapshot, reverse, (FDBStreamingMode)FDB_STREAMING_MODE_EXACT)); + FDBStandalone rTemp = wait(tr->getRange(keyRange, + GetRangeLimits(limits, bytes_limit), + snapshot, + reverse, + (FDBStreamingMode)FDB_STREAMING_MODE_EXACT)); r = rTemp; } else { - FDBStandalone rTemp = wait(tr->getRange(keyRange, limits, snapshot, reverse, streamingMode)); + FDBStandalone rTemp = + wait(tr->getRange(keyRange, limits, snapshot, reverse, streamingMode)); r = rTemp; } iteration += 1; // printf("=====DB: count:%d\n", r.size()); - for (auto & s : r) { - // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); + for (auto& s : r) { + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); results.push_back_deep(results.arena(), s); if (reverse) @@ -211,7 +227,7 @@ ACTOR Future< Standalone > getRange(Reference tr, K return results; } - if(limits > 0) { + if (limits > 0) { limits -= r.size(); } } @@ -221,10 +237,13 @@ ACTOR static Future debugPrintRange(Reference tr, std::string if (!tr) return Void(); - Standalone results = wait(getRange(tr, KeyRange(KeyRangeRef(subspace + '\x00', subspace + '\xff')))); - printf("==================================================DB:%s:%s, count:%d\n", msg.c_str(), - StringRef(subspace).printable().c_str(), results.size()); - for (auto & s : results) { + Standalone results = + wait(getRange(tr, KeyRange(KeyRangeRef(subspace + '\x00', subspace + '\xff')))); + printf("==================================================DB:%s:%s, count:%d\n", + msg.c_str(), + StringRef(subspace).printable().c_str(), + results.size()); + for (auto& s : results) { printf("=====key:%s, value:%s\n", StringRef(s.key).printable().c_str(), StringRef(s.value).printable().c_str()); } @@ -252,7 +271,7 @@ ACTOR Future stackSub(FlowTesterStack* stack) { } ACTOR Future stackConcat(FlowTesterStack* stack) { - if(stack->data.size() < 2) + if (stack->data.size() < 2) return Void(); StackItem a = stack->data.back(); @@ -282,7 +301,7 @@ ACTOR Future stackSwap(FlowTesterStack* stack) { int64_t idx = stack->data.size() - 1; int64_t idx1 = idx - Tuple::unpack(sv).getInt(0); if (idx1 < idx) { - //printf("=============SWAP:%d,%d\n", idx, stack->data.size()); + // printf("=============SWAP:%d,%d\n", idx, stack->data.size()); StackItem item = stack->data[idx]; stack->data[idx] = stack->data[idx1]; stack->data[idx1] = item; @@ -295,7 +314,8 @@ ACTOR Future printFlowTesterStack(FlowTesterStack* stack) { state int idx; for (idx = stack->data.size() - 1; idx >= 0; --idx) { Standalone value = wait(stack->data[idx].value); - // printf("==========stack item:%d, index:%d, value:%s\n", idx, stack->data[idx].index, value.printable().c_str()); + // printf("==========stack item:%d, index:%d, value:%s\n", idx, stack->data[idx].index, + // value.printable().c_str()); } return Void(); } @@ -332,8 +352,8 @@ struct EmptyStackFunc : InstructionFunc { static const char* name; static Future call(Reference const& data, Reference const& instruction) { - //wait(printFlowTesterStack(&(data->stack))); - //wait(debugPrintRange(instruction->tr, "\x01test_results", "")); + // wait(printFlowTesterStack(&(data->stack))); + // wait(debugPrintRange(instruction->tr, "\x01test_results", "")); data->stack.clear(); return Void(); } @@ -357,7 +377,7 @@ struct PopFunc : InstructionFunc { ACTOR static Future call(Reference data, Reference instruction) { state std::vector items = data->stack.pop(); - for(StackItem item : items) { + for (StackItem item : items) { wait(success(item.value)); } return Void(); @@ -391,11 +411,13 @@ REGISTER_INSTRUCTION_FUNC(ConcatFunc); struct LogStackFunc : InstructionFunc { static const char* name; - ACTOR static Future logStack(Reference data, std::map entries, Standalone prefix) { + ACTOR static Future logStack(Reference data, + std::map entries, + Standalone prefix) { loop { state Reference tr = data->db->createTransaction(); try { - for(auto it : entries) { + for (auto it : entries) { Tuple tk; tk.append(it.first); tk.append((int64_t)it.second.index); @@ -406,8 +428,7 @@ struct LogStackFunc : InstructionFunc { wait(tr->commit()); return Void(); - } - catch(Error &e) { + } catch (Error& e) { wait(tr->onError(e)); } } @@ -422,11 +443,11 @@ struct LogStackFunc : InstructionFunc { state Standalone prefix = Tuple::unpack(s1).getString(0); state std::map entries; - while(data->stack.data.size() > 0) { + while (data->stack.data.size() > 0) { state std::vector it = data->stack.pop(); ASSERT(it.size() == 1); entries[data->stack.data.size()] = it.front(); - if(entries.size() == 100) { + if (entries.size() == 100) { wait(logStack(data, entries, prefix)); entries.clear(); } @@ -444,13 +465,12 @@ REGISTER_INSTRUCTION_FUNC(LogStackFunc); // FoundationDB Operations // ACTOR Future> waitForVoid(Future f) { - try{ + try { wait(f); Tuple t; t.append(LiteralStringRef("RESULT_NOT_PRESENT")); return t.pack(); - } - catch (Error& e){ + } catch (Error& e) { // printf("FDBError1:%d\n", e.code()); Tuple t; t.append(LiteralStringRef("ERROR")); @@ -463,13 +483,12 @@ ACTOR Future> waitForVoid(Future f) { } ACTOR Future> waitForValue(Future> f) { - try{ + try { FDBStandalone value = wait(f); Tuple t; t.append(value); return t.pack(); - } - catch (Error& e){ + } catch (Error& e) { // printf("FDBError2:%d\n", e.code()); Tuple t; t.append(LiteralStringRef("ERROR")); @@ -481,8 +500,8 @@ ACTOR Future> waitForValue(Future> f } } -ACTOR Future> waitForValue(Future< Optional> > f) { - try{ +ACTOR Future> waitForValue(Future>> f) { + try { Optional> value = wait(f); Standalone str; if (value.present()) @@ -493,8 +512,7 @@ ACTOR Future> waitForValue(Future< Optional> getKey(Future> f, Stan FDBStandalone key = wait(f); Tuple t; - if(key.startsWith(prefixFilter)) { + if (key.startsWith(prefixFilter)) { t.append(key); - } - else if(key < prefixFilter) { + } else if (key < prefixFilter) { t.append(prefixFilter); - } - else { + } else { t.append(strinc(prefixFilter)); } return t.pack(); - } - catch(Error& e){ + } catch (Error& e) { // printf("FDBError4:%d\n", e.code()); Tuple t; t.append(LiteralStringRef("ERROR")); @@ -554,7 +569,7 @@ struct UseTransactionFunc : InstructionFunc { Standalone name = wait(items[0].value); data->trName = name; - if(trMap.count(data->trName) == 0) { + if (trMap.count(data->trName) == 0) { trMap[data->trName] = data->db->createTransaction(); } return Void(); @@ -593,23 +608,22 @@ struct SetFunc : InstructionFunc { Standalone sk = wait(items[0].value); state Standalone key = Tuple::unpack(sk).getString(0); // if (instruction->isDatabase) - // printf("SetDatabase:%s, isDatabase:%d\n", printable(key).c_str(), instruction->isDatabase); + // printf("SetDatabase:%s, isDatabase:%d\n", printable(key).c_str(), instruction->isDatabase); Standalone sv = wait(items[1].value); Standalone value = Tuple::unpack(sv).getString(0); - //printf("SetDatabase:%s:%s:%s\n", printable(key).c_str(), printable(sv).c_str(), printable(value).c_str()); + // printf("SetDatabase:%s:%s:%s\n", printable(key).c_str(), printable(sv).c_str(), printable(value).c_str()); Reference instructionCopy = instruction; Standalone keyCopy = key; - Future mutation = executeMutation(instruction, [instructionCopy, keyCopy, value] () -> Future { + Future mutation = executeMutation(instruction, [instructionCopy, keyCopy, value]() -> Future { instructionCopy->tr->set(keyCopy, value); return Void(); }); if (instruction->isDatabase) { data->stack.push(waitForVoid(mutation)); - } - else { + } else { wait(mutation); } @@ -630,7 +644,7 @@ struct GetFunc : InstructionFunc { Standalone sk = wait(items[0].value); state Standalone key = Tuple::unpack(sk).getString(0); - Future< Optional> > fk = instruction->tr->get(StringRef(key), instruction->isSnapshot); + Future>> fk = instruction->tr->get(StringRef(key), instruction->isSnapshot); data->stack.push(waitForValue(holdWhile(instruction->tr, fk))); return Void(); @@ -659,8 +673,9 @@ struct GetKeyFunc : InstructionFunc { Standalone s4 = wait(items[3].value); Standalone prefix = Tuple::unpack(s4).getString(0); - //printf("===================GET_KEY:%s, %ld, %ld\n", printable(key).c_str(), or_equal, offset); - Future> fk = instruction->tr->getKey(KeySelector(KeySelectorRef(key, or_equal, offset)), instruction->isSnapshot); + // printf("===================GET_KEY:%s, %ld, %ld\n", printable(key).c_str(), or_equal, offset); + Future> fk = + instruction->tr->getKey(KeySelector(KeySelectorRef(key, or_equal, offset)), instruction->isSnapshot); data->stack.push(getKey(holdWhile(instruction->tr, fk), prefix)); return Void(); @@ -712,7 +727,7 @@ struct GetApproximateSizeFunc : InstructionFunc { ACTOR static Future call(Reference data, Reference instruction) { int64_t _ = wait(instruction->tr->getApproximateSize()); - (void) _; // disable unused variable warning + (void)_; // disable unused variable warning data->stack.pushTuple(LiteralStringRef("GOT_APPROXIMATE_SIZE")); return Void(); } @@ -775,15 +790,14 @@ struct ClearFunc : InstructionFunc { Reference instructionCopy = instruction; - Future mutation = executeMutation(instruction, [instructionCopy, key] () -> Future { + Future mutation = executeMutation(instruction, [instructionCopy, key]() -> Future { instructionCopy->tr->clear(key); return Void(); }); if (instruction->isDatabase) { data->stack.push(waitForVoid(mutation)); - } - else { + } else { wait(mutation); } @@ -840,16 +854,23 @@ struct GetRangeFunc : InstructionFunc { Standalone s5 = wait(items[4].value); FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s5).getInt(0); - // printf("================GetRange: %s, %s, %d, %d, %d, %d\n", printable(begin).c_str(), printable(end).c_str(), limit, reverse, mode, instruction->isSnapshot); + // printf("================GetRange: %s, %s, %d, %d, %d, %d\n", printable(begin).c_str(), + // printable(end).c_str(), limit, reverse, mode, instruction->isSnapshot); - Standalone results = wait(getRange(instruction->tr, KeyRange(KeyRangeRef(begin, end > begin ? end : begin)), limit, instruction->isSnapshot, reverse, mode)); + Standalone results = wait(getRange(instruction->tr, + KeyRange(KeyRangeRef(begin, end > begin ? end : begin)), + limit, + instruction->isSnapshot, + reverse, + mode)); Tuple t; - for (auto & s : results) { + for (auto& s : results) { t.append(s.key); t.append(s.value); - //printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); } - //printf("=====Results Count:%d, size:%d\n", results.size(), str.size()); + // printf("=====Results Count:%d, size:%d\n", results.size(), str.size()); data->stack.push(Tuple().append(t.pack()).pack()); return Void(); @@ -878,14 +899,21 @@ struct GetRangeStartsWithFunc : InstructionFunc { Standalone s4 = wait(items[3].value); FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s4).getInt(0); - //printf("================GetRangeStartsWithFunc: %s, %d, %d, %d, %d\n", printable(prefix).c_str(), limit, reverse, mode, isSnapshot); - Standalone results = wait(getRange(instruction->tr, KeyRange(KeyRangeRef(prefix, strinc(prefix))), limit, instruction->isSnapshot, reverse, mode)); + // printf("================GetRangeStartsWithFunc: %s, %d, %d, %d, %d\n", printable(prefix).c_str(), limit, + // reverse, mode, isSnapshot); + Standalone results = wait(getRange(instruction->tr, + KeyRange(KeyRangeRef(prefix, strinc(prefix))), + limit, + instruction->isSnapshot, + reverse, + mode)); Tuple t; - //printf("=====Results Count:%d\n", results.size()); - for (auto & s : results) { + // printf("=====Results Count:%d\n", results.size()); + for (auto& s : results) { t.append(s.key); t.append(s.value); - //printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); } data->stack.push(Tuple().append(t.pack()).pack()); @@ -912,15 +940,14 @@ struct ClearRangeFunc : InstructionFunc { Reference instructionCopy = instruction; Standalone beginCopy = begin; - Future mutation = executeMutation(instruction, [instructionCopy, beginCopy, end] () -> Future { + Future mutation = executeMutation(instruction, [instructionCopy, beginCopy, end]() -> Future { instructionCopy->tr->clear(KeyRangeRef(beginCopy, end)); return Void(); }); if (instruction->isDatabase) { data->stack.push(waitForVoid(mutation)); - } - else { + } else { wait(mutation); } @@ -943,15 +970,14 @@ struct ClearRangeStartWithFunc : InstructionFunc { Reference instructionCopy = instruction; - Future mutation = executeMutation(instruction, [instructionCopy, begin] () -> Future { + Future mutation = executeMutation(instruction, [instructionCopy, begin]() -> Future { instructionCopy->tr->clear(KeyRangeRef(begin, strinc(begin))); return Void(); }); if (instruction->isDatabase) { data->stack.push(waitForVoid(mutation)); - } - else { + } else { wait(mutation); } @@ -994,27 +1020,34 @@ struct GetRangeSelectorFunc : InstructionFunc { state int reverse = Tuple::unpack(s8).getInt(0); Standalone s9 = wait(items[8].value); - state FDBStreamingMode mode = (FDBStreamingMode) Tuple::unpack(s9).getInt(0); + state FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s9).getInt(0); Standalone s10 = wait(items[9].value); state Optional> prefix; Tuple t10 = Tuple::unpack(s10); - if(t10.getType(0) != Tuple::ElementType::NULL_TYPE) { + if (t10.getType(0) != Tuple::ElementType::NULL_TYPE) { prefix = t10.getString(0); } - //printf("================GetRangeSelectorFunc: %s, %d, %ld, %s, %d, %ld, %d, %d, %d, %d, %s\n", printable(begin).c_str(), begin_or_equal, begin_offset, - // printable(end).c_str(), end_or_equal, end_offset, + // printf("================GetRangeSelectorFunc: %s, %d, %ld, %s, %d, %ld, %d, %d, %d, %d, %s\n", + // printable(begin).c_str(), begin_or_equal, begin_offset, printable(end).c_str(), end_or_equal, end_offset, // limit, reverse, mode, instruction->isSnapshot, printable(prefix).c_str()); - Future> f = getRange(instruction->tr, KeySelectorRef(begin, begin_or_equal, begin_offset), KeySelectorRef(end, end_or_equal, end_offset), limit, instruction->isSnapshot, reverse, mode); + Future> f = getRange(instruction->tr, + KeySelectorRef(begin, begin_or_equal, begin_offset), + KeySelectorRef(end, end_or_equal, end_offset), + limit, + instruction->isSnapshot, + reverse, + mode); Standalone results = wait(holdWhile(instruction->tr, f)); Tuple t; - //printf("=====Results Count:%d\n", results.size()); - for (auto & s : results) { - if(!prefix.present() || s.key.startsWith(prefix.get())) { + // printf("=====Results Count:%d\n", results.size()); + for (auto& s : results) { + if (!prefix.present() || s.key.startsWith(prefix.get())) { t.append(s.key); t.append(s.value); - //printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); } } @@ -1047,40 +1080,30 @@ struct TuplePackFunc : InstructionFunc { for (; i < items1.size(); ++i) { Standalone str = wait(items1[i].value); Tuple itemTuple = Tuple::unpack(str); - if(deterministicRandom()->coinflip()) { + if (deterministicRandom()->coinflip()) { Tuple::ElementType type = itemTuple.getType(0); - if(type == Tuple::NULL_TYPE) { + if (type == Tuple::NULL_TYPE) { tuple.appendNull(); - } - else if(type == Tuple::INT) { + } else if (type == Tuple::INT) { tuple << itemTuple.getInt(0); - } - else if(type == Tuple::BYTES) { + } else if (type == Tuple::BYTES) { tuple.append(itemTuple.getString(0), false); - } - else if(type == Tuple::UTF8) { + } else if (type == Tuple::UTF8) { tuple.append(itemTuple.getString(0), true); - } - else if(type == Tuple::FLOAT) { + } else if (type == Tuple::FLOAT) { tuple << itemTuple.getFloat(0); - } - else if(type == Tuple::DOUBLE) { + } else if (type == Tuple::DOUBLE) { tuple << itemTuple.getDouble(0); - } - else if(type == Tuple::BOOL) { + } else if (type == Tuple::BOOL) { tuple << itemTuple.getBool(0); - } - else if(type == Tuple::UUID) { + } else if (type == Tuple::UUID) { tuple << itemTuple.getUuid(0); - } - else if(type == Tuple::NESTED) { + } else if (type == Tuple::NESTED) { tuple.appendNested(itemTuple.getNested(0)); - } - else { + } else { ASSERT(false); } - } - else { + } else { tuple << itemTuple; } } @@ -1106,7 +1129,7 @@ struct TupleUnpackFunc : InstructionFunc { for (int i = 0; i < t.size(); ++i) { Standalone str = t.subTuple(i, i + 1).pack(); - //printf("=====value:%s\n", printable(str).c_str()); + // printf("=====value:%s\n", printable(str).c_str()); data->stack.pushTuple(str); } return Void(); @@ -1136,40 +1159,30 @@ struct TupleRangeFunc : InstructionFunc { for (; i < items1.size(); ++i) { Standalone str = wait(items1[i].value); Tuple itemTuple = Tuple::unpack(str); - if(deterministicRandom()->coinflip()) { + if (deterministicRandom()->coinflip()) { Tuple::ElementType type = itemTuple.getType(0); - if(type == Tuple::NULL_TYPE) { + if (type == Tuple::NULL_TYPE) { tuple.appendNull(); - } - else if(type == Tuple::INT) { + } else if (type == Tuple::INT) { tuple << itemTuple.getInt(0); - } - else if(type == Tuple::BYTES) { + } else if (type == Tuple::BYTES) { tuple.append(itemTuple.getString(0), false); - } - else if(type == Tuple::UTF8) { + } else if (type == Tuple::UTF8) { tuple.append(itemTuple.getString(0), true); - } - else if(type == Tuple::FLOAT) { + } else if (type == Tuple::FLOAT) { tuple << itemTuple.getFloat(0); - } - else if(type == Tuple::DOUBLE) { + } else if (type == Tuple::DOUBLE) { tuple << itemTuple.getDouble(0); - } - else if(type == Tuple::BOOL) { + } else if (type == Tuple::BOOL) { tuple << itemTuple.getBool(0); - } - else if(type == Tuple::UUID) { + } else if (type == Tuple::UUID) { tuple << itemTuple.getUuid(0); - } - else if(type == Tuple::NESTED) { + } else if (type == Tuple::NESTED) { tuple.appendNested(itemTuple.getNested(0)); - } - else { + } else { ASSERT(false); } - } - else { + } else { tuple << itemTuple; } } @@ -1202,13 +1215,13 @@ struct TupleSortFunc : InstructionFunc { state std::vector tuples; state size_t i = 0; - for(; i < items1.size(); i++) { + for (; i < items1.size(); i++) { Standalone value = wait(items1[i].value); tuples.push_back(Tuple::unpack(value)); } std::sort(tuples.begin(), tuples.end()); - for(Tuple const& t : tuples) { + for (Tuple const& t : tuples) { data->stack.push(t.pack()); } @@ -1514,16 +1527,17 @@ struct AtomicOPFunc : InstructionFunc { Standalone keyCopy = key; Standalone valueCopy = value; - // printf("=========ATOMIC_OP:%s:%s:%s\n", printable(op).c_str(), printable(key).c_str(), printable(value).c_str()); - Future mutation = executeMutation(instruction, [instructionCopy, keyCopy, valueCopy, atomicOp] () -> Future { - instructionCopy->tr->atomicOp(keyCopy, valueCopy, atomicOp); - return Void(); - }); + // printf("=========ATOMIC_OP:%s:%s:%s\n", printable(op).c_str(), printable(key).c_str(), + // printable(value).c_str()); + Future mutation = + executeMutation(instruction, [instructionCopy, keyCopy, valueCopy, atomicOp]() -> Future { + instructionCopy->tr->atomicOp(keyCopy, valueCopy, atomicOp); + return Void(); + }); if (instruction->isDatabase) { data->stack.push(waitForVoid(mutation)); - } - else { + } else { wait(mutation); } @@ -1542,27 +1556,25 @@ struct UnitTestsFunc : InstructionFunc { ASSERT(!data->api->evaluatePredicate(FDBErrorPredicate::FDB_ERROR_PREDICATE_RETRYABLE, Error(10))); ASSERT(API::isAPIVersionSelected()); - state API *fdb = API::getInstance(); + state API* fdb = API::getInstance(); ASSERT(fdb->getAPIVersion() <= FDB_API_VERSION); try { API::selectAPIVersion(fdb->getAPIVersion() + 1); ASSERT(false); - } - catch(Error &e) { + } catch (Error& e) { ASSERT(e.code() == error_code_api_version_already_set); } try { API::selectAPIVersion(fdb->getAPIVersion() - 1); ASSERT(false); - } - catch(Error &e) { + } catch (Error& e) { ASSERT(e.code() == error_code_api_version_already_set); } API::selectAPIVersion(fdb->getAPIVersion()); const uint64_t locationCacheSize = 100001; const uint64_t maxWatches = 10001; - const uint64_t timeout = 60*1000; + const uint64_t timeout = 60 * 1000; const uint64_t noTimeout = 0; const uint64_t retryLimit = 50; const uint64_t noRetryLimit = -1; @@ -1570,19 +1582,30 @@ struct UnitTestsFunc : InstructionFunc { const uint64_t sizeLimit = 100000; const uint64_t maxFieldLength = 1000; - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_LOCATION_CACHE_SIZE, Optional(StringRef((const uint8_t*)&locationCacheSize, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MAX_WATCHES, Optional(StringRef((const uint8_t*)&maxWatches, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_DATACENTER_ID, Optional(LiteralStringRef("dc_id"))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MACHINE_ID, Optional(LiteralStringRef("machine_id"))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_LOCATION_CACHE_SIZE, + Optional(StringRef((const uint8_t*)&locationCacheSize, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MAX_WATCHES, + Optional(StringRef((const uint8_t*)&maxWatches, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_DATACENTER_ID, + Optional(LiteralStringRef("dc_id"))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MACHINE_ID, + Optional(LiteralStringRef("machine_id"))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_ENABLE); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_DISABLE); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, Optional(StringRef((const uint8_t*)&maxFieldLength, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, Optional(StringRef((const uint8_t*)&timeout, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, Optional(StringRef((const uint8_t*)&noTimeout, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_MAX_RETRY_DELAY, Optional(StringRef((const uint8_t*)&maxRetryDelay, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, Optional(StringRef((const uint8_t*)&sizeLimit, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, Optional(StringRef((const uint8_t*)&retryLimit, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, Optional(StringRef((const uint8_t*)&noRetryLimit, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, + Optional(StringRef((const uint8_t*)&maxFieldLength, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, + Optional(StringRef((const uint8_t*)&timeout, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, + Optional(StringRef((const uint8_t*)&noTimeout, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_MAX_RETRY_DELAY, + Optional(StringRef((const uint8_t*)&maxRetryDelay, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, + Optional(StringRef((const uint8_t*)&sizeLimit, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, + Optional(StringRef((const uint8_t*)&retryLimit, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, + Optional(StringRef((const uint8_t*)&noRetryLimit, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_CAUSAL_READ_RISKY); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_INCLUDE_PORT_IN_ADDRESS); @@ -1595,21 +1618,25 @@ struct UnitTestsFunc : InstructionFunc { tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_SYSTEM_KEYS); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, Optional(StringRef((const uint8_t*)&maxFieldLength, 8))); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TIMEOUT, Optional(StringRef((const uint8_t*)&timeout, 8))); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_RETRY_LIMIT, Optional(StringRef((const uint8_t*)&retryLimit, 8))); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_MAX_RETRY_DELAY, Optional(StringRef((const uint8_t*)&maxRetryDelay, 8))); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, + Optional(StringRef((const uint8_t*)&maxFieldLength, 8))); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TIMEOUT, + Optional(StringRef((const uint8_t*)&timeout, 8))); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_RETRY_LIMIT, + Optional(StringRef((const uint8_t*)&retryLimit, 8))); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_MAX_RETRY_DELAY, + Optional(StringRef((const uint8_t*)&maxRetryDelay, 8))); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_USED_DURING_COMMIT_PROTECTION_DISABLE); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_ENABLE, Optional(LiteralStringRef("my_transaction"))); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_ENABLE, + Optional(LiteralStringRef("my_transaction"))); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_LOCK_AWARE); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_LOCK_AWARE); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_INCLUDE_PORT_IN_ADDRESS); - Optional > _ = wait(tr->get(LiteralStringRef("\xff"))); + Optional> _ = wait(tr->get(LiteralStringRef("\xff"))); tr->cancel(); return Void(); - } }; const char* UnitTestsFunc::name = "UNIT_TESTS"; @@ -1626,8 +1653,7 @@ ACTOR static Future getInstructions(Reference data, String Standalone results = wait(getRange(tr, testSpec.range())); data->instructions = results; return Void(); - } - catch(Error &e) { + } catch (Error& e) { wait(tr->onError(e)); } } @@ -1645,8 +1671,8 @@ ACTOR static Future doInstructions(Reference data) { state bool isDirectory = op.startsWith(LiteralStringRef("DIRECTORY_")); try { - if(LOG_INSTRUCTIONS) { - if(op != LiteralStringRef("SWAP") && op != LiteralStringRef("PUSH")) { + if (LOG_INSTRUCTIONS) { + if (op != LiteralStringRef("SWAP") && op != LiteralStringRef("PUSH")) { printf("%zu. %s\n", idx, tupleToString(opTuple).c_str()); fflush(stdout); } @@ -1658,18 +1684,19 @@ ACTOR static Future doInstructions(Reference data) { op = op.substr(0, op.size() - 9); // printf("[==========]%ld/%ld:%s:%s: isDatabase:%d, isSnapshot:%d, stack count:%ld\n", - // idx, data->instructions.size(), StringRef(data->instructions[idx].key).printable().c_str(), StringRef(data->instructions[idx].value).printable().c_str(), - // isDatabase, isSnapshot, data->stack.data.size()); + // idx, data->instructions.size(), StringRef(data->instructions[idx].key).printable().c_str(), + // StringRef(data->instructions[idx].value).printable().c_str(), isDatabase, isSnapshot, + // data->stack.data.size()); - //wait(printFlowTesterStack(&(data->stack))); - //wait(debugPrintRange(instruction->tr, "\x01test_results", "")); + // wait(printFlowTesterStack(&(data->stack))); + // wait(debugPrintRange(instruction->tr, "\x01test_results", "")); - state Reference instruction = Reference(new InstructionData(isDatabase, isSnapshot, data->instructions[idx].value, Reference())); + state Reference instruction = Reference( + new InstructionData(isDatabase, isSnapshot, data->instructions[idx].value, Reference())); if (isDatabase) { state Reference tr = data->db->createTransaction(); instruction->tr = tr; - } - else { + } else { instruction->tr = trMap[data->trName]; } @@ -1678,20 +1705,18 @@ ACTOR static Future doInstructions(Reference data) { data->stack.index = idx; wait(InstructionFunc::call(op.toString(), data, instruction)); - } - catch (Error& e) { - if(LOG_ERRORS) { + } catch (Error& e) { + if (LOG_ERRORS) { printf("Error: %s (%d)\n", e.name(), e.code()); fflush(stdout); } - if(isDirectory) { - if(opsThatCreateDirectories.count(op.toString())) { + if (isDirectory) { + if (opsThatCreateDirectories.count(op.toString())) { data->directoryData.directoryList.push_back(DirectoryOrSubspace()); } data->stack.pushTuple(LiteralStringRef("DIRECTORY_ERROR")); - } - else { + } else { data->stack.pushError(e.code()); } } @@ -1707,8 +1732,7 @@ ACTOR static Future runTest(Reference data, ReferencesubThreads)); - } - catch (Error& e) { + } catch (Error& e) { TraceEvent(SevError, "FlowTesterDataRunError").error(e); } @@ -1755,15 +1779,14 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi try { API::getInstance(); ASSERT(false); - } - catch(Error& e) { + } catch (Error& e) { ASSERT(e.code() == error_code_api_version_unset); } - API *fdb = API::selectAPIVersion(apiVersion); + API* fdb = API::selectAPIVersion(apiVersion); ASSERT(API::isAPIVersionSelected()); ASSERT(fdb->getAPIVersion() == apiVersion); - //fdb->setNetworkOption(FDBNetworkOption::FDB_NET_OPTION_TRACE_ENABLE); + // fdb->setNetworkOption(FDBNetworkOption::FDB_NET_OPTION_TRACE_ENABLE); // We have to start the fdb_flow network and thread separately! fdb->setupNetwork(); @@ -1778,23 +1801,21 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi // Stopping the network returns from g_network->run() and allows // the program to terminate g_network->stop(); - } - catch(Error &e) { + } catch (Error& e) { TraceEvent("ErrorRunningTest").error(e); - if(LOG_ERRORS) { + if (LOG_ERRORS) { printf("Flow tester encountered error: %s\n", e.name()); fflush(stdout); } flushAndExit(1); } - } ACTOR void _test_versionstamp() { try { g_network = newNet2(TLSConfig()); - API *fdb = FDB::API::selectAPIVersion(620); + API* fdb = FDB::API::selectAPIVersion(620); fdb->setupNetwork(); startThread(networkThread, fdb); @@ -1804,7 +1825,9 @@ ACTOR void _test_versionstamp() { state Future> ftrVersion = tr->getVersionstamp(); - tr->atomicOp(LiteralStringRef("foo"), LiteralStringRef("blahblahbl\x00\x00\x00\x00"), FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE); + tr->atomicOp(LiteralStringRef("foo"), + LiteralStringRef("blahblahbl\x00\x00\x00\x00"), + FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE); wait(tr->commit()); // should use retry loop @@ -1819,8 +1842,7 @@ ACTOR void _test_versionstamp() { fprintf(stderr, "%s\n", trVersion.printable().c_str()); g_network->stop(); - } - catch (Error &e) { + } catch (Error& e) { TraceEvent("ErrorRunningTest").error(e); if (LOG_ERRORS) { printf("Flow tester encountered error: %s\n", e.name()); @@ -1830,7 +1852,7 @@ ACTOR void _test_versionstamp() { } } -int main( int argc, char** argv ) { +int main(int argc, char** argv) { try { platformInit(); registerCrashHandler(); @@ -1861,13 +1883,11 @@ int main( int argc, char** argv ) { g_network->run(); flushAndExit(FDB_EXIT_SUCCESS); - } - catch (Error& e) { + } catch (Error& e) { fprintf(stderr, "Error: %s\n", e.name()); TraceEvent(SevError, "MainError").error(e); flushAndExit(FDB_EXIT_MAIN_ERROR); - } - catch (std::exception& e) { + } catch (std::exception& e) { fprintf(stderr, "std::exception: %s\n", e.what()); TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); flushAndExit(FDB_EXIT_MAIN_EXCEPTION); diff --git a/bindings/flow/tester/Tester.actor.h b/bindings/flow/tester/Tester.actor.h index e33fae3f91..e450c1a6a4 100644 --- a/bindings/flow/tester/Tester.actor.h +++ b/bindings/flow/tester/Tester.actor.h @@ -18,12 +18,13 @@ * limitations under the License. */ -// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version. +// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source +// version. #if defined(NO_INTELLISENSE) && !defined(FDB_FLOW_TESTER_TESTER_ACTOR_G_H) - #define FDB_FLOW_TESTER_TESTER_ACTOR_G_H - #include "Tester.actor.g.h" +#define FDB_FLOW_TESTER_TESTER_ACTOR_G_H +#include "Tester.actor.g.h" #elif !defined(FDB_FLOW_TESTER_TESTER_ACTOR_H) - #define FDB_FLOW_TESTER_TESTER_ACTOR_H +#define FDB_FLOW_TESTER_TESTER_ACTOR_H #pragma once @@ -32,7 +33,7 @@ #include "bindings/flow/IDirectory.h" #include "bindings/flow/Subspace.h" #include "bindings/flow/DirectoryLayer.h" -#include "flow/actorcompiler.h" // This must be the last #include. +#include "flow/actorcompiler.h" // This must be the last #include. #define LOG_ALL 0 #define LOG_INSTRUCTIONS LOG_ALL || 0 @@ -54,19 +55,13 @@ struct FlowTesterStack { uint32_t index; std::vector data; - void push(Future> value) { - data.push_back(StackItem(index, value)); - } - - void push(Standalone value) { - push(Future>(value)); - } + void push(Future> value) { data.push_back(StackItem(index, value)); } - void push(const StackItem& item) { - data.push_back(item); - } + void push(Standalone value) { push(Future>(value)); } - void pushTuple(StringRef value, bool utf8=false) { + void push(const StackItem& item) { data.push_back(item); } + + void pushTuple(StringRef value, bool utf8 = false) { FDB::Tuple t; t.append(value, utf8); data.push_back(StackItem(index, t.pack())); @@ -86,10 +81,10 @@ struct FlowTesterStack { items.push_back(data.back()); data.pop_back(); count--; - } + } return items; } - + Future> waitAndPop(int count); Future waitAndPop(); @@ -99,33 +94,31 @@ struct FlowTesterStack { data.push_back(data.back()); } - void clear() { - data.clear(); - } + void clear() { data.clear(); } }; struct InstructionData : public ReferenceCounted { bool isDatabase; - bool isSnapshot; + bool isSnapshot; StringRef instruction; Reference tr; InstructionData(bool _isDatabase, bool _isSnapshot, StringRef _instruction, Reference _tr) - : isDatabase(_isDatabase) - , isSnapshot(_isSnapshot) - , instruction(_instruction) - , tr(_tr) {} + : isDatabase(_isDatabase), isSnapshot(_isSnapshot), instruction(_instruction), tr(_tr) {} }; struct FlowTesterData; -struct InstructionFunc : IDispatched(Reference data, Reference instruction)>> { +struct InstructionFunc + : IDispatched(Reference data, Reference instruction)>> { static Future call(std::string op, Reference data, Reference instruction) { ASSERT(data); ASSERT(instruction); auto it = dispatches().find(op); - if(it == dispatches().end()) { + if (it == dispatches().end()) { fprintf(stderr, "Unrecognized instruction: %s\n", op.c_str()); ASSERT(false); } @@ -141,24 +134,20 @@ struct DirectoryOrSubspace { DirectoryOrSubspace() {} DirectoryOrSubspace(Reference directory) : directory(directory) {} - DirectoryOrSubspace(FDB::Subspace *subspace) : subspace(subspace) {} - DirectoryOrSubspace(Reference dirSubspace) : directory(dirSubspace), subspace(dirSubspace.getPtr()) {} + DirectoryOrSubspace(FDB::Subspace* subspace) : subspace(subspace) {} + DirectoryOrSubspace(Reference dirSubspace) + : directory(dirSubspace), subspace(dirSubspace.getPtr()) {} - bool valid() { - return directory.present() || subspace.present(); - } + bool valid() { return directory.present() || subspace.present(); } std::string typeString() { - if(directory.present() && subspace.present()) { + if (directory.present() && subspace.present()) { return "DirectorySubspace"; - } - else if(directory.present()) { - return "IDirectory"; - } - else if(subspace.present()) { + } else if (directory.present()) { + return "IDirectory"; + } else if (subspace.present()) { return "Subspace"; - } - else { + } else { return "InvalidDirectory"; } } @@ -169,10 +158,10 @@ struct DirectoryTesterData { int directoryListIndex; int directoryErrorIndex; - Reference directory() { + Reference directory() { ASSERT(directoryListIndex < directoryList.size()); ASSERT(directoryList[directoryListIndex].directory.present()); - return directoryList[directoryListIndex].directory.get(); + return directoryList[directoryListIndex].directory.get(); } FDB::Subspace* subspace() { @@ -188,8 +177,8 @@ struct DirectoryTesterData { template void push(T item) { directoryList.push_back(DirectoryOrSubspace(item)); - if(LOG_DIRS) { - printf("Pushed %s at %lu\n", directoryList.back().typeString().c_str(), directoryList.size()-1); + if (LOG_DIRS) { + printf("Pushed %s at %lu\n", directoryList.back().typeString().c_str(), directoryList.size() - 1); fflush(stdout); } } @@ -198,7 +187,7 @@ struct DirectoryTesterData { }; struct FlowTesterData : public ReferenceCounted { - FDB::API *api; + FDB::API* api; Reference db; Standalone instructions; Standalone trName; @@ -209,12 +198,11 @@ struct FlowTesterData : public ReferenceCounted { std::vector> subThreads; Future processInstruction(Reference instruction) { - return InstructionFunc::call(instruction->instruction.toString(), Reference::addRef(this), instruction); + return InstructionFunc::call( + instruction->instruction.toString(), Reference::addRef(this), instruction); } - FlowTesterData(FDB::API *api) { - this->api = api; - } + FlowTesterData(FDB::API* api) { this->api = api; } }; std::string tupleToString(FDB::Tuple const& tuple); @@ -224,16 +212,14 @@ Future()().getValue())> executeMutation(Reference()().getValue()) result = wait(func()); - if(instruction->isDatabase) { + if (instruction->isDatabase) { wait(instruction->tr->commit()); } return result; - } - catch(Error &e) { - if(instruction->isDatabase) { + } catch (Error& e) { + if (instruction->isDatabase) { wait(instruction->tr->onError(e)); - } - else { + } else { throw; } } diff --git a/bindings/java/JavaWorkload.cpp b/bindings/java/JavaWorkload.cpp index 6dff50042f..cc246fcbda 100644 --- a/bindings/java/JavaWorkload.cpp +++ b/bindings/java/JavaWorkload.cpp @@ -280,7 +280,8 @@ struct JVM { w.name = name; w.signature = sig; w.fnPtr = std::get<2>(t); - log->trace(info, "PreparedNativeMethod", + log->trace(info, + "PreparedNativeMethod", { { "Name", w.name }, { "Signature", w.signature }, { "Ptr", std::to_string(reinterpret_cast(w.fnPtr)) } }); @@ -362,7 +363,8 @@ struct JVM { { "getOption", "(JLjava/lang/String;Z)Z", reinterpret_cast(&getOptionBool) }, { "getOption", "(JLjava/lang/String;J)J", reinterpret_cast(&getOptionLong) }, { "getOption", "(JLjava/lang/String;D)D", reinterpret_cast(&getOptionDouble) }, - { "getOption", "(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;", + { "getOption", + "(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;", reinterpret_cast(&getOptionString) }, { "getClientID", "(J)I", reinterpret_cast(&getClientID) }, { "getClientCount", "(J)I", reinterpret_cast(&getClientCount) }, @@ -389,7 +391,8 @@ struct JVM { auto impl = env->GetLongField(res, field); checkException(); if (impl != jContext) { - log->trace(error, "ContextNotCorrect", + log->trace(error, + "ContextNotCorrect", { { "Expected", std::to_string(jContext) }, { "Impl", std::to_string(impl) } }); std::terminate(); } @@ -469,14 +472,16 @@ struct JVM { } jobject createDatabase(jobject workload, FDBDatabase* db) { - auto executor = - env->CallObjectMethod(workload, getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"), - "getExecutor", "()Ljava/util/concurrent/Executor;")); + auto executor = env->CallObjectMethod(workload, + getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"), + "getExecutor", + "()Ljava/util/concurrent/Executor;")); auto databaseClass = getClass("com/apple/foundationdb/FDBDatabase"); jlong databasePtr = reinterpret_cast(db); - jobject javaDatabase = - env->NewObject(databaseClass, getMethod(databaseClass, "", "(JLjava/util/concurrent/Executor;)V"), - databasePtr, executor); + jobject javaDatabase = env->NewObject(databaseClass, + getMethod(databaseClass, "", "(JLjava/util/concurrent/Executor;)V"), + databasePtr, + executor); env->DeleteLocalRef(executor); return javaDatabase; } @@ -489,9 +494,10 @@ struct JVM { jPromise = createPromise(std::move(promise)); env->CallVoidMethod( workload, - getMethod(clazz, method, - "(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"), - jdb, jPromise); + getMethod( + clazz, method, "(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"), + jdb, + jPromise); env->DeleteLocalRef(jdb); env->DeleteLocalRef(jPromise); jPromise = nullptr; @@ -513,7 +519,7 @@ struct JavaWorkload : FDBWorkload { bool failed = false; jobject workload = nullptr; JavaWorkload(const std::shared_ptr& jvm, FDBLogger& log, const std::string& name) - : jvm(jvm), log(log), name(name) { + : jvm(jvm), log(log), name(name) { boost::replace_all(this->name, ".", "/"); } ~JavaWorkload() { @@ -586,9 +592,7 @@ struct JavaWorkload : FDBWorkload { log.trace(error, "CheckFailedWithJNIError", { { "Error", e.toString() }, { "Location", e.location() } }); } } - void getMetrics(std::vector& out) const override { - jvm->getMetrics(workload, name, out); - } + void getMetrics(std::vector& out) const override { jvm->getMetrics(workload, name, out); } }; struct JavaWorkloadFactory : FDBWorkloadFactory { diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp index 5a49987a85..e16db38841 100644 --- a/bindings/java/fdbJNI.cpp +++ b/bindings/java/fdbJNI.cpp @@ -25,35 +25,36 @@ #include -#define JNI_NULL nullptr +#define JNI_NULL nullptr #if defined(__GNUG__) #undef JNIEXPORT -#define JNIEXPORT __attribute__ ((visibility ("default"))) +#define JNIEXPORT __attribute__((visibility("default"))) #endif -static JavaVM* g_jvm = nullptr; -static thread_local JNIEnv* g_thread_jenv = nullptr; // Defined for the network thread once it is running, and for any thread that has called registerCallback +static JavaVM* g_jvm = nullptr; +static thread_local JNIEnv* g_thread_jenv = + nullptr; // Defined for the network thread once it is running, and for any thread that has called registerCallback static thread_local jmethodID g_IFutureCallback_call_methodID = JNI_NULL; static thread_local bool is_external = false; -void detachIfExternalThread(void *ignore) { - if(is_external && g_thread_jenv != nullptr) { +void detachIfExternalThread(void* ignore) { + if (is_external && g_thread_jenv != nullptr) { g_thread_jenv = nullptr; g_IFutureCallback_call_methodID = JNI_NULL; g_jvm->DetachCurrentThread(); } } -void throwOutOfMem(JNIEnv *jenv) { - const char *className = "java/lang/OutOfMemoryError"; - jclass illegalArgClass = jenv->FindClass( className ); +void throwOutOfMem(JNIEnv* jenv) { + const char* className = "java/lang/OutOfMemoryError"; + jclass illegalArgClass = jenv->FindClass(className); - if(jenv->ExceptionOccurred()) + if (jenv->ExceptionOccurred()) return; - if( jenv->ThrowNew( illegalArgClass, nullptr ) != 0 ) { - if( !jenv->ExceptionOccurred() ) { + if (jenv->ThrowNew(illegalArgClass, nullptr) != 0) { + if (!jenv->ExceptionOccurred()) { jenv->FatalError("Could not throw OutOfMemoryError"); } else { // This means that an exception is pending. We do not know what it is, but are sure @@ -62,45 +63,45 @@ void throwOutOfMem(JNIEnv *jenv) { } } -static jthrowable getThrowable(JNIEnv *jenv, fdb_error_t e, const char* msg = nullptr) { +static jthrowable getThrowable(JNIEnv* jenv, fdb_error_t e, const char* msg = nullptr) { jclass excepClass = jenv->FindClass("com/apple/foundationdb/FDBException"); - if(jenv->ExceptionOccurred()) + if (jenv->ExceptionOccurred()) return JNI_NULL; jmethodID excepCtor = jenv->GetMethodID(excepClass, "", "(Ljava/lang/String;I)V"); - if(jenv->ExceptionOccurred()) + if (jenv->ExceptionOccurred()) return JNI_NULL; - const char *fdb_message = msg ? msg : fdb_get_error(e); + const char* fdb_message = msg ? msg : fdb_get_error(e); jstring m = jenv->NewStringUTF(fdb_message); - if(jenv->ExceptionOccurred()) + if (jenv->ExceptionOccurred()) return JNI_NULL; jthrowable t = (jthrowable)jenv->NewObject(excepClass, excepCtor, m, e); - if(jenv->ExceptionOccurred()) + if (jenv->ExceptionOccurred()) return JNI_NULL; return t; } -void throwNamedException(JNIEnv *jenv, const char *class_full_name, const char* message ) { - jclass exceptionClass = jenv->FindClass( class_full_name ); - if(jenv->ExceptionOccurred()) +void throwNamedException(JNIEnv* jenv, const char* class_full_name, const char* message) { + jclass exceptionClass = jenv->FindClass(class_full_name); + if (jenv->ExceptionOccurred()) return; - if( jenv->ThrowNew( exceptionClass, message ) != 0 ) { - if(jenv->ExceptionOccurred()) + if (jenv->ThrowNew(exceptionClass, message) != 0) { + if (jenv->ExceptionOccurred()) return; jenv->FatalError("FDB: Error throwing exception"); } } -void throwRuntimeEx(JNIEnv *jenv, const char* message) { - throwNamedException( jenv, "java/lang/RuntimeException", message ); +void throwRuntimeEx(JNIEnv* jenv, const char* message) { + throwNamedException(jenv, "java/lang/RuntimeException", message); } -void throwParamNotNull(JNIEnv *jenv) { - throwNamedException( jenv, "java/lang/IllegalArgumentException", "Argument cannot be null" ); +void throwParamNotNull(JNIEnv* jenv) { + throwNamedException(jenv, "java/lang/IllegalArgumentException", "Argument cannot be null"); } #ifdef __cplusplus @@ -109,25 +110,25 @@ extern "C" { // If the methods are not found, exceptions are thrown and this will return false. // Returns TRUE on success, false otherwise. -static bool findCallbackMethods(JNIEnv *jenv) { +static bool findCallbackMethods(JNIEnv* jenv) { jclass cls = jenv->FindClass("java/lang/Runnable"); - if(jenv->ExceptionOccurred()) + if (jenv->ExceptionOccurred()) return false; g_IFutureCallback_call_methodID = jenv->GetMethodID(cls, "run", "()V"); - if(jenv->ExceptionOccurred()) + if (jenv->ExceptionOccurred()) return false; return true; } -static void callCallback( FDBFuture* f, void* data ) { +static void callCallback(FDBFuture* f, void* data) { if (g_thread_jenv == nullptr) { // We are on an external thread and must attach to the JVM. // The shutdown hook will later detach this thread. is_external = true; - if( g_jvm != nullptr && g_jvm->AttachCurrentThreadAsDaemon((void **) &g_thread_jenv, nullptr) == JNI_OK ) { - if( !findCallbackMethods( g_thread_jenv ) ) { + if (g_jvm != nullptr && g_jvm->AttachCurrentThreadAsDaemon((void**)&g_thread_jenv, nullptr) == JNI_OK) { + if (!findCallbackMethods(g_thread_jenv)) { g_thread_jenv->FatalError("FDB: Could not find callback method.\n"); } } else { @@ -138,117 +139,128 @@ static void callCallback( FDBFuture* f, void* data ) { } jobject callback = (jobject)data; - g_thread_jenv->CallVoidMethod( callback, g_IFutureCallback_call_methodID ); + g_thread_jenv->CallVoidMethod(callback, g_IFutureCallback_call_methodID); g_thread_jenv->DeleteGlobalRef(callback); } // Attempts to throw 't', attempts to shut down the JVM if this fails. -void safeThrow( JNIEnv *jenv, jthrowable t ) { - if( jenv->Throw( t ) != 0 ) { +void safeThrow(JNIEnv* jenv, jthrowable t) { + if (jenv->Throw(t) != 0) { jenv->FatalError("FDB: Unable to throw exception"); } } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1registerCallback(JNIEnv *jenv, jobject cls, jlong future, jobject callback) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1registerCallback(JNIEnv* jenv, + jobject cls, + jlong future, + jobject callback) { // SOMEDAY: Do this on module load instead. Can we cache method ids across threads? - if( !g_IFutureCallback_call_methodID ) { - if( !findCallbackMethods( jenv ) ) { + if (!g_IFutureCallback_call_methodID) { + if (!findCallbackMethods(jenv)) { return; } } - if( !future || !callback ) { + if (!future || !callback) { throwParamNotNull(jenv); return; } - FDBFuture *f = (FDBFuture *)future; + FDBFuture* f = (FDBFuture*)future; // This is documented as not throwing, but simply returning null on OOM. // As belt and suspenders, we will check for pending exceptions and then, // if there are none and the result is null, we'll throw our own OOM. - callback = jenv->NewGlobalRef( callback ); - if( !callback ) { - if( !jenv->ExceptionOccurred() ) + callback = jenv->NewGlobalRef(callback); + if (!callback) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); return; } // Here we cache a thread-local reference to jenv g_thread_jenv = jenv; - fdb_error_t err = fdb_future_set_callback( f, &callCallback, callback ); - if( err ) { - jenv->DeleteGlobalRef( callback ); - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_future_set_callback(f, &callCallback, callback); + if (err) { + jenv->DeleteGlobalRef(callback); + safeThrow(jenv, getThrowable(jenv, err)); } } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1blockUntilReady(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1blockUntilReady(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return; } - FDBFuture *sav = (FDBFuture *)future; + FDBFuture* sav = (FDBFuture*)future; - fdb_error_t err = fdb_future_block_until_ready( sav ); - if( err ) - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_future_block_until_ready(sav); + if (err) + safeThrow(jenv, getThrowable(jenv, err)); } -JNIEXPORT jthrowable JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1getError(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT jthrowable JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1getError(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return JNI_NULL; } - FDBFuture *sav = (FDBFuture *)future; - fdb_error_t err = fdb_future_get_error( sav ); - if( err ) - return getThrowable( jenv, err ); + FDBFuture* sav = (FDBFuture*)future; + fdb_error_t err = fdb_future_get_error(sav); + if (err) + return getThrowable(jenv, err); else return JNI_NULL; } -JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1isReady(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1isReady(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return JNI_FALSE; } - FDBFuture *var = (FDBFuture *)future; + FDBFuture* var = (FDBFuture*)future; return (jboolean)fdb_future_is_ready(var); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1dispose(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1dispose(JNIEnv* jenv, jobject, jlong future) { + if (!future) { throwParamNotNull(jenv); return; } - FDBFuture *var = (FDBFuture *)future; + FDBFuture* var = (FDBFuture*)future; fdb_future_destroy(var); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1cancel(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1cancel(JNIEnv* jenv, jobject, jlong future) { + if (!future) { throwParamNotNull(jenv); return; } - FDBFuture *var = (FDBFuture *)future; + FDBFuture* var = (FDBFuture*)future; fdb_future_cancel(var); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1releaseMemory(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1releaseMemory(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return; } - FDBFuture *var = (FDBFuture *)future; + FDBFuture* var = (FDBFuture*)future; fdb_future_release_memory(var); } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FutureInt64_FutureInt64_1get(JNIEnv *jenv, jobject, jlong future) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FutureInt64_FutureInt64_1get(JNIEnv* jenv, jobject, jlong future) { if (!future) { throwParamNotNull(jenv); return 0; } - FDBFuture *f = (FDBFuture *)future; + FDBFuture* f = (FDBFuture*)future; int64_t value = 0; fdb_error_t err = fdb_future_get_int64(f, &value); @@ -260,293 +272,312 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FutureInt64_FutureInt64_1get return (jlong)value; } -JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureStrings_1get(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureStrings_1get(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return JNI_NULL; } - FDBFuture *f = (FDBFuture *)future; + FDBFuture* f = (FDBFuture*)future; - const char **strings; + const char** strings; int count; - fdb_error_t err = fdb_future_get_string_array( f, &strings, &count ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_future_get_string_array(f, &strings, &count); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return JNI_NULL; } jclass str_clazz = jenv->FindClass("java/lang/String"); - if( jenv->ExceptionOccurred() ) + if (jenv->ExceptionOccurred()) return JNI_NULL; jobjectArray arr = jenv->NewObjectArray(count, str_clazz, JNI_NULL); - if( !arr ) { - if( !jenv->ExceptionOccurred() ) + if (!arr) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); return JNI_NULL; } - for(int i = 0; i < count; i++) { - jstring str = jenv->NewStringUTF( strings[i] ); - if( !str ) { - if( !jenv->ExceptionOccurred() ) + for (int i = 0; i < count; i++) { + jstring str = jenv->NewStringUTF(strings[i]); + if (!str) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); return JNI_NULL; } - jenv->SetObjectArrayElement( arr, i, str ); - if( jenv->ExceptionOccurred() ) + jenv->SetObjectArrayElement(arr, i, str); + if (jenv->ExceptionOccurred()) return JNI_NULL; } return arr; } -JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1getSummary(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1getSummary(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return JNI_NULL; } jclass resultCls = jenv->FindClass("com/apple/foundationdb/RangeResultSummary"); - if( jenv->ExceptionOccurred() ) + if (jenv->ExceptionOccurred()) return JNI_NULL; jmethodID resultCtorId = jenv->GetMethodID(resultCls, "", "([BIZ)V"); - if( jenv->ExceptionOccurred() ) + if (jenv->ExceptionOccurred()) return JNI_NULL; - - FDBFuture *f = (FDBFuture *)future; - const FDBKeyValue *kvs; + FDBFuture* f = (FDBFuture*)future; + + const FDBKeyValue* kvs; int count; fdb_bool_t more; - fdb_error_t err = fdb_future_get_keyvalue_array( f, &kvs, &count, &more ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_future_get_keyvalue_array(f, &kvs, &count, &more); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return JNI_NULL; } jbyteArray lastKey = JNI_NULL; - if(count) { + if (count) { lastKey = jenv->NewByteArray(kvs[count - 1].key_length); - if( !lastKey ) { - if( !jenv->ExceptionOccurred() ) + if (!lastKey) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); return JNI_NULL; } - jenv->SetByteArrayRegion(lastKey, 0, kvs[count - 1].key_length, (jbyte *)kvs[count - 1].key); + jenv->SetByteArrayRegion(lastKey, 0, kvs[count - 1].key_length, (jbyte*)kvs[count - 1].key); } jobject result = jenv->NewObject(resultCls, resultCtorId, lastKey, count, (jboolean)more); - if( jenv->ExceptionOccurred() ) + if (jenv->ExceptionOccurred()) return JNI_NULL; return result; } // SOMEDAY: explore doing this more efficiently with Direct ByteBuffers -JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1get(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1get(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return JNI_NULL; } jclass resultCls = jenv->FindClass("com/apple/foundationdb/RangeResult"); jmethodID resultCtorId = jenv->GetMethodID(resultCls, "", "([B[IZ)V"); - - FDBFuture *f = (FDBFuture *)future; - const FDBKeyValue *kvs; + FDBFuture* f = (FDBFuture*)future; + + const FDBKeyValue* kvs; int count; fdb_bool_t more; - fdb_error_t err = fdb_future_get_keyvalue_array( f, &kvs, &count, &more ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_future_get_keyvalue_array(f, &kvs, &count, &more); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return JNI_NULL; } int totalKeyValueSize = 0; - for(int i = 0; i < count; i++) { + for (int i = 0; i < count; i++) { totalKeyValueSize += kvs[i].key_length + kvs[i].value_length; } jbyteArray keyValueArray = jenv->NewByteArray(totalKeyValueSize); - if( !keyValueArray ) { - if( !jenv->ExceptionOccurred() ) + if (!keyValueArray) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); return JNI_NULL; } - uint8_t *keyvalues_barr = (uint8_t *)jenv->GetByteArrayElements(keyValueArray, JNI_NULL); + uint8_t* keyvalues_barr = (uint8_t*)jenv->GetByteArrayElements(keyValueArray, JNI_NULL); if (!keyvalues_barr) { - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + throwRuntimeEx(jenv, "Error getting handle to native resources"); return JNI_NULL; } jintArray lengthArray = jenv->NewIntArray(count * 2); - if( !lengthArray ) { - if( !jenv->ExceptionOccurred() ) + if (!lengthArray) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); - jenv->ReleaseByteArrayElements(keyValueArray, (jbyte *)keyvalues_barr, 0); + jenv->ReleaseByteArrayElements(keyValueArray, (jbyte*)keyvalues_barr, 0); return JNI_NULL; } - jint *length_barr = jenv->GetIntArrayElements(lengthArray, JNI_NULL); - if( !length_barr ) { - if( !jenv->ExceptionOccurred() ) + jint* length_barr = jenv->GetIntArrayElements(lengthArray, JNI_NULL); + if (!length_barr) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); - jenv->ReleaseByteArrayElements(keyValueArray, (jbyte *)keyvalues_barr, 0); + jenv->ReleaseByteArrayElements(keyValueArray, (jbyte*)keyvalues_barr, 0); return JNI_NULL; } int offset = 0; - for(int i = 0; i < count; i++) { + for (int i = 0; i < count; i++) { memcpy(keyvalues_barr + offset, kvs[i].key, kvs[i].key_length); - length_barr[ i * 2 ] = kvs[i].key_length; + length_barr[i * 2] = kvs[i].key_length; offset += kvs[i].key_length; memcpy(keyvalues_barr + offset, kvs[i].value, kvs[i].value_length); - length_barr[ (i * 2) + 1 ] = kvs[i].value_length; + length_barr[(i * 2) + 1] = kvs[i].value_length; offset += kvs[i].value_length; } - jenv->ReleaseByteArrayElements(keyValueArray, (jbyte *)keyvalues_barr, 0); + jenv->ReleaseByteArrayElements(keyValueArray, (jbyte*)keyvalues_barr, 0); jenv->ReleaseIntArrayElements(lengthArray, length_barr, 0); jobject result = jenv->NewObject(resultCls, resultCtorId, keyValueArray, lengthArray, (jboolean)more); - if( jenv->ExceptionOccurred() ) + if (jenv->ExceptionOccurred()) return JNI_NULL; return result; } // SOMEDAY: explore doing this more efficiently with Direct ByteBuffers -JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureResult_FutureResult_1get(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureResult_FutureResult_1get(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return JNI_NULL; } - FDBFuture *f = (FDBFuture *)future; + FDBFuture* f = (FDBFuture*)future; fdb_bool_t present; - const uint8_t *value; + const uint8_t* value; int length; fdb_error_t err = fdb_future_get_value(f, &present, &value, &length); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return JNI_NULL; } - if( !present ) + if (!present) return JNI_NULL; jbyteArray result = jenv->NewByteArray(length); - if( !result ) { - if( !jenv->ExceptionOccurred() ) + if (!result) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); return JNI_NULL; } - jenv->SetByteArrayRegion(result, 0, length, (const jbyte *)value); + jenv->SetByteArrayRegion(result, 0, length, (const jbyte*)value); return result; } -JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureKey_FutureKey_1get(JNIEnv * jenv, jclass, jlong future) { - if( !future ) { +JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureKey_FutureKey_1get(JNIEnv* jenv, jclass, jlong future) { + if (!future) { throwParamNotNull(jenv); return JNI_NULL; } - FDBFuture *f = (FDBFuture *)future; + FDBFuture* f = (FDBFuture*)future; - const uint8_t *value; + const uint8_t* value; int length; fdb_error_t err = fdb_future_get_key(f, &value, &length); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return JNI_NULL; } jbyteArray result = jenv->NewByteArray(length); - if( !result ) { - if( !jenv->ExceptionOccurred() ) + if (!result) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); return JNI_NULL; } - jenv->SetByteArrayRegion(result, 0, length, (const jbyte *)value); + jenv->SetByteArrayRegion(result, 0, length, (const jbyte*)value); return result; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1createTransaction(JNIEnv *jenv, jobject, jlong dbPtr) { - if( !dbPtr ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1createTransaction(JNIEnv* jenv, + jobject, + jlong dbPtr) { + if (!dbPtr) { throwParamNotNull(jenv); return 0; } - FDBDatabase *database = (FDBDatabase *)dbPtr; - FDBTransaction *tr; + FDBDatabase* database = (FDBDatabase*)dbPtr; + FDBTransaction* tr; fdb_error_t err = fdb_database_create_transaction(database, &tr); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return 0; } return (jlong)tr; } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1dispose(JNIEnv *jenv, jobject, jlong dPtr) { - if( !dPtr ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1dispose(JNIEnv* jenv, jobject, jlong dPtr) { + if (!dPtr) { throwParamNotNull(jenv); return; } - fdb_database_destroy( (FDBDatabase *)dPtr ); + fdb_database_destroy((FDBDatabase*)dPtr); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1setOption(JNIEnv *jenv, jobject, jlong dPtr, jint code, jbyteArray value) { - if( !dPtr ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1setOption(JNIEnv* jenv, + jobject, + jlong dPtr, + jint code, + jbyteArray value) { + if (!dPtr) { throwParamNotNull(jenv); return; } - FDBDatabase *c = (FDBDatabase *)dPtr; - uint8_t *barr = nullptr; + FDBDatabase* c = (FDBDatabase*)dPtr; + uint8_t* barr = nullptr; int size = 0; - if(value != JNI_NULL) { - barr = (uint8_t *)jenv->GetByteArrayElements( value, JNI_NULL ); + if (value != JNI_NULL) { + barr = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL); if (!barr) { - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - size = jenv->GetArrayLength( value ); + size = jenv->GetArrayLength(value); } - fdb_error_t err = fdb_database_set_option( c, (FDBDatabaseOption)code, barr, size ); - if(value != JNI_NULL) - jenv->ReleaseByteArrayElements( value, (jbyte *)barr, JNI_ABORT ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_database_set_option(c, (FDBDatabaseOption)code, barr, size); + if (value != JNI_NULL) + jenv->ReleaseByteArrayElements(value, (jbyte*)barr, JNI_ABORT); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); } } -JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIEnv *jenv, jobject, jint predicate, jint code) { +JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIEnv* jenv, + jobject, + jint predicate, + jint code) { return (jboolean)fdb_error_predicate(predicate, code); } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Database_1create(JNIEnv *jenv, jobject, jstring clusterFileName) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Database_1create(JNIEnv* jenv, + jobject, + jstring clusterFileName) { const char* fileName = nullptr; - if(clusterFileName != JNI_NULL) { + if (clusterFileName != JNI_NULL) { fileName = jenv->GetStringUTFChars(clusterFileName, JNI_NULL); - if(jenv->ExceptionOccurred()) { + if (jenv->ExceptionOccurred()) { return 0; } } - FDBDatabase *db; + FDBDatabase* db; fdb_error_t err = fdb_create_database(fileName, &db); - if(clusterFileName != JNI_NULL) { + if (clusterFileName != JNI_NULL) { jenv->ReleaseStringUTFChars(clusterFileName, fileName); } - if(err) { + if (err) { safeThrow(jenv, getThrowable(jenv, err)); return 0; } @@ -554,258 +585,315 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Database_1create(JNIEnv return (jlong)db; } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setVersion(JNIEnv *jenv, jobject, jlong tPtr, jlong version) { - if( !tPtr ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setVersion(JNIEnv* jenv, + jobject, + jlong tPtr, + jlong version) { + if (!tPtr) { throwParamNotNull(jenv); return; } - FDBTransaction *tr = (FDBTransaction *)tPtr; - fdb_transaction_set_read_version( tr, version ); + FDBTransaction* tr = (FDBTransaction*)tPtr; + fdb_transaction_set_read_version(tr, version); } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getReadVersion(JNIEnv *jenv, jobject, jlong tPtr) { - if( !tPtr ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getReadVersion(JNIEnv* jenv, + jobject, + jlong tPtr) { + if (!tPtr) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; - FDBFuture *f = fdb_transaction_get_read_version( tr ); + FDBTransaction* tr = (FDBTransaction*)tPtr; + FDBFuture* f = fdb_transaction_get_read_version(tr); return (jlong)f; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1get(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes, jboolean snapshot) { - if( !tPtr || !keyBytes ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1get(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray keyBytes, + jboolean snapshot) { + if (!tPtr || !keyBytes) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( keyBytes, JNI_NULL ); - if(!barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); - return 0; - } - - FDBFuture *f = fdb_transaction_get( tr, barr, jenv->GetArrayLength( keyBytes ), (fdb_bool_t)snapshot ); - jenv->ReleaseByteArrayElements( keyBytes, (jbyte *)barr, JNI_ABORT ); - return (jlong)f; -} - -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKey(JNIEnv *jenv, jobject, jlong tPtr, - jbyteArray keyBytes, jboolean orEqual, jint offset, jboolean snapshot) { - if( !tPtr || !keyBytes ) { - throwParamNotNull(jenv); - return 0; - } - FDBTransaction *tr = (FDBTransaction *)tPtr; - - uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( keyBytes, JNI_NULL ); - if(!barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); - return 0; - } - - FDBFuture *f = fdb_transaction_get_key( tr, barr, jenv->GetArrayLength( keyBytes ), orEqual, offset, (fdb_bool_t)snapshot ); - jenv->ReleaseByteArrayElements( keyBytes, (jbyte *)barr, JNI_ABORT ); - return (jlong)f; -} - -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRange - (JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBeginBytes, jboolean orEqualBegin, jint offsetBegin, - jbyteArray keyEndBytes, jboolean orEqualEnd, jint offsetEnd, jint rowLimit, jint targetBytes, - jint streamingMode, jint iteration, jboolean snapshot, jboolean reverse) { - if( !tPtr || !keyBeginBytes || !keyEndBytes ) { - throwParamNotNull(jenv); - return 0; - } - FDBTransaction *tr = (FDBTransaction *)tPtr; - - uint8_t *barrBegin = (uint8_t *)jenv->GetByteArrayElements( keyBeginBytes, JNI_NULL ); - if (!barrBegin) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); - return 0; - } - - uint8_t *barrEnd = (uint8_t *)jenv->GetByteArrayElements( keyEndBytes, JNI_NULL ); - if (!barrEnd) { - jenv->ReleaseByteArrayElements( keyBeginBytes, (jbyte *)barrBegin, JNI_ABORT ); - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); - return 0; - } - - FDBFuture *f = fdb_transaction_get_range( tr, - barrBegin, jenv->GetArrayLength( keyBeginBytes ), orEqualBegin, offsetBegin, - barrEnd, jenv->GetArrayLength( keyEndBytes ), orEqualEnd, offsetEnd, rowLimit, - targetBytes, (FDBStreamingMode)streamingMode, iteration, snapshot, reverse); - jenv->ReleaseByteArrayElements( keyBeginBytes, (jbyte *)barrBegin, JNI_ABORT ); - jenv->ReleaseByteArrayElements( keyEndBytes, (jbyte *)barrEnd, JNI_ABORT ); - return (jlong)f; -} - -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1set(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes, jbyteArray valueBytes) { - if( !tPtr || !keyBytes || !valueBytes ) { - throwParamNotNull(jenv); - return; - } - FDBTransaction *tr = (FDBTransaction *)tPtr; - - uint8_t *barrKey = (uint8_t *)jenv->GetByteArrayElements( keyBytes, JNI_NULL ); - if (!barrKey) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); - return; - } - - uint8_t *barrValue = (uint8_t *)jenv->GetByteArrayElements( valueBytes, JNI_NULL ); - if (!barrValue) { - jenv->ReleaseByteArrayElements( keyBytes, (jbyte *)barrKey, JNI_ABORT ); - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); - return; - } - - fdb_transaction_set( tr, - barrKey, jenv->GetArrayLength( keyBytes ), - barrValue, jenv->GetArrayLength( valueBytes ) ); - jenv->ReleaseByteArrayElements( keyBytes, (jbyte *)barrKey, JNI_ABORT ); - jenv->ReleaseByteArrayElements( valueBytes, (jbyte *)barrValue, JNI_ABORT ); -} - -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes) { - if( !tPtr || !keyBytes ) { - throwParamNotNull(jenv); - return; - } - FDBTransaction *tr = (FDBTransaction *)tPtr; - - uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( keyBytes, JNI_NULL ); + uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); if (!barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); - return; - } - - fdb_transaction_clear( tr, barr, jenv->GetArrayLength( keyBytes ) ); - jenv->ReleaseByteArrayElements( keyBytes, (jbyte *)barr, JNI_ABORT ); -} - -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B_3B(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBeginBytes, jbyteArray keyEndBytes) { - if( !tPtr || !keyBeginBytes || !keyEndBytes ) { - throwParamNotNull(jenv); - return; - } - FDBTransaction *tr = (FDBTransaction *)tPtr; - - uint8_t *barrKeyBegin = (uint8_t *)jenv->GetByteArrayElements( keyBeginBytes, JNI_NULL ); - if (!barrKeyBegin) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); - return; - } - - uint8_t *barrKeyEnd = (uint8_t *)jenv->GetByteArrayElements( keyEndBytes, JNI_NULL ); - if (!barrKeyEnd) { - jenv->ReleaseByteArrayElements( keyBeginBytes, (jbyte *)barrKeyBegin, JNI_ABORT ); - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); - return; - } - - fdb_transaction_clear_range( tr, - barrKeyBegin, jenv->GetArrayLength( keyBeginBytes ), - barrKeyEnd, jenv->GetArrayLength( keyEndBytes ) ); - jenv->ReleaseByteArrayElements( keyBeginBytes, (jbyte *)barrKeyBegin, JNI_ABORT ); - jenv->ReleaseByteArrayElements( keyEndBytes, (jbyte *)barrKeyEnd, JNI_ABORT ); -} - -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1mutate(JNIEnv *jenv, jobject, jlong tPtr, jint code, - jbyteArray key, jbyteArray value ) { - if( !tPtr || !key || !value ) { - throwParamNotNull(jenv); - return; - } - FDBTransaction *tr = (FDBTransaction *)tPtr; - - uint8_t *barrKey = (uint8_t *)jenv->GetByteArrayElements( key, JNI_NULL ); - if (!barrKey) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); - return; - } - - uint8_t *barrValue = (uint8_t *)jenv->GetByteArrayElements( value, JNI_NULL ); - if (!barrValue) { - jenv->ReleaseByteArrayElements( key, (jbyte *)barrKey, JNI_ABORT ); - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); - return; - } - - fdb_transaction_atomic_op( tr, - barrKey, jenv->GetArrayLength( key ), - barrValue, jenv->GetArrayLength( value ), - (FDBMutationType)code); - - jenv->ReleaseByteArrayElements( key, (jbyte *)barrKey, JNI_ABORT ); - jenv->ReleaseByteArrayElements( value, (jbyte *)barrValue, JNI_ABORT ); -} - -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1commit(JNIEnv *jenv, jobject, jlong tPtr) { - if( !tPtr ) { - throwParamNotNull(jenv); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; - FDBFuture *f = fdb_transaction_commit( tr ); + + FDBFuture* f = fdb_transaction_get(tr, barr, jenv->GetArrayLength(keyBytes), (fdb_bool_t)snapshot); + jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barr, JNI_ABORT); return (jlong)f; } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setOption(JNIEnv *jenv, jobject, jlong tPtr, jint code, jbyteArray value) { - if( !tPtr ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKey(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray keyBytes, + jboolean orEqual, + jint offset, + jboolean snapshot) { + if (!tPtr || !keyBytes) { + throwParamNotNull(jenv); + return 0; + } + FDBTransaction* tr = (FDBTransaction*)tPtr; + + uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); + if (!barr) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return 0; + } + + FDBFuture* f = + fdb_transaction_get_key(tr, barr, jenv->GetArrayLength(keyBytes), orEqual, offset, (fdb_bool_t)snapshot); + jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barr, JNI_ABORT); + return (jlong)f; +} + +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRange(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray keyBeginBytes, + jboolean orEqualBegin, + jint offsetBegin, + jbyteArray keyEndBytes, + jboolean orEqualEnd, + jint offsetEnd, + jint rowLimit, + jint targetBytes, + jint streamingMode, + jint iteration, + jboolean snapshot, + jboolean reverse) { + if (!tPtr || !keyBeginBytes || !keyEndBytes) { + throwParamNotNull(jenv); + return 0; + } + FDBTransaction* tr = (FDBTransaction*)tPtr; + + uint8_t* barrBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL); + if (!barrBegin) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return 0; + } + + uint8_t* barrEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL); + if (!barrEnd) { + jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return 0; + } + + FDBFuture* f = fdb_transaction_get_range(tr, + barrBegin, + jenv->GetArrayLength(keyBeginBytes), + orEqualBegin, + offsetBegin, + barrEnd, + jenv->GetArrayLength(keyEndBytes), + orEqualEnd, + offsetEnd, + rowLimit, + targetBytes, + (FDBStreamingMode)streamingMode, + iteration, + snapshot, + reverse); + jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT); + jenv->ReleaseByteArrayElements(keyEndBytes, (jbyte*)barrEnd, JNI_ABORT); + return (jlong)f; +} + +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1set(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray keyBytes, + jbyteArray valueBytes) { + if (!tPtr || !keyBytes || !valueBytes) { throwParamNotNull(jenv); return; } - FDBTransaction *tr = (FDBTransaction *)tPtr; - uint8_t *barr = nullptr; + FDBTransaction* tr = (FDBTransaction*)tPtr; + + uint8_t* barrKey = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); + if (!barrKey) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return; + } + + uint8_t* barrValue = (uint8_t*)jenv->GetByteArrayElements(valueBytes, JNI_NULL); + if (!barrValue) { + jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barrKey, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return; + } + + fdb_transaction_set(tr, barrKey, jenv->GetArrayLength(keyBytes), barrValue, jenv->GetArrayLength(valueBytes)); + jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barrKey, JNI_ABORT); + jenv->ReleaseByteArrayElements(valueBytes, (jbyte*)barrValue, JNI_ABORT); +} + +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray keyBytes) { + if (!tPtr || !keyBytes) { + throwParamNotNull(jenv); + return; + } + FDBTransaction* tr = (FDBTransaction*)tPtr; + + uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); + if (!barr) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return; + } + + fdb_transaction_clear(tr, barr, jenv->GetArrayLength(keyBytes)); + jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barr, JNI_ABORT); +} + +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B_3B(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray keyBeginBytes, + jbyteArray keyEndBytes) { + if (!tPtr || !keyBeginBytes || !keyEndBytes) { + throwParamNotNull(jenv); + return; + } + FDBTransaction* tr = (FDBTransaction*)tPtr; + + uint8_t* barrKeyBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL); + if (!barrKeyBegin) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return; + } + + uint8_t* barrKeyEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL); + if (!barrKeyEnd) { + jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrKeyBegin, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return; + } + + fdb_transaction_clear_range( + tr, barrKeyBegin, jenv->GetArrayLength(keyBeginBytes), barrKeyEnd, jenv->GetArrayLength(keyEndBytes)); + jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrKeyBegin, JNI_ABORT); + jenv->ReleaseByteArrayElements(keyEndBytes, (jbyte*)barrKeyEnd, JNI_ABORT); +} + +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1mutate(JNIEnv* jenv, + jobject, + jlong tPtr, + jint code, + jbyteArray key, + jbyteArray value) { + if (!tPtr || !key || !value) { + throwParamNotNull(jenv); + return; + } + FDBTransaction* tr = (FDBTransaction*)tPtr; + + uint8_t* barrKey = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL); + if (!barrKey) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return; + } + + uint8_t* barrValue = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL); + if (!barrValue) { + jenv->ReleaseByteArrayElements(key, (jbyte*)barrKey, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); + return; + } + + fdb_transaction_atomic_op( + tr, barrKey, jenv->GetArrayLength(key), barrValue, jenv->GetArrayLength(value), (FDBMutationType)code); + + jenv->ReleaseByteArrayElements(key, (jbyte*)barrKey, JNI_ABORT); + jenv->ReleaseByteArrayElements(value, (jbyte*)barrValue, JNI_ABORT); +} + +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1commit(JNIEnv* jenv, + jobject, + jlong tPtr) { + if (!tPtr) { + throwParamNotNull(jenv); + return 0; + } + FDBTransaction* tr = (FDBTransaction*)tPtr; + FDBFuture* f = fdb_transaction_commit(tr); + return (jlong)f; +} + +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setOption(JNIEnv* jenv, + jobject, + jlong tPtr, + jint code, + jbyteArray value) { + if (!tPtr) { + throwParamNotNull(jenv); + return; + } + FDBTransaction* tr = (FDBTransaction*)tPtr; + uint8_t* barr = nullptr; int size = 0; - if(value != JNI_NULL) { - barr = (uint8_t *)jenv->GetByteArrayElements( value, JNI_NULL ); + if (value != JNI_NULL) { + barr = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL); if (!barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - size = jenv->GetArrayLength( value ); + size = jenv->GetArrayLength(value); } - fdb_error_t err = fdb_transaction_set_option( tr, (FDBTransactionOption)code, barr, size ); - if(value != JNI_NULL) - jenv->ReleaseByteArrayElements( value, (jbyte *)barr, JNI_ABORT ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_transaction_set_option(tr, (FDBTransactionOption)code, barr, size); + if (value != JNI_NULL) + jenv->ReleaseByteArrayElements(value, (jbyte*)barr, JNI_ABORT); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); } } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getCommittedVersion(JNIEnv *jenv, jobject, jlong tPtr) { - if( !tPtr ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getCommittedVersion(JNIEnv* jenv, + jobject, + jlong tPtr) { + if (!tPtr) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; int64_t version; - fdb_error_t err = fdb_transaction_get_committed_version( tr, &version ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_transaction_get_committed_version(tr, &version); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return 0; } return (jlong)version; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getApproximateSize(JNIEnv *jenv, jobject, jlong tPtr) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getApproximateSize(JNIEnv* jenv, + jobject, + jlong tPtr) { if (!tPtr) { throwParamNotNull(jenv); return 0; @@ -814,205 +902,234 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 return (jlong)f; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getVersionstamp(JNIEnv *jenv, jobject, jlong tPtr) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getVersionstamp(JNIEnv* jenv, + jobject, + jlong tPtr) { if (!tPtr) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; - FDBFuture *f = fdb_transaction_get_versionstamp(tr); + FDBTransaction* tr = (FDBTransaction*)tPtr; + FDBFuture* f = fdb_transaction_get_versionstamp(tr); return (jlong)f; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKeyLocations(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray key) { - if( !tPtr || !key ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKeyLocations(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray key) { + if (!tPtr || !key) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( key, JNI_NULL ); + uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL); if (!barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - int size = jenv->GetArrayLength( key ); + int size = jenv->GetArrayLength(key); - FDBFuture *f = fdb_transaction_get_addresses_for_key( tr, barr, size ); + FDBFuture* f = fdb_transaction_get_addresses_for_key(tr, barr, size); - jenv->ReleaseByteArrayElements( key, (jbyte *)barr, JNI_ABORT ); + jenv->ReleaseByteArrayElements(key, (jbyte*)barr, JNI_ABORT); return (jlong)f; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1onError(JNIEnv *jenv, jobject, jlong tPtr, jint errorCode) { - if( !tPtr ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1onError(JNIEnv* jenv, + jobject, + jlong tPtr, + jint errorCode) { + if (!tPtr) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; - FDBFuture *f = fdb_transaction_on_error( tr, (fdb_error_t)errorCode ); + FDBTransaction* tr = (FDBTransaction*)tPtr; + FDBFuture* f = fdb_transaction_on_error(tr, (fdb_error_t)errorCode); return (jlong)f; } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1dispose(JNIEnv *jenv, jobject, jlong tPtr) { - if( !tPtr ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1dispose(JNIEnv* jenv, + jobject, + jlong tPtr) { + if (!tPtr) { throwParamNotNull(jenv); return; } - fdb_transaction_destroy( (FDBTransaction *)tPtr ); + fdb_transaction_destroy((FDBTransaction*)tPtr); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1reset(JNIEnv *jenv, jobject, jlong tPtr) { - if( !tPtr ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1reset(JNIEnv* jenv, + jobject, + jlong tPtr) { + if (!tPtr) { throwParamNotNull(jenv); return; } - fdb_transaction_reset( (FDBTransaction *)tPtr ); + fdb_transaction_reset((FDBTransaction*)tPtr); } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1watch(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray key) { - if( !tPtr || !key ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1watch(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray key) { + if (!tPtr || !key) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( key, JNI_NULL ); + uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL); if (!barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - int size = jenv->GetArrayLength( key ); - FDBFuture *f = fdb_transaction_watch( tr, barr, size ); + int size = jenv->GetArrayLength(key); + FDBFuture* f = fdb_transaction_watch(tr, barr, size); - jenv->ReleaseByteArrayElements( key, (jbyte *)barr, JNI_ABORT ); + jenv->ReleaseByteArrayElements(key, (jbyte*)barr, JNI_ABORT); return (jlong)f; } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1cancel(JNIEnv *jenv, jobject, jlong tPtr) { - if( !tPtr ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1cancel(JNIEnv* jenv, + jobject, + jlong tPtr) { + if (!tPtr) { throwParamNotNull(jenv); return; } - fdb_transaction_cancel( (FDBTransaction *)tPtr ); + fdb_transaction_cancel((FDBTransaction*)tPtr); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1addConflictRange( - JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBegin, jbyteArray keyEnd, jint conflictType) { - if( !tPtr || !keyBegin || !keyEnd ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1addConflictRange(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray keyBegin, + jbyteArray keyEnd, + jint conflictType) { + if (!tPtr || !keyBegin || !keyEnd) { throwParamNotNull(jenv); return; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *begin_barr = (uint8_t *)jenv->GetByteArrayElements( keyBegin, JNI_NULL ); + uint8_t* begin_barr = (uint8_t*)jenv->GetByteArrayElements(keyBegin, JNI_NULL); if (!begin_barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - int begin_size = jenv->GetArrayLength( keyBegin ); + int begin_size = jenv->GetArrayLength(keyBegin); - uint8_t *end_barr = (uint8_t *)jenv->GetByteArrayElements( keyEnd, JNI_NULL ); + uint8_t* end_barr = (uint8_t*)jenv->GetByteArrayElements(keyEnd, JNI_NULL); if (!end_barr) { - jenv->ReleaseByteArrayElements( keyBegin, (jbyte *)begin_barr, JNI_ABORT ); - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + jenv->ReleaseByteArrayElements(keyBegin, (jbyte*)begin_barr, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - int end_size = jenv->GetArrayLength( keyEnd ); + int end_size = jenv->GetArrayLength(keyEnd); - fdb_error_t err = fdb_transaction_add_conflict_range( tr, begin_barr, begin_size, end_barr, end_size, (FDBConflictRangeType)conflictType ); + fdb_error_t err = fdb_transaction_add_conflict_range( + tr, begin_barr, begin_size, end_barr, end_size, (FDBConflictRangeType)conflictType); - jenv->ReleaseByteArrayElements( keyBegin, (jbyte *)begin_barr, JNI_ABORT ); - jenv->ReleaseByteArrayElements( keyEnd, (jbyte *)end_barr, JNI_ABORT ); + jenv->ReleaseByteArrayElements(keyBegin, (jbyte*)begin_barr, JNI_ABORT); + jenv->ReleaseByteArrayElements(keyEnd, (jbyte*)end_barr, JNI_ABORT); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); } } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Select_1API_1version(JNIEnv *jenv, jclass, jint version) { - fdb_error_t err = fdb_select_api_version( (int)version ); - if( err ) { - if( err == 2203 ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Select_1API_1version(JNIEnv* jenv, jclass, jint version) { + fdb_error_t err = fdb_select_api_version((int)version); + if (err) { + if (err == 2203) { int maxSupportedVersion = fdb_get_max_api_version(); char errorStr[1024]; - if(FDB_API_VERSION > maxSupportedVersion) { - snprintf(errorStr, sizeof(errorStr), "This version of the FoundationDB Java binding is not supported by the installed " - "FoundationDB C library. The binding requires a library that supports API version " - "%d, but the installed library supports a maximum version of %d.", - FDB_API_VERSION, maxSupportedVersion); - } - else { - snprintf(errorStr, sizeof(errorStr), "API version %d is not supported by the installed FoundationDB C library.", version); + if (FDB_API_VERSION > maxSupportedVersion) { + snprintf(errorStr, + sizeof(errorStr), + "This version of the FoundationDB Java binding is not supported by the installed " + "FoundationDB C library. The binding requires a library that supports API version " + "%d, but the installed library supports a maximum version of %d.", + FDB_API_VERSION, + maxSupportedVersion); + } else { + snprintf(errorStr, + sizeof(errorStr), + "API version %d is not supported by the installed FoundationDB C library.", + version); } - safeThrow( jenv, getThrowable( jenv, err, errorStr ) ); - } - else { - safeThrow( jenv, getThrowable( jenv, err ) ); + safeThrow(jenv, getThrowable(jenv, err, errorStr)); + } else { + safeThrow(jenv, getThrowable(jenv, err)); } } } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1setOption(JNIEnv *jenv, jobject, jint code, jbyteArray value) { - uint8_t *barr = nullptr; +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1setOption(JNIEnv* jenv, + jobject, + jint code, + jbyteArray value) { + uint8_t* barr = nullptr; int size = 0; - if(value != JNI_NULL) { - barr = (uint8_t *)jenv->GetByteArrayElements( value, JNI_NULL ); + if (value != JNI_NULL) { + barr = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL); if (!barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - size = jenv->GetArrayLength( value ); + size = jenv->GetArrayLength(value); } fdb_error_t err = fdb_network_set_option((FDBNetworkOption)code, barr, size); - if(value != JNI_NULL) - jenv->ReleaseByteArrayElements( value, (jbyte *)barr, JNI_ABORT ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (value != JNI_NULL) + jenv->ReleaseByteArrayElements(value, (jbyte*)barr, JNI_ABORT); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); } } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1setup(JNIEnv *jenv, jobject) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1setup(JNIEnv* jenv, jobject) { fdb_error_t err = fdb_setup_network(); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); } } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1run(JNIEnv *jenv, jobject) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1run(JNIEnv* jenv, jobject) { // initialize things for the callbacks on the network thread g_thread_jenv = jenv; - if( !g_IFutureCallback_call_methodID ) { - if( !findCallbackMethods( jenv ) ) + if (!g_IFutureCallback_call_methodID) { + if (!findCallbackMethods(jenv)) return; } - fdb_error_t hookErr = fdb_add_network_thread_completion_hook( &detachIfExternalThread, nullptr ); - if( hookErr ) { - safeThrow( jenv, getThrowable( jenv, hookErr ) ); + fdb_error_t hookErr = fdb_add_network_thread_completion_hook(&detachIfExternalThread, nullptr); + if (hookErr) { + safeThrow(jenv, getThrowable(jenv, hookErr)); } fdb_error_t err = fdb_run_network(); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); } } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1stop(JNIEnv *jenv, jobject) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1stop(JNIEnv* jenv, jobject) { fdb_error_t err = fdb_stop_network(); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); } } -jint JNI_OnLoad(JavaVM *vm, void *reserved) { +jint JNI_OnLoad(JavaVM* vm, void* reserved) { g_jvm = vm; return JNI_VERSION_1_1; } @@ -1020,4 +1137,3 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) { #ifdef __cplusplus } #endif - diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 7b24516747..418918613c 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -43,7 +43,7 @@ #include #include #include -#include // std::transform +#include // std::transform #include #include #include @@ -57,7 +57,7 @@ using std::endl; #endif #include -#ifdef __linux__ +#ifdef __linux__ #include #ifdef ALLOC_INSTRUMENTATION #include @@ -68,51 +68,94 @@ using std::endl; #include "versions.h" #endif #include "flow/SimpleOpt.h" -#include "flow/actorcompiler.h" // This must be the last #include. - +#include "flow/actorcompiler.h" // This must be the last #include. // Type of program being executed -enum enumProgramExe { - EXE_AGENT, EXE_BACKUP, EXE_RESTORE, EXE_DR_AGENT, EXE_DB_BACKUP, EXE_UNDEFINED -}; +enum enumProgramExe { EXE_AGENT, EXE_BACKUP, EXE_RESTORE, EXE_DR_AGENT, EXE_DB_BACKUP, EXE_UNDEFINED }; enum enumBackupType { - BACKUP_UNDEFINED=0, BACKUP_START, BACKUP_MODIFY, BACKUP_STATUS, BACKUP_ABORT, BACKUP_WAIT, BACKUP_DISCONTINUE, BACKUP_PAUSE, BACKUP_RESUME, BACKUP_EXPIRE, BACKUP_DELETE, BACKUP_DESCRIBE, BACKUP_LIST, BACKUP_DUMP, BACKUP_CLEANUP + BACKUP_UNDEFINED = 0, + BACKUP_START, + BACKUP_MODIFY, + BACKUP_STATUS, + BACKUP_ABORT, + BACKUP_WAIT, + BACKUP_DISCONTINUE, + BACKUP_PAUSE, + BACKUP_RESUME, + BACKUP_EXPIRE, + BACKUP_DELETE, + BACKUP_DESCRIBE, + BACKUP_LIST, + BACKUP_DUMP, + BACKUP_CLEANUP }; -enum enumDBType { - DB_UNDEFINED=0, DB_START, DB_STATUS, DB_SWITCH, DB_ABORT, DB_PAUSE, DB_RESUME -}; +enum enumDBType { DB_UNDEFINED = 0, DB_START, DB_STATUS, DB_SWITCH, DB_ABORT, DB_PAUSE, DB_RESUME }; -enum enumRestoreType { - RESTORE_UNKNOWN, RESTORE_START, RESTORE_STATUS, RESTORE_ABORT, RESTORE_WAIT -}; +enum enumRestoreType { RESTORE_UNKNOWN, RESTORE_START, RESTORE_STATUS, RESTORE_ABORT, RESTORE_WAIT }; // enum { // Backup constants - OPT_DESTCONTAINER, OPT_SNAPSHOTINTERVAL, OPT_ERRORLIMIT, OPT_NOSTOPWHENDONE, - OPT_EXPIRE_BEFORE_VERSION, OPT_EXPIRE_BEFORE_DATETIME, OPT_EXPIRE_DELETE_BEFORE_DAYS, - OPT_EXPIRE_RESTORABLE_AFTER_VERSION, OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, OPT_EXPIRE_MIN_RESTORABLE_DAYS, - OPT_BASEURL, OPT_BLOB_CREDENTIALS, OPT_DESCRIBE_DEEP, OPT_DESCRIBE_TIMESTAMPS, - OPT_DUMP_BEGIN, OPT_DUMP_END, OPT_JSON, OPT_DELETE_DATA, OPT_MIN_CLEANUP_SECONDS, + OPT_DESTCONTAINER, + OPT_SNAPSHOTINTERVAL, + OPT_ERRORLIMIT, + OPT_NOSTOPWHENDONE, + OPT_EXPIRE_BEFORE_VERSION, + OPT_EXPIRE_BEFORE_DATETIME, + OPT_EXPIRE_DELETE_BEFORE_DAYS, + OPT_EXPIRE_RESTORABLE_AFTER_VERSION, + OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, + OPT_EXPIRE_MIN_RESTORABLE_DAYS, + OPT_BASEURL, + OPT_BLOB_CREDENTIALS, + OPT_DESCRIBE_DEEP, + OPT_DESCRIBE_TIMESTAMPS, + OPT_DUMP_BEGIN, + OPT_DUMP_END, + OPT_JSON, + OPT_DELETE_DATA, + OPT_MIN_CLEANUP_SECONDS, // Backup and Restore constants - OPT_TAGNAME, OPT_BACKUPKEYS, OPT_WAITFORDONE, + OPT_TAGNAME, + OPT_BACKUPKEYS, + OPT_WAITFORDONE, // Backup Modify - OPT_MOD_ACTIVE_INTERVAL, OPT_MOD_VERIFY_UID, + OPT_MOD_ACTIVE_INTERVAL, + OPT_MOD_VERIFY_UID, // Restore constants - OPT_RESTORECONTAINER, OPT_RESTORE_VERSION, OPT_RESTORE_TIMESTAMP, OPT_PREFIX_ADD, OPT_PREFIX_REMOVE, OPT_RESTORE_CLUSTERFILE_DEST, OPT_RESTORE_CLUSTERFILE_ORIG, + OPT_RESTORECONTAINER, + OPT_RESTORE_VERSION, + OPT_RESTORE_TIMESTAMP, + OPT_PREFIX_ADD, + OPT_PREFIX_REMOVE, + OPT_RESTORE_CLUSTERFILE_DEST, + OPT_RESTORE_CLUSTERFILE_ORIG, // Shared constants - OPT_CLUSTERFILE, OPT_QUIET, OPT_DRYRUN, OPT_FORCE, - OPT_HELP, OPT_DEVHELP, OPT_VERSION, OPT_PARENTPID, OPT_CRASHONERROR, - OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR, OPT_TRACE, OPT_TRACE_DIR, - OPT_KNOB, OPT_TRACE_LOG_GROUP, OPT_MEMLIMIT, OPT_LOCALITY, + OPT_CLUSTERFILE, + OPT_QUIET, + OPT_DRYRUN, + OPT_FORCE, + OPT_HELP, + OPT_DEVHELP, + OPT_VERSION, + OPT_PARENTPID, + OPT_CRASHONERROR, + OPT_NOBUFSTDOUT, + OPT_BUFSTDOUTERR, + OPT_TRACE, + OPT_TRACE_DIR, + OPT_KNOB, + OPT_TRACE_LOG_GROUP, + OPT_MEMLIMIT, + OPT_LOCALITY, - //DB constants + // DB constants OPT_SOURCE_CLUSTER, OPT_DEST_CLUSTER, OPT_CLEANUP, @@ -152,75 +195,75 @@ CSimpleOpt::SOption g_rgAgentOptions[] = { CSimpleOpt::SOption g_rgBackupStartOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_WAITFORDONE, "-w", SO_NONE }, - { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, - { OPT_NOSTOPWHENDONE, "-z", SO_NONE }, - { OPT_NOSTOPWHENDONE, "--no-stop-when-done",SO_NONE }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP }, - { OPT_SNAPSHOTINTERVAL, "--snapshot_interval", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, - { OPT_DRYRUN, "-n", SO_NONE }, - { OPT_DRYRUN, "--dryrun", SO_NONE }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_WAITFORDONE, "-w", SO_NONE }, + { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, + { OPT_NOSTOPWHENDONE, "-z", SO_NONE }, + { OPT_NOSTOPWHENDONE, "--no-stop-when-done", SO_NONE }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP }, + { OPT_SNAPSHOTINTERVAL, "--snapshot_interval", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, + { OPT_DRYRUN, "-n", SO_NONE }, + { OPT_DRYRUN, "--dryrun", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupModifyOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_MOD_VERIFY_UID, "--verify_uid", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_MOD_VERIFY_UID, "--verify_uid", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP }, { OPT_SNAPSHOTINTERVAL, "--snapshot_interval", SO_REQ_SEP }, { OPT_MOD_ACTIVE_INTERVAL, "--active_snapshot_interval", SO_REQ_SEP }, @@ -229,589 +272,589 @@ CSimpleOpt::SOption g_rgBackupModifyOptions[] = { CSimpleOpt::SOption g_rgBackupStatusOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_ERRORLIMIT, "-e", SO_REQ_SEP }, - { OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_JSON, "--json", SO_NONE}, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_ERRORLIMIT, "-e", SO_REQ_SEP }, + { OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_JSON, "--json", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupAbortOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupCleanupOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_DELETE_DATA, "--delete_data", SO_NONE }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_DELETE_DATA, "--delete_data", SO_NONE }, { OPT_MIN_CLEANUP_SECONDS, "--min_cleanup_seconds", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupDiscontinueOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_WAITFORDONE, "-w", SO_NONE }, - { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_WAITFORDONE, "-w", SO_NONE }, + { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupWaitOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_NOSTOPWHENDONE, "-z", SO_NONE }, - { OPT_NOSTOPWHENDONE, "--no-stop-when-done",SO_NONE }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_NOSTOPWHENDONE, "-z", SO_NONE }, + { OPT_NOSTOPWHENDONE, "--no-stop-when-done", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupPauseOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupExpireOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_FORCE, "-f", SO_NONE }, - { OPT_FORCE, "--force", SO_NONE }, - { OPT_EXPIRE_RESTORABLE_AFTER_VERSION, "--restorable_after_version", SO_REQ_SEP }, - { OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, "--restorable_after_timestamp", SO_REQ_SEP }, - { OPT_EXPIRE_BEFORE_VERSION, "--expire_before_version", SO_REQ_SEP }, - { OPT_EXPIRE_BEFORE_DATETIME, "--expire_before_timestamp", SO_REQ_SEP }, - { OPT_EXPIRE_MIN_RESTORABLE_DAYS, "--min_restorable_days", SO_REQ_SEP }, - { OPT_EXPIRE_DELETE_BEFORE_DAYS, "--delete_before_days", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_FORCE, "-f", SO_NONE }, + { OPT_FORCE, "--force", SO_NONE }, + { OPT_EXPIRE_RESTORABLE_AFTER_VERSION, "--restorable_after_version", SO_REQ_SEP }, + { OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, "--restorable_after_timestamp", SO_REQ_SEP }, + { OPT_EXPIRE_BEFORE_VERSION, "--expire_before_version", SO_REQ_SEP }, + { OPT_EXPIRE_BEFORE_DATETIME, "--expire_before_timestamp", SO_REQ_SEP }, + { OPT_EXPIRE_MIN_RESTORABLE_DAYS, "--min_restorable_days", SO_REQ_SEP }, + { OPT_EXPIRE_DELETE_BEFORE_DAYS, "--delete_before_days", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupDeleteOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupDescribeOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_DESCRIBE_DEEP, "--deep", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_DESCRIBE_DEEP, "--deep", SO_NONE }, { OPT_DESCRIBE_TIMESTAMPS, "--version_timestamps", SO_NONE }, - { OPT_JSON, "--json", SO_NONE}, + { OPT_JSON, "--json", SO_NONE }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupDumpOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_DUMP_BEGIN, "--begin", SO_REQ_SEP }, - { OPT_DUMP_END, "--end", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_DUMP_BEGIN, "--begin", SO_REQ_SEP }, + { OPT_DUMP_END, "--end", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupListOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_BASEURL, "-b", SO_REQ_SEP }, - { OPT_BASEURL, "--base_url", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_BASEURL, "-b", SO_REQ_SEP }, + { OPT_BASEURL, "--base_url", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgRestoreOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_RESTORE_CLUSTERFILE_DEST, "--dest_cluster_file", SO_REQ_SEP }, - { OPT_RESTORE_CLUSTERFILE_ORIG, "--orig_cluster_file", SO_REQ_SEP }, - { OPT_RESTORE_TIMESTAMP, "--timestamp", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_RESTORECONTAINER,"-r", SO_REQ_SEP }, - { OPT_PREFIX_ADD, "-add_prefix", SO_REQ_SEP }, // TODO: Remove in 6.3 - { OPT_PREFIX_ADD, "--add_prefix", SO_REQ_SEP }, - { OPT_PREFIX_REMOVE, "-remove_prefix", SO_REQ_SEP }, // TODO: Remove in 6.3 - { OPT_PREFIX_REMOVE, "--remove_prefix", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, - { OPT_WAITFORDONE, "-w", SO_NONE }, - { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, - { OPT_RESTORE_VERSION, "--version", SO_REQ_SEP }, - { OPT_RESTORE_VERSION, "-v", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_DRYRUN, "-n", SO_NONE }, - { OPT_DRYRUN, "--dryrun", SO_NONE }, - { OPT_FORCE, "-f", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_RESTORE_CLUSTERFILE_DEST, "--dest_cluster_file", SO_REQ_SEP }, + { OPT_RESTORE_CLUSTERFILE_ORIG, "--orig_cluster_file", SO_REQ_SEP }, + { OPT_RESTORE_TIMESTAMP, "--timestamp", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_RESTORECONTAINER, "-r", SO_REQ_SEP }, + { OPT_PREFIX_ADD, "-add_prefix", SO_REQ_SEP }, // TODO: Remove in 6.3 + { OPT_PREFIX_ADD, "--add_prefix", SO_REQ_SEP }, + { OPT_PREFIX_REMOVE, "-remove_prefix", SO_REQ_SEP }, // TODO: Remove in 6.3 + { OPT_PREFIX_REMOVE, "--remove_prefix", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, + { OPT_WAITFORDONE, "-w", SO_NONE }, + { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, + { OPT_RESTORE_VERSION, "--version", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "-v", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_DRYRUN, "-n", SO_NONE }, + { OPT_DRYRUN, "--dryrun", SO_NONE }, + { OPT_FORCE, "-f", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgDBAgentOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_LOCALITY, "--locality_", SO_REQ_SEP }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_LOCALITY, "--locality_", SO_REQ_SEP }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgDBStartOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgDBStatusOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_ERRORLIMIT, "-e", SO_REQ_SEP }, - { OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_ERRORLIMIT, "-e", SO_REQ_SEP }, + { OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgDBSwitchOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_FORCE, "-f", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_FORCE, "-f", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgDBAbortOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_CLEANUP, "--cleanup", SO_NONE }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_CLEANUP, "--cleanup", SO_NONE }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgDBPauseOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; const KeyRef exeAgent = LiteralStringRef("backup_agent"); @@ -823,7 +866,7 @@ const KeyRef exeDatabaseBackup = LiteralStringRef("fdbdr"); extern const char* getHGVersion(); #ifdef _WIN32 -void parentWatcher(void *parentHandle) { +void parentWatcher(void* parentHandle) { HANDLE parent = (HANDLE)parentHandle; int signal = WaitForSingleObject(parent, INFINITE); CloseHandle(parentHandle); @@ -837,22 +880,25 @@ void parentWatcher(void *parentHandle) { static void printVersion() { printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); printf("source version %s\n", getHGVersion()); - printf("protocol %llx\n", (long long) currentProtocolVersion.version()); + printf("protocol %llx\n", (long long)currentProtocolVersion.version()); } -const char *BlobCredentialInfo = - " BLOB CREDENTIALS\n" - " Blob account secret keys can optionally be omitted from blobstore:// URLs, in which case they will be\n" - " loaded, if possible, from 1 or more blob credentials definition files.\n\n" - " These files can be specified with the --blob_credentials argument described above or via the environment variable\n" - " FDB_BLOB_CREDENTIALS, whose value is a colon-separated list of files. The command line takes priority over\n" - " over the environment but all files from both sources are used.\n\n" - " At connect time, the specified files are read in order and the first matching account specification (user@host)\n" - " will be used to obtain the secret key.\n\n" - " The JSON schema is:\n" - " { \"accounts\" : { \"user@host\" : { \"secret\" : \"SECRETKEY\" }, \"user2@host2\" : { \"secret\" : \"SECRET\" } } }\n"; +const char* BlobCredentialInfo = + " BLOB CREDENTIALS\n" + " Blob account secret keys can optionally be omitted from blobstore:// URLs, in which case they will be\n" + " loaded, if possible, from 1 or more blob credentials definition files.\n\n" + " These files can be specified with the --blob_credentials argument described above or via the environment " + "variable\n" + " FDB_BLOB_CREDENTIALS, whose value is a colon-separated list of files. The command line takes priority over\n" + " over the environment but all files from both sources are used.\n\n" + " At connect time, the specified files are read in order and the first matching account specification " + "(user@host)\n" + " will be used to obtain the secret key.\n\n" + " The JSON schema is:\n" + " { \"accounts\" : { \"user@host\" : { \"secret\" : \"SECRETKEY\" }, \"user2@host2\" : { \"secret\" : " + "\"SECRET\" } } }\n"; -static void printHelpTeaser( const char *name ) { +static void printHelpTeaser(const char* name) { fprintf(stderr, "Try `%s --help' for more information.\n", name); } @@ -860,22 +906,23 @@ static void printAgentUsage(bool devhelp) { printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); printf("Usage: %s [OPTIONS]\n\n", exeAgent.toString().c_str()); printf(" -C CONNFILE The path of a file containing the connection string for the\n" - " FoundationDB cluster. The default is first the value of the\n" - " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" - " then `%s'.\n", platform::getDefaultClusterFilePath().c_str()); + " FoundationDB cluster. The default is first the value of the\n" + " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" + " then `%s'.\n", + platform::getDefaultClusterFilePath().c_str()); printf(" --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifes the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n"); + " --logdir PATH Specifes the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); printf(" --loggroup LOG_GROUP\n" " Sets the LogGroup field with the specified value for all\n" " events in the trace output (defaults to `default').\n"); printf(" --trace_format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); printf(" -m SIZE, --memory SIZE\n" - " Memory limit. The default value is 8GiB. When specified\n" - " without a unit, MiB is assumed.\n"); + " Memory limit. The default value is 8GiB. When specified\n" + " without a unit, MiB is assumed.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif @@ -900,76 +947,94 @@ static void printAgentUsage(bool devhelp) { void printBackupContainerInfo() { printf(" Backup URL forms:\n\n"); std::vector formats = IBackupContainer::getURLFormats(); - for(auto &f : formats) + for (auto& f : formats) printf(" %s\n", f.c_str()); printf("\n"); } static void printBackupUsage(bool devhelp) { printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); - printf("Usage: %s (start | status | abort | wait | discontinue | pause | resume | expire | delete | describe | list | cleanup) [OPTIONS]\n\n", exeBackup.toString().c_str()); + printf("Usage: %s (start | status | abort | wait | discontinue | pause | resume | expire | delete | describe | " + "list | cleanup) [OPTIONS]\n\n", + exeBackup.toString().c_str()); printf(" -C CONNFILE The path of a file containing the connection string for the\n" - " FoundationDB cluster. The default is first the value of the\n" - " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" - " then `%s'.\n", platform::getDefaultClusterFilePath().c_str()); + " FoundationDB cluster. The default is first the value of the\n" + " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" + " then `%s'.\n", + platform::getDefaultClusterFilePath().c_str()); printf(" -d, --destcontainer URL\n" " The Backup container URL for start, modify, describe, expire, and delete operations.\n"); printBackupContainerInfo(); printf(" -b, --base_url BASEURL\n" - " Base backup URL for list operations. This looks like a Backup URL but without a backup name.\n"); + " Base backup URL for list operations. This looks like a Backup URL but without a backup " + "name.\n"); printf(" --blob_credentials FILE\n" - " File containing blob credentials in JSON format. Can be specified multiple times for multiple files. See below for more details.\n"); + " File containing blob credentials in JSON format. Can be specified multiple times for " + "multiple files. See below for more details.\n"); printf(" --expire_before_timestamp DATETIME\n" - " Datetime cutoff for expire operations. Requires a cluster file and will use version/timestamp metadata\n" - " in the database to obtain a cutoff version very close to the timestamp given in %s.\n", BackupAgentBase::timeFormat().c_str()); + " Datetime cutoff for expire operations. Requires a cluster file and will use " + "version/timestamp metadata\n" + " in the database to obtain a cutoff version very close to the timestamp given in %s.\n", + BackupAgentBase::timeFormat().c_str()); printf(" --expire_before_version VERSION\n" - " Version cutoff for expire operations. Deletes data files containing no data at or after VERSION.\n"); + " Version cutoff for expire operations. Deletes data files containing no data at or after " + "VERSION.\n"); printf(" --delete_before_days NUM_DAYS\n" - " Another way to specify version cutoff for expire operations. Deletes data files containing no data at or after a\n" - " version approximately NUM_DAYS days worth of versions prior to the latest log version in the backup.\n"); + " Another way to specify version cutoff for expire operations. Deletes data files " + "containing no data at or after a\n" + " version approximately NUM_DAYS days worth of versions prior to the latest log version in " + "the backup.\n"); printf(" --restorable_after_timestamp DATETIME\n" - " For expire operations, set minimum acceptable restorability to the version equivalent of DATETIME and later.\n"); + " For expire operations, set minimum acceptable restorability to the version equivalent of " + "DATETIME and later.\n"); printf(" --restorable_after_version VERSION\n" - " For expire operations, set minimum acceptable restorability to the VERSION and later.\n"); + " For expire operations, set minimum acceptable restorability to the VERSION and later.\n"); printf(" --min_restorable_days NUM_DAYS\n" - " For expire operations, set minimum acceptable restorability to approximately NUM_DAYS days worth of versions\n" - " prior to the latest log version in the backup.\n"); + " For expire operations, set minimum acceptable restorability to approximately NUM_DAYS " + "days worth of versions\n" + " prior to the latest log version in the backup.\n"); printf(" --version_timestamps\n"); - printf(" For describe operations, lookup versions in the database to obtain timestamps. A cluster file is required.\n"); - printf(" -f, --force For expire operations, force expiration even if minimum restorability would be violated.\n"); + printf(" For describe operations, lookup versions in the database to obtain timestamps. A cluster " + "file is required.\n"); + printf( + " -f, --force For expire operations, force expiration even if minimum restorability would be violated.\n"); printf(" -s, --snapshot_interval DURATION\n" - " For start or modify operations, specifies the backup's default target snapshot interval as DURATION seconds. Defaults to %d for start operations.\n", CLIENT_KNOBS->BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC); + " For start or modify operations, specifies the backup's default target snapshot interval " + "as DURATION seconds. Defaults to %d for start operations.\n", + CLIENT_KNOBS->BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC); printf(" --active_snapshot_interval DURATION\n" - " For modify operations, sets the desired interval for the backup's currently active snapshot, relative to the start of the snapshot.\n"); + " For modify operations, sets the desired interval for the backup's currently active " + "snapshot, relative to the start of the snapshot.\n"); printf(" --verify_uid UID\n" - " Specifies a UID to verify against the BackupUID of the running backup. If provided, the UID is verified in the same transaction\n" + " Specifies a UID to verify against the BackupUID of the running backup. If provided, the " + "UID is verified in the same transaction\n" " which sets the new backup parameters (if the UID matches).\n"); printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n"); printf(" -k KEYS List of key ranges to backup.\n" - " If not specified, the entire database will be backed up.\n"); + " If not specified, the entire database will be backed up.\n"); printf(" -n, --dryrun For backup start or restore start, performs a trial run with no actual changes made.\n"); printf(" --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifes the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n"); + " --logdir PATH Specifes the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); printf(" --loggroup LOG_GROUP\n" " Sets the LogGroup field with the specified value for all\n" " events in the trace output (defaults to `default').\n"); printf(" --trace_format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); printf(" --max_cleanup_seconds SECONDS\n" " Specifies the amount of time a backup or DR needs to be stale before cleanup will\n" " remove mutations for it. By default this is set to one hour.\n"); printf(" --delete_data\n" - " This flag will cause cleanup to remove mutations for the most stale backup or DR.\n"); + " This flag will cause cleanup to remove mutations for the most stale backup or DR.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif printf(" -v, --version Print version information and exit.\n"); printf(" -w, --wait Wait for the backup to complete (allowed with `start' and `discontinue').\n"); printf(" -z, --no-stop-when-done\n" - " Do not stop backup when restorable.\n"); + " Do not stop backup when restorable.\n"); printf(" -h, --help Display this help and exit.\n"); if (devhelp) { @@ -980,20 +1045,19 @@ static void printBackupUsage(bool devhelp) { printf(" Specify a process after whose termination to exit.\n"); #endif printf(" --deep For describe operations, do not use cached metadata. Warning: Very slow\n"); - } printf("\n" - " KEYS FORMAT: \" \" [...]\n"); + " KEYS FORMAT: \" \" [...]\n"); printf("\n"); puts(BlobCredentialInfo); return; } -static void printRestoreUsage(bool devhelp ) { +static void printRestoreUsage(bool devhelp) { printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); printf("Usage: %s (start | status | abort | wait) [OPTIONS]\n\n", exeRestore.toString().c_str()); - //printf(" FOLDERS Paths to folders containing the backup files.\n"); + // printf(" FOLDERS Paths to folders containing the backup files.\n"); printf("Options for all commands:\n\n"); printf(" --dest_cluster_file CONNFILE\n"); printf(" The cluster file to restore data into.\n"); @@ -1011,27 +1075,30 @@ static void printRestoreUsage(bool devhelp ) { printf(" Prefix to add to the restored keys\n"); printf(" -n, --dryrun Perform a trial run with no changes made.\n"); printf(" --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifes the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n"); + " --logdir PATH Specifes the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); printf(" --loggroup LOG_GROUP\n" " Sets the LogGroup field with the specified value for all\n" " events in the trace output (defaults to `default').\n"); printf(" --trace_format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif printf(" -v DBVERSION The version at which the database will be restored.\n"); - printf(" --timestamp Instead of a numeric version, use this to specify a timestamp in %s\n", BackupAgentBase::timeFormat().c_str()); - printf(" and it will be converted to a version from that time using metadata in orig_cluster_file.\n"); + printf(" --timestamp Instead of a numeric version, use this to specify a timestamp in %s\n", + BackupAgentBase::timeFormat().c_str()); + printf( + " and it will be converted to a version from that time using metadata in orig_cluster_file.\n"); printf(" --orig_cluster_file CONNFILE\n"); - printf(" The cluster file for the original database from which the backup was created. The original database\n"); + printf(" The cluster file for the original database from which the backup was created. The " + "original database\n"); printf(" is only needed to convert a --timestamp argument to a database version.\n"); printf(" -h, --help Display this help and exit.\n"); - if( devhelp ) { + if (devhelp) { #ifdef _WIN32 printf(" -q Disable error dialog on crash.\n"); printf(" --parentpid PID\n"); @@ -1040,7 +1107,7 @@ static void printRestoreUsage(bool devhelp ) { } printf("\n" - " KEYS FORMAT: \" \" [...]\n"); + " KEYS FORMAT: \" \" [...]\n"); printf("\n"); puts(BlobCredentialInfo); @@ -1051,22 +1118,22 @@ static void printDBAgentUsage(bool devhelp) { printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); printf("Usage: %s [OPTIONS]\n\n", exeDatabaseAgent.toString().c_str()); printf(" -d CONNFILE The path of a file containing the connection string for the\n" - " destination FoundationDB cluster.\n"); + " destination FoundationDB cluster.\n"); printf(" -s CONNFILE The path of a file containing the connection string for the\n" - " source FoundationDB cluster.\n"); + " source FoundationDB cluster.\n"); printf(" --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifes the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n"); + " --logdir PATH Specifes the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); printf(" --loggroup LOG_GROUP\n" " Sets the LogGroup field with the specified value for all\n" " events in the trace output (defaults to `default').\n"); printf(" --trace_format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); printf(" -m SIZE, --memory SIZE\n" - " Memory limit. The default value is 8GiB. When specified\n" - " without a unit, MiB is assumed.\n"); + " Memory limit. The default value is 8GiB. When specified\n" + " without a unit, MiB is assumed.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif @@ -1086,34 +1153,35 @@ static void printDBAgentUsage(bool devhelp) { static void printDBBackupUsage(bool devhelp) { printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); - printf("Usage: %s (start | status | switch | abort | pause | resume) [OPTIONS]\n\n", exeDatabaseBackup.toString().c_str()); + printf("Usage: %s (start | status | switch | abort | pause | resume) [OPTIONS]\n\n", + exeDatabaseBackup.toString().c_str()); printf(" -d, --destination CONNFILE\n" " The path of a file containing the connection string for the\n"); printf(" destination FoundationDB cluster.\n"); printf(" -s, --source CONNFILE\n" " The path of a file containing the connection string for the\n" - " source FoundationDB cluster.\n"); + " source FoundationDB cluster.\n"); printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n"); printf(" -k KEYS List of key ranges to backup.\n" - " If not specified, the entire database will be backed up.\n"); + " If not specified, the entire database will be backed up.\n"); printf(" --cleanup Abort will attempt to stop mutation logging on the source cluster.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif printf(" --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifes the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n"); + " --logdir PATH Specifes the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); printf(" --loggroup LOG_GROUP\n" " Sets the LogGroup field with the specified value for all\n" " events in the trace output (defaults to `default').\n"); printf(" --trace_format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); printf(" -v, --version Print version information and exit.\n"); printf(" -h, --help Display this help and exit.\n"); printf("\n" - " KEYS FORMAT: \" \" [...]\n"); + " KEYS FORMAT: \" \" [...]\n"); if (devhelp) { #ifdef _WIN32 @@ -1127,11 +1195,9 @@ static void printDBBackupUsage(bool devhelp) { return; } -static void printUsage(enumProgramExe programExe, bool devhelp) -{ +static void printUsage(enumProgramExe programExe, bool devhelp) { - switch (programExe) - { + switch (programExe) { case EXE_AGENT: printAgentUsage(devhelp); break; @@ -1158,9 +1224,8 @@ static void printUsage(enumProgramExe programExe, bool devhelp) extern bool g_crashOnError; // Return the type of program executable based on the name of executable file -enumProgramExe getProgramType(std::string programExe) -{ - enumProgramExe enProgramExe = EXE_UNDEFINED; +enumProgramExe getProgramType(std::string programExe) { + enumProgramExe enProgramExe = EXE_UNDEFINED; // lowercase the string std::transform(programExe.begin(), programExe.end(), programExe.begin(), ::tolower); @@ -1172,64 +1237,63 @@ enumProgramExe getProgramType(std::string programExe) size_t lastSlash = programExe.find_last_of("\\"); // Ensure last dot is after last slash, if present - if ((lastSlash == std::string::npos)|| - (lastSlash < lastDot) ) - { + if ((lastSlash == std::string::npos) || (lastSlash < lastDot)) { programExe = programExe.substr(0, lastDot); } } #endif // For debugging convenience, remove .debug suffix if present. - if(StringRef(programExe).endsWith(LiteralStringRef(".debug"))) + if (StringRef(programExe).endsWith(LiteralStringRef(".debug"))) programExe = programExe.substr(0, programExe.size() - 6); // Check if backup agent - if ((programExe.length() >= exeAgent.size()) && - (programExe.compare(programExe.length()-exeAgent.size(), exeAgent.size(), (const char*) exeAgent.begin()) == 0) ) - { + if ((programExe.length() >= exeAgent.size()) && + (programExe.compare(programExe.length() - exeAgent.size(), exeAgent.size(), (const char*)exeAgent.begin()) == + 0)) { enProgramExe = EXE_AGENT; } // Check if backup - else if ((programExe.length() >= exeBackup.size()) && - (programExe.compare(programExe.length() - exeBackup.size(), exeBackup.size(), (const char*)exeBackup.begin()) == 0)) - { + else if ((programExe.length() >= exeBackup.size()) && + (programExe.compare( + programExe.length() - exeBackup.size(), exeBackup.size(), (const char*)exeBackup.begin()) == 0)) { enProgramExe = EXE_BACKUP; } // Check if restore - else if ((programExe.length() >= exeRestore.size()) && - (programExe.compare(programExe.length() - exeRestore.size(), exeRestore.size(), (const char*)exeRestore.begin()) == 0)) - { + else if ((programExe.length() >= exeRestore.size()) && + (programExe.compare( + programExe.length() - exeRestore.size(), exeRestore.size(), (const char*)exeRestore.begin()) == 0)) { enProgramExe = EXE_RESTORE; } // Check if db agent - else if ((programExe.length() >= exeDatabaseAgent.size()) && - (programExe.compare(programExe.length() - exeDatabaseAgent.size(), exeDatabaseAgent.size(), (const char*)exeDatabaseAgent.begin()) == 0)) - { + else if ((programExe.length() >= exeDatabaseAgent.size()) && + (programExe.compare(programExe.length() - exeDatabaseAgent.size(), + exeDatabaseAgent.size(), + (const char*)exeDatabaseAgent.begin()) == 0)) { enProgramExe = EXE_DR_AGENT; } // Check if db backup - else if ((programExe.length() >= exeDatabaseBackup.size()) && - (programExe.compare(programExe.length() - exeDatabaseBackup.size(), exeDatabaseBackup.size(), (const char*)exeDatabaseBackup.begin()) == 0)) - { + else if ((programExe.length() >= exeDatabaseBackup.size()) && + (programExe.compare(programExe.length() - exeDatabaseBackup.size(), + exeDatabaseBackup.size(), + (const char*)exeDatabaseBackup.begin()) == 0)) { enProgramExe = EXE_DB_BACKUP; } return enProgramExe; } -enumBackupType getBackupType(std::string backupType) -{ - enumBackupType enBackupType = BACKUP_UNDEFINED; +enumBackupType getBackupType(std::string backupType) { + enumBackupType enBackupType = BACKUP_UNDEFINED; // lowercase the string std::transform(backupType.begin(), backupType.end(), backupType.begin(), ::tolower); static std::map values; - if(values.empty()) { + if (values.empty()) { values["start"] = BACKUP_START; values["status"] = BACKUP_STATUS; values["abort"] = BACKUP_ABORT; @@ -1247,29 +1311,32 @@ enumBackupType getBackupType(std::string backupType) } auto i = values.find(backupType); - if(i != values.end()) + if (i != values.end()) enBackupType = i->second; return enBackupType; } enumRestoreType getRestoreType(std::string name) { - if(name == "start") return RESTORE_START; - if(name == "abort") return RESTORE_ABORT; - if(name == "status") return RESTORE_STATUS; - if(name == "wait") return RESTORE_WAIT; + if (name == "start") + return RESTORE_START; + if (name == "abort") + return RESTORE_ABORT; + if (name == "status") + return RESTORE_STATUS; + if (name == "wait") + return RESTORE_WAIT; return RESTORE_UNKNOWN; } -enumDBType getDBType(std::string dbType) -{ +enumDBType getDBType(std::string dbType) { enumDBType enBackupType = DB_UNDEFINED; // lowercase the string std::transform(dbType.begin(), dbType.end(), dbType.begin(), ::tolower); static std::map values; - if(values.empty()) { + if (values.empty()) { values["start"] = DB_START; values["status"] = DB_STATUS; values["switch"] = DB_SWITCH; @@ -1279,13 +1346,18 @@ enumDBType getDBType(std::string dbType) } auto i = values.find(dbType); - if(i != values.end()) + if (i != values.end()) enBackupType = i->second; return enBackupType; } -ACTOR Future getLayerStatus(Reference tr, std::string name, std::string id, enumProgramExe exe, Database dest, bool snapshot = false) { +ACTOR Future getLayerStatus(Reference tr, + std::string name, + std::string id, + enumProgramExe exe, + Database dest, + bool snapshot = false) { // This process will write a document that looks like this: // { backup : { $expires : {}, version: } // so that the value under 'backup' will eventually expire to null and thus be ignored by @@ -1294,9 +1366,9 @@ ACTOR Future getLayerStatus(Reference tr state Version readVer = wait(tr->getReadVersion()); - state json_spirit::mValue layersRootValue; // Will contain stuff that goes into the doc at the layers status root - JSONDoc layersRoot(layersRootValue); // Convenient mutator / accessor for the layers root - JSONDoc op = layersRoot.subDoc(name); // Operator object for the $expires operation + state json_spirit::mValue layersRootValue; // Will contain stuff that goes into the doc at the layers status root + JSONDoc layersRoot(layersRootValue); // Convenient mutator / accessor for the layers root + JSONDoc op = layersRoot.subDoc(name); // Operator object for the $expires operation // Create the $expires key which is where the rest of the status output will go state JSONDoc layerRoot = op.subDoc("$expires"); @@ -1309,15 +1381,15 @@ ACTOR Future getLayerStatus(Reference tr state JSONDoc o = layerRoot.subDoc("instances." + id); o.create("version") = FDB_VT_VERSION; - o.create("id") = id; + o.create("id") = id; o.create("last_updated") = now(); - o.create("memory_usage") = (int64_t)getMemoryUsage(); + o.create("memory_usage") = (int64_t)getMemoryUsage(); o.create("resident_size") = (int64_t)getResidentMemoryUsage(); o.create("main_thread_cpu_seconds") = getProcessorTimeThread(); - o.create("process_cpu_seconds") = getProcessorTimeProcess(); + o.create("process_cpu_seconds") = getProcessorTimeProcess(); o.create("configured_workers") = CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT; - if(exe == EXE_AGENT) { + if (exe == EXE_AGENT) { static BlobStoreEndpoint::Stats last_stats; static double last_ts = 0; BlobStoreEndpoint::Stats current_stats = BlobStoreEndpoint::s_stats; @@ -1325,14 +1397,14 @@ ACTOR Future getLayerStatus(Reference tr blobstats.create("total") = current_stats.getJSON(); BlobStoreEndpoint::Stats diff = current_stats - last_stats; json_spirit::mObject diffObj = diff.getJSON(); - if(last_ts > 0) + if (last_ts > 0) diffObj["bytes_per_second"] = double(current_stats.bytes_sent - last_stats.bytes_sent) / (now() - last_ts); blobstats.create("recent") = diffObj; last_stats = current_stats; last_ts = now(); JSONDoc totalBlobStats = layerRoot.subDoc("blob_recent_io"); - for(auto &p : diffObj) + for (auto& p : diffObj) totalBlobStats.create(p.first + ".$sum") = p.second; state FileBackupAgent fba; @@ -1360,19 +1432,22 @@ ACTOR Future getLayerStatus(Reference tr tagLastRestorableVersions.push_back(fba.getLastRestorable(tr, StringRef(tag->tagName), snapshot)); } - wait( waitForAll(tagLastRestorableVersions) && waitForAll(tagStates) && waitForAll(tagContainers) && waitForAll(tagRangeBytes) && waitForAll(tagLogBytes) && success(fBackupPaused)); + wait(waitForAll(tagLastRestorableVersions) && waitForAll(tagStates) && waitForAll(tagContainers) && + waitForAll(tagRangeBytes) && waitForAll(tagLogBytes) && success(fBackupPaused)); JSONDoc tagsRoot = layerRoot.subDoc("tags.$latest"); layerRoot.create("tags.timestamp") = now(); - layerRoot.create("total_workers.$sum") = fBackupPaused.get().present() ? 0 : CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT; + layerRoot.create("total_workers.$sum") = + fBackupPaused.get().present() ? 0 : CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT; layerRoot.create("paused.$latest") = fBackupPaused.get().present(); int j = 0; for (KeyBackedTag eachTag : backupTags) { Version last_restorable_version = tagLastRestorableVersions[j].get(); - double last_restorable_seconds_behind = ((double)readVer - last_restorable_version) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND; + double last_restorable_seconds_behind = + ((double)readVer - last_restorable_version) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND; BackupAgentBase::enumState status = (BackupAgentBase::enumState)tagStates[j].get(); - const char *statusText = fba.getStateText(status); + const char* statusText = fba.getStateText(status); // The object for this backup tag inside this instance's subdocument JSONDoc tagRoot = tagsRoot.subDoc(eachTag.tagName); @@ -1380,7 +1455,8 @@ ACTOR Future getLayerStatus(Reference tr tagRoot.create("current_status") = statusText; tagRoot.create("last_restorable_version") = tagLastRestorableVersions[j].get(); tagRoot.create("last_restorable_seconds_behind") = last_restorable_seconds_behind; - tagRoot.create("running_backup") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || status == BackupAgentBase::STATE_RUNNING); + tagRoot.create("running_backup") = + (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || status == BackupAgentBase::STATE_RUNNING); tagRoot.create("running_backup_is_restorable") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL); tagRoot.create("range_bytes_written") = tagRangeBytes[j].get(); tagRoot.create("mutation_log_bytes_written") = tagLogBytes[j].get(); @@ -1388,8 +1464,7 @@ ACTOR Future getLayerStatus(Reference tr j++; } - } - else if(exe == EXE_DR_AGENT) { + } else if (exe == EXE_DR_AGENT) { state DatabaseBackupAgent dba; state Reference tr2(new ReadYourWritesTransaction(dest)); tr2->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); @@ -1402,7 +1477,7 @@ ACTOR Future getLayerStatus(Reference tr state Future> fDRPaused = tr->get(dba.taskBucket->getPauseKey(), snapshot); state std::vector drTagUids; - for(int i = 0; i < tagNames.size(); i++) { + for (int i = 0; i < tagNames.size(); i++) { backupVersion.push_back(tr2->get(tagNames[i].value.withPrefix(applyMutationsBeginRange.begin), snapshot)); UID tagUID = BinaryReader::fromStringRef(tagNames[i].value, Unversioned()); drTagUids.push_back(tagUID); @@ -1411,7 +1486,8 @@ ACTOR Future getLayerStatus(Reference tr tagLogBytesDR.push_back(dba.getLogBytesWritten(tr2, tagUID, snapshot)); } - wait(waitForAll(backupStatus) && waitForAll(backupVersion) && waitForAll(tagRangeBytesDR) && waitForAll(tagLogBytesDR) && success(fDRPaused)); + wait(waitForAll(backupStatus) && waitForAll(backupVersion) && waitForAll(tagRangeBytesDR) && + waitForAll(tagLogBytesDR) && success(fDRPaused)); JSONDoc tagsRoot = layerRoot.subDoc("tags.$latest"); layerRoot.create("tags.timestamp") = now(); @@ -1424,14 +1500,17 @@ ACTOR Future getLayerStatus(Reference tr BackupAgentBase::enumState status = (BackupAgentBase::enumState)backupStatus[i].get(); JSONDoc tagRoot = tagsRoot.create(tagName); - tagRoot.create("running_backup") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || status == BackupAgentBase::STATE_RUNNING); + tagRoot.create("running_backup") = + (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || status == BackupAgentBase::STATE_RUNNING); tagRoot.create("running_backup_is_restorable") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL); tagRoot.create("range_bytes_written") = tagRangeBytesDR[i].get(); tagRoot.create("mutation_log_bytes_written") = tagLogBytesDR[i].get(); tagRoot.create("mutation_stream_id") = drTagUids[i].toString(); if (backupVersion[i].get().present()) { - double seconds_behind = ((double)readVer - BinaryReader::fromStringRef(backupVersion[i].get().get(), Unversioned())) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND; + double seconds_behind = ((double)readVer - BinaryReader::fromStringRef( + backupVersion[i].get().get(), Unversioned())) / + CLIENT_KNOBS->CORE_VERSIONSPERSECOND; tagRoot.create("seconds_behind") = seconds_behind; //TraceEvent("BackupMetrics").detail("SecondsBehind", seconds_behind); } @@ -1447,11 +1526,15 @@ ACTOR Future getLayerStatus(Reference tr // Check for unparseable or expired statuses and delete them. // First checks the first doc in the key range, and if it is valid, alive and not "me" then // returns. Otherwise, checks the rest of the range as well. -ACTOR Future cleanupStatus(Reference tr, std::string rootKey, std::string name, std::string id, int limit = 1) { +ACTOR Future cleanupStatus(Reference tr, + std::string rootKey, + std::string name, + std::string id, + int limit = 1) { state Standalone docs = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, true)); state bool readMore = false; state int i; - for(i = 0; i < docs.size(); ++i) { + for (i = 0; i < docs.size(); ++i) { json_spirit::mValue docValue; try { json_spirit::read_string(docs[i].value.toString(), docValue); @@ -1460,22 +1543,22 @@ ACTOR Future cleanupStatus(Reference tr, std::s JSONDoc::expires_reference_version = tr->getReadVersion().get(); // Evaluate the operators in the document, which will reduce to nothing if it is expired. doc.cleanOps(); - if(!doc.has(name + ".last_updated")) + if (!doc.has(name + ".last_updated")) throw Error(); // Alive and valid. // If limit == 1 and id is present then read more - if(limit == 1 && doc.has(name + ".instances." + id)) + if (limit == 1 && doc.has(name + ".instances." + id)) readMore = true; - } catch(Error &e) { + } catch (Error& e) { // If doc can't be parsed or isn't alive, delete it. TraceEvent(SevWarn, "RemovedDeadBackupLayerStatus").detail("Key", docs[i].key); tr->clear(docs[i].key); // If limit is 1 then read more. - if(limit == 1) + if (limit == 1) readMore = true; } - if(readMore) { + if (readMore) { limit = 10000; Standalone docs2 = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, true)); docs = std::move(docs2); @@ -1494,10 +1577,11 @@ ACTOR Future getLayerStatus(Database src, std::string root try { tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr.setOption(FDBTransactionOptions::LOCK_AWARE); - state Standalone kvPairs = wait(tr.getRange(KeyRangeRef(rootKey, strinc(rootKey)), CLIENT_KNOBS->ROW_LIMIT_UNLIMITED)); + state Standalone kvPairs = + wait(tr.getRange(KeyRangeRef(rootKey, strinc(rootKey)), CLIENT_KNOBS->ROW_LIMIT_UNLIMITED)); json_spirit::mObject statusDoc; JSONDoc modifier(statusDoc); - for(auto &kv : kvPairs) { + for (auto& kv : kvPairs) { json_spirit::mValue docValue; json_spirit::read_string(kv.value.toString(), docValue); modifier.absorb(docValue); @@ -1505,34 +1589,39 @@ ACTOR Future getLayerStatus(Database src, std::string root JSONDoc::expires_reference_version = (uint64_t)tr.getReadVersion().get(); modifier.cleanOps(); return statusDoc; - } - catch (Error& e) { + } catch (Error& e) { wait(tr.onError(e)); } } } -// Read layer status for this layer and get the total count of agent processes (instances) then adjust the poll delay based on that and BACKUP_AGGREGATE_POLL_RATE -ACTOR Future updateAgentPollRate(Database src, std::string rootKey, std::string name, double *pollDelay) { +// Read layer status for this layer and get the total count of agent processes (instances) then adjust the poll delay +// based on that and BACKUP_AGGREGATE_POLL_RATE +ACTOR Future updateAgentPollRate(Database src, std::string rootKey, std::string name, double* pollDelay) { loop { try { json_spirit::mObject status = wait(getLayerStatus(src, rootKey)); int64_t processes = 0; // If instances count is present and greater than 0 then update pollDelay - if(JSONDoc(status).tryGet(name + ".instances_running", processes) && processes > 0) { + if (JSONDoc(status).tryGet(name + ".instances_running", processes) && processes > 0) { // The aggregate poll rate is the target poll rate for all agent processes in the cluster - // The poll rate (polls/sec) for a single processes is aggregate poll rate / processes, and pollDelay is the inverse of that + // The poll rate (polls/sec) for a single processes is aggregate poll rate / processes, and pollDelay is + // the inverse of that *pollDelay = (double)processes / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE; } - } catch(Error &e) { + } catch (Error& e) { TraceEvent(SevWarn, "BackupAgentPollRateUpdateError").error(e); } wait(delay(CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE_UPDATE_INTERVAL)); } } -ACTOR Future statusUpdateActor(Database statusUpdateDest, std::string name, enumProgramExe exe, double *pollDelay, Database taskDest = Database(), - std::string id = nondeterministicRandom()->randomUniqueID().toString()) { +ACTOR Future statusUpdateActor(Database statusUpdateDest, + std::string name, + enumProgramExe exe, + double* pollDelay, + Database taskDest = Database(), + std::string id = nondeterministicRandom()->randomUniqueID().toString()) { state std::string metaKey = layerStatusMetaPrefixRange.begin.toString() + "json/" + name; state std::string rootKey = backupStatusPrefixRange.begin.toString() + name + "/json"; state std::string instanceKey = rootKey + "/" + "agent-" + id; @@ -1547,8 +1636,7 @@ ACTOR Future statusUpdateActor(Database statusUpdateDest, std::string name tr->set(metaKey, rootKey); wait(tr->commit()); break; - } - catch (Error& e) { + } catch (Error& e) { wait(tr->onError(e)); } } @@ -1557,7 +1645,7 @@ ACTOR Future statusUpdateActor(Database statusUpdateDest, std::string name loop { tr->reset(); try { - loop{ + loop { try { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); @@ -1567,19 +1655,20 @@ ACTOR Future statusUpdateActor(Database statusUpdateDest, std::string name tr->set(instanceKey, statusdoc); wait(tr->commit()); break; - } - catch (Error& e) { + } catch (Error& e) { wait(tr->onError(e)); } } - wait(delay(CLIENT_KNOBS->BACKUP_STATUS_DELAY * ( ( 1.0 - CLIENT_KNOBS->BACKUP_STATUS_JITTER ) + 2 * deterministicRandom()->random01() * CLIENT_KNOBS->BACKUP_STATUS_JITTER ))); + wait(delay(CLIENT_KNOBS->BACKUP_STATUS_DELAY * + ((1.0 - CLIENT_KNOBS->BACKUP_STATUS_JITTER) + + 2 * deterministicRandom()->random01() * CLIENT_KNOBS->BACKUP_STATUS_JITTER))); - // Now that status was written at least once by this process (and hopefully others), start the poll rate control updater if it wasn't started yet - if(!pollRateUpdater.isValid() && pollDelay != nullptr) + // Now that status was written at least once by this process (and hopefully others), start the poll rate + // control updater if it wasn't started yet + if (!pollRateUpdater.isValid() && pollDelay != nullptr) pollRateUpdater = updateAgentPollRate(statusUpdateDest, rootKey, name, pollDelay); - } - catch (Error& e) { + } catch (Error& e) { TraceEvent(SevWarnAlways, "UnableToWriteStatus").error(e); wait(delay(10.0)); } @@ -1598,15 +1687,14 @@ ACTOR Future runDBAgent(Database src, Database dest) { try { wait(backupAgent.run(dest, &pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT)); break; - } - catch (Error& e) { + } catch (Error& e) { if (e.code() == error_code_operation_cancelled) throw; TraceEvent(SevError, "DA_runAgent").error(e); fprintf(stderr, "ERROR: DR agent encountered fatal error `%s'\n", e.what()); - wait( delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY) ); + wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY)); } } @@ -1623,24 +1711,25 @@ ACTOR Future runAgent(Database db) { try { wait(backupAgent.run(db, &pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT)); break; - } - catch (Error& e) { + } catch (Error& e) { if (e.code() == error_code_operation_cancelled) throw; TraceEvent(SevError, "BA_runAgent").error(e); fprintf(stderr, "ERROR: backup agent encountered fatal error `%s'\n", e.what()); - wait( delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY) ); + wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY)); } } return Void(); } -ACTOR Future submitDBBackup(Database src, Database dest, Standalone> backupRanges, std::string tagName) { - try - { +ACTOR Future submitDBBackup(Database src, + Database dest, + Standalone> backupRanges, + std::string tagName) { + try { state DatabaseBackupAgent backupAgent(src); // Backup everything, if no ranges were specified @@ -1648,36 +1737,34 @@ ACTOR Future submitDBBackup(Database src, Database dest, Standalone submitDBBackup(Database src, Database dest, Standalone submitBackup(Database db, std::string url, int snapshotIntervalSeconds, Standalone> backupRanges, std::string tagName, bool dryRun, bool waitForCompletion, bool stopWhenDone) { - try - { +ACTOR Future submitBackup(Database db, + std::string url, + int snapshotIntervalSeconds, + Standalone> backupRanges, + std::string tagName, + bool dryRun, + bool waitForCompletion, + bool stopWhenDone) { + try { state FileBackupAgent backupAgent; // Backup everything, if no ranges were specified @@ -1712,7 +1805,8 @@ ACTOR Future submitBackup(Database db, std::string url, int snapshotInterv } if (waitForCompletion) { - printf("Submitted and now waiting for the backup on tag `%s' to complete. (DRY RUN)\n", printable(StringRef(tagName)).c_str()); + printf("Submitted and now waiting for the backup on tag `%s' to complete. (DRY RUN)\n", + printable(StringRef(tagName)).c_str()); } else { @@ -1720,54 +1814,56 @@ ACTOR Future submitBackup(Database db, std::string url, int snapshotInterv bool agentRunning = wait(backupAgent.checkActive(db)); if (!agentRunning) { - printf("The backup on tag `%s' was successfully submitted but no backup agents are responding. (DRY RUN)\n", printable(StringRef(tagName)).c_str()); + printf("The backup on tag `%s' was successfully submitted but no backup agents are responding. " + "(DRY RUN)\n", + printable(StringRef(tagName)).c_str()); // Throw an error that will not display any additional information throw actor_cancelled(); - } - else { - printf("The backup on tag `%s' was successfully submitted. (DRY RUN)\n", printable(StringRef(tagName)).c_str()); + } else { + printf("The backup on tag `%s' was successfully submitted. (DRY RUN)\n", + printable(StringRef(tagName)).c_str()); } } } else { - wait(backupAgent.submitBackup(db, KeyRef(url), snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone)); + wait(backupAgent.submitBackup( + db, KeyRef(url), snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone)); // Wait for the backup to complete, if requested if (waitForCompletion) { - printf("Submitted and now waiting for the backup on tag `%s' to complete.\n", printable(StringRef(tagName)).c_str()); + printf("Submitted and now waiting for the backup on tag `%s' to complete.\n", + printable(StringRef(tagName)).c_str()); wait(success(backupAgent.waitBackup(db, tagName))); - } - else { + } else { // Check if a backup agent is running bool agentRunning = wait(backupAgent.checkActive(db)); if (!agentRunning) { - printf("The backup on tag `%s' was successfully submitted but no backup agents are responding.\n", printable(StringRef(tagName)).c_str()); + printf("The backup on tag `%s' was successfully submitted but no backup agents are responding.\n", + printable(StringRef(tagName)).c_str()); // Throw an error that will not display any additional information throw actor_cancelled(); - } - else { - printf("The backup on tag `%s' was successfully submitted.\n", printable(StringRef(tagName)).c_str()); + } else { + printf("The backup on tag `%s' was successfully submitted.\n", + printable(StringRef(tagName)).c_str()); } } } - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; - switch (e.code()) - { - case error_code_backup_error: - fprintf(stderr, "ERROR: An error was encountered during submission\n"); + switch (e.code()) { + case error_code_backup_error: + fprintf(stderr, "ERROR: An error was encountered during submission\n"); break; - case error_code_backup_duplicate: - fprintf(stderr, "ERROR: A backup is already running on tag `%s'\n", printable(StringRef(tagName)).c_str()); + case error_code_backup_duplicate: + fprintf(stderr, "ERROR: A backup is already running on tag `%s'\n", printable(StringRef(tagName)).c_str()); break; - default: - fprintf(stderr, "ERROR: %s\n", e.what()); + default: + fprintf(stderr, "ERROR: %s\n", e.what()); break; } @@ -1777,9 +1873,12 @@ ACTOR Future submitBackup(Database db, std::string url, int snapshotInterv return Void(); } -ACTOR Future switchDBBackup(Database src, Database dest, Standalone> backupRanges, std::string tagName, bool forceAction) { - try - { +ACTOR Future switchDBBackup(Database src, + Database dest, + Standalone> backupRanges, + std::string tagName, + bool forceAction) { + try { state DatabaseBackupAgent backupAgent(src); // Backup everything, if no ranges were specified @@ -1787,24 +1886,22 @@ ACTOR Future switchDBBackup(Database src, Database dest, Standalone switchDBBackup(Database src, Database dest, Standalone statusDBBackup(Database src, Database dest, std::string tagName, int errorLimit) { - try - { + try { state DatabaseBackupAgent backupAgent(src); - std::string statusText = wait(backupAgent.getStatus(dest, errorLimit, StringRef(tagName))); + std::string statusText = wait(backupAgent.getStatus(dest, errorLimit, StringRef(tagName))); printf("%s\n", statusText.c_str()); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -1833,15 +1928,14 @@ ACTOR Future statusDBBackup(Database src, Database dest, std::string tagNa } ACTOR Future statusBackup(Database db, std::string tagName, bool showErrors, bool json) { - try - { + try { state FileBackupAgent backupAgent; - std::string statusText = wait(json ? backupAgent.getStatusJSON(db, tagName) : backupAgent.getStatus(db, showErrors, tagName)); + std::string statusText = + wait(json ? backupAgent.getStatusJSON(db, tagName) : backupAgent.getStatus(db, showErrors, tagName)); printf("%s\n", statusText.c_str()); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -1851,28 +1945,25 @@ ACTOR Future statusBackup(Database db, std::string tagName, bool showError } ACTOR Future abortDBBackup(Database src, Database dest, std::string tagName, bool partial) { - try - { + try { state DatabaseBackupAgent backupAgent(src); wait(backupAgent.abortBackup(dest, Key(tagName), partial)); wait(backupAgent.unlockBackup(dest, Key(tagName))); printf("The DR on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str()); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; - switch (e.code()) - { - case error_code_backup_error: - fprintf(stderr, "ERROR: An error was encountered during submission\n"); + switch (e.code()) { + case error_code_backup_error: + fprintf(stderr, "ERROR: An error was encountered during submission\n"); break; - case error_code_backup_unneeded: - fprintf(stderr, "ERROR: A DR was not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); + case error_code_backup_unneeded: + fprintf(stderr, "ERROR: A DR was not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); break; - default: - fprintf(stderr, "ERROR: %s\n", e.what()); + default: + fprintf(stderr, "ERROR: %s\n", e.what()); break; } throw; @@ -1882,27 +1973,24 @@ ACTOR Future abortDBBackup(Database src, Database dest, std::string tagNam } ACTOR Future abortBackup(Database db, std::string tagName) { - try - { + try { state FileBackupAgent backupAgent; wait(backupAgent.abortBackup(db, tagName)); printf("The backup on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str()); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; - switch (e.code()) - { - case error_code_backup_error: - fprintf(stderr, "ERROR: An error was encountered during submission\n"); + switch (e.code()) { + case error_code_backup_error: + fprintf(stderr, "ERROR: An error was encountered during submission\n"); break; - case error_code_backup_unneeded: - fprintf(stderr, "ERROR: A backup was not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); + case error_code_backup_unneeded: + fprintf(stderr, "ERROR: A backup was not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); break; - default: - fprintf(stderr, "ERROR: %s\n", e.what()); + default: + fprintf(stderr, "ERROR: %s\n", e.what()); break; } throw; @@ -1912,12 +2000,10 @@ ACTOR Future abortBackup(Database db, std::string tagName) { } ACTOR Future cleanupMutations(Database db, bool deleteData) { - try - { + try { wait(cleanupBackup(db, deleteData)); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -1927,17 +2013,16 @@ ACTOR Future cleanupMutations(Database db, bool deleteData) { } ACTOR Future waitBackup(Database db, std::string tagName, bool stopWhenDone) { - try - { + try { state FileBackupAgent backupAgent; int status = wait(backupAgent.waitBackup(db, tagName, stopWhenDone)); - printf("The backup on tag `%s' %s.\n", printable(StringRef(tagName)).c_str(), - BackupAgentBase::getStateText((BackupAgentBase::enumState) status)); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + printf("The backup on tag `%s' %s.\n", + printable(StringRef(tagName)).c_str(), + BackupAgentBase::getStateText((BackupAgentBase::enumState)status)); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -1947,38 +2032,37 @@ ACTOR Future waitBackup(Database db, std::string tagName, bool stopWhenDon } ACTOR Future discontinueBackup(Database db, std::string tagName, bool waitForCompletion) { - try - { + try { state FileBackupAgent backupAgent; wait(backupAgent.discontinueBackup(db, StringRef(tagName))); // Wait for the backup to complete, if requested if (waitForCompletion) { - printf("Discontinued and now waiting for the backup on tag `%s' to complete.\n", printable(StringRef(tagName)).c_str()); + printf("Discontinued and now waiting for the backup on tag `%s' to complete.\n", + printable(StringRef(tagName)).c_str()); wait(success(backupAgent.waitBackup(db, tagName))); - } - else { + } else { printf("The backup on tag `%s' was successfully discontinued.\n", printable(StringRef(tagName)).c_str()); } - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; - switch (e.code()) - { - case error_code_backup_error: - fprintf(stderr, "ERROR: An encounter was error during submission\n"); + switch (e.code()) { + case error_code_backup_error: + fprintf(stderr, "ERROR: An encounter was error during submission\n"); break; - case error_code_backup_unneeded: - fprintf(stderr, "ERROR: A backup in not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); + case error_code_backup_unneeded: + fprintf(stderr, "ERROR: A backup in not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); break; - case error_code_backup_duplicate: - fprintf(stderr, "ERROR: The backup on tag `%s' is already discontinued\n", printable(StringRef(tagName)).c_str()); + case error_code_backup_duplicate: + fprintf(stderr, + "ERROR: The backup on tag `%s' is already discontinued\n", + printable(StringRef(tagName)).c_str()); break; - default: - fprintf(stderr, "ERROR: %s\n", e.what()); + default: + fprintf(stderr, "ERROR: %s\n", e.what()); break; } throw; @@ -1992,9 +2076,8 @@ ACTOR Future changeBackupResumed(Database db, bool pause) { state FileBackupAgent backupAgent; wait(backupAgent.taskBucket->changePause(db, pause)); printf("All backup agents have been %s.\n", pause ? "paused" : "resumed"); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -2008,9 +2091,8 @@ ACTOR Future changeDBBackupResumed(Database src, Database dest, bool pause state DatabaseBackupAgent backupAgent(src); wait(backupAgent.taskBucket->changePause(dest, pause)); printf("All DR agents have been %s.\n", pause ? "paused" : "resumed"); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -2019,7 +2101,7 @@ ACTOR Future changeDBBackupResumed(Database src, Database dest, bool pause return Void(); } -Reference openBackupContainer(const char *name, std::string destinationContainer) { +Reference openBackupContainer(const char* name, std::string destinationContainer) { // Error, if no dest container was specified if (destinationContainer.empty()) { fprintf(stderr, "ERROR: No backup destination was specified.\n"); @@ -2030,10 +2112,9 @@ Reference openBackupContainer(const char *name, std::string de Reference c; try { c = IBackupContainer::openContainer(destinationContainer); - } - catch (Error& e) { + } catch (Error& e) { std::string msg = format("ERROR: '%s' on URL '%s'", e.what(), destinationContainer.c_str()); - if(e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { + if (e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { msg += format(": %s", IBackupContainer::lastOpenError.c_str()); } fprintf(stderr, "%s\n", msg.c_str()); @@ -2044,12 +2125,23 @@ Reference openBackupContainer(const char *name, std::string de return c; } -ACTOR Future runRestore(Database db, std::string originalClusterFile, std::string tagName, std::string container, Standalone> ranges, Version targetVersion, std::string targetTimestamp, bool performRestore, bool verbose, bool waitForDone, std::string addPrefix, std::string removePrefix) { - if(ranges.empty()) { +ACTOR Future runRestore(Database db, + std::string originalClusterFile, + std::string tagName, + std::string container, + Standalone> ranges, + Version targetVersion, + std::string targetTimestamp, + bool performRestore, + bool verbose, + bool waitForDone, + std::string addPrefix, + std::string removePrefix) { + if (ranges.empty()) { ranges.push_back_deep(ranges.arena(), normalKeys); } - if(targetVersion != invalidVersion && !targetTimestamp.empty()) { + if (targetVersion != invalidVersion && !targetTimestamp.empty()) { fprintf(stderr, "Restore target version and target timestamp cannot both be specified\n"); throw restore_error(); } @@ -2057,14 +2149,17 @@ ACTOR Future runRestore(Database db, std::string originalClusterFile, std: state Optional origDb; // Resolve targetTimestamp if given - if(!targetTimestamp.empty()) { - if(originalClusterFile.empty()) { - fprintf(stderr, "An original cluster file must be given in order to resolve restore target timestamp '%s'\n", targetTimestamp.c_str()); + if (!targetTimestamp.empty()) { + if (originalClusterFile.empty()) { + fprintf(stderr, + "An original cluster file must be given in order to resolve restore target timestamp '%s'\n", + targetTimestamp.c_str()); throw restore_error(); } - if(!fileExists(originalClusterFile)) { - fprintf(stderr, "Original source database cluster file '%s' does not exist.\n", originalClusterFile.c_str()); + if (!fileExists(originalClusterFile)) { + fprintf( + stderr, "Original source database cluster file '%s' does not exist.\n", originalClusterFile.c_str()); throw restore_error(); } @@ -2080,45 +2175,55 @@ ACTOR Future runRestore(Database db, std::string originalClusterFile, std: state Reference bc = openBackupContainer(exeRestore.toString().c_str(), container); // If targetVersion is unset then use the maximum restorable version from the backup description - if(targetVersion == invalidVersion) { - if(verbose) - printf("No restore target version given, will use maximum restorable version from backup description.\n"); + if (targetVersion == invalidVersion) { + if (verbose) + printf( + "No restore target version given, will use maximum restorable version from backup description.\n"); BackupDescription desc = wait(bc->describeBackup()); - if(!desc.maxRestorableVersion.present()) { + if (!desc.maxRestorableVersion.present()) { fprintf(stderr, "The specified backup is not restorable to any version.\n"); throw restore_error(); } targetVersion = desc.maxRestorableVersion.get(); - if(verbose) + if (verbose) printf("Using target restore version %" PRId64 "\n", targetVersion); } if (performRestore) { - Version restoredVersion = wait(backupAgent.restore(db, origDb, KeyRef(tagName), KeyRef(container), ranges, waitForDone, targetVersion, verbose, KeyRef(addPrefix), KeyRef(removePrefix))); + Version restoredVersion = wait(backupAgent.restore(db, + origDb, + KeyRef(tagName), + KeyRef(container), + ranges, + waitForDone, + targetVersion, + verbose, + KeyRef(addPrefix), + KeyRef(removePrefix))); - if(waitForDone && verbose) { + if (waitForDone && verbose) { // If restore is now complete then report version restored printf("Restored to version %" PRId64 "\n", restoredVersion); } - } - else { + } else { state Optional rset = wait(bc->getRestoreSet(targetVersion)); - if(!rset.present()) { - fprintf(stderr, "Insufficient data to restore to version %" PRId64 ". Describe backup for more information.\n", targetVersion); + if (!rset.present()) { + fprintf(stderr, + "Insufficient data to restore to version %" PRId64 ". Describe backup for more information.\n", + targetVersion); throw restore_invalid_version(); } printf("Backup can be used to restore to version %" PRId64 "\n", targetVersion); } - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -2127,22 +2232,25 @@ ACTOR Future runRestore(Database db, std::string originalClusterFile, std: return Void(); } -ACTOR Future dumpBackupData(const char *name, std::string destinationContainer, Version beginVersion, Version endVersion) { +ACTOR Future dumpBackupData(const char* name, + std::string destinationContainer, + Version beginVersion, + Version endVersion) { state Reference c = openBackupContainer(name, destinationContainer); - if(beginVersion < 0 || endVersion < 0) { + if (beginVersion < 0 || endVersion < 0) { BackupDescription desc = wait(c->describeBackup()); - if(!desc.maxLogEnd.present()) { + if (!desc.maxLogEnd.present()) { fprintf(stderr, "ERROR: Backup must have log data in order to use relative begin/end versions.\n"); throw backup_invalid_info(); } - if(beginVersion < 0) { + if (beginVersion < 0) { beginVersion += desc.maxLogEnd.get(); } - if(endVersion < 0) { + if (endVersion < 0) { endVersion += desc.maxLogEnd.get(); } } @@ -2154,21 +2262,29 @@ ACTOR Future dumpBackupData(const char *name, std::string destinationConta return Void(); } -ACTOR Future expireBackupData(const char *name, std::string destinationContainer, Version endVersion, std::string endDatetime, Database db, bool force, Version restorableAfterVersion, std::string restorableAfterDatetime) { +ACTOR Future expireBackupData(const char* name, + std::string destinationContainer, + Version endVersion, + std::string endDatetime, + Database db, + bool force, + Version restorableAfterVersion, + std::string restorableAfterDatetime) { if (!endDatetime.empty()) { - Version v = wait( timeKeeperVersionFromDatetime(endDatetime, db) ); + Version v = wait(timeKeeperVersionFromDatetime(endDatetime, db)); endVersion = v; } if (!restorableAfterDatetime.empty()) { - Version v = wait( timeKeeperVersionFromDatetime(restorableAfterDatetime, db) ); + Version v = wait(timeKeeperVersionFromDatetime(restorableAfterDatetime, db)); restorableAfterVersion = v; } if (endVersion == invalidVersion) { fprintf(stderr, "ERROR: No version or date/time is specified.\n"); printHelpTeaser(name); - throw backup_error();; + throw backup_error(); + ; } try { @@ -2182,32 +2298,34 @@ ACTOR Future expireBackupData(const char *name, std::string destinationCon choose { when(wait(delay(5))) { std::string p = progress.toString(); - if(p != lastProgress) { + if (p != lastProgress) { int spaces = lastProgress.size() - p.size(); - printf("\r%s%s", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "") ); + printf("\r%s%s", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "")); lastProgress = p; } } - when(wait(expire)) { - break; - } + when(wait(expire)) { break; } } } std::string p = progress.toString(); int spaces = lastProgress.size() - p.size(); - printf("\r%s%s\n", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "") ); + printf("\r%s%s\n", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "")); - if(endVersion < 0) - printf("All data before %" PRId64 " versions (%" PRId64 " days) prior to latest backup log has been deleted.\n", -endVersion, -endVersion / ((int64_t)24 * 3600 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); + if (endVersion < 0) + printf("All data before %" PRId64 " versions (%" PRId64 + " days) prior to latest backup log has been deleted.\n", + -endVersion, + -endVersion / ((int64_t)24 * 3600 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); else printf("All data before version %" PRId64 " has been deleted.\n", endVersion); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; - if(e.code() == error_code_backup_cannot_expire) - fprintf(stderr, "ERROR: Requested expiration would be unsafe. Backup would not meet minimum restorability. Use --force to delete data anyway.\n"); + if (e.code() == error_code_backup_cannot_expire) + fprintf(stderr, + "ERROR: Requested expiration would be unsafe. Backup would not meet minimum " + "restorability. Use --force to delete data anyway.\n"); else fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -2216,7 +2334,7 @@ ACTOR Future expireBackupData(const char *name, std::string destinationCon return Void(); } -ACTOR Future deleteBackupContainer(const char *name, std::string destinationContainer) { +ACTOR Future deleteBackupContainer(const char* name, std::string destinationContainer) { try { state Reference c = openBackupContainer(name, destinationContainer); state int numDeleted = 0; @@ -2227,11 +2345,9 @@ ACTOR Future deleteBackupContainer(const char *name, std::string destinati loop { choose { - when ( wait(done) ) { - break; - } - when ( wait(delay(5)) ) { - if(numDeleted != lastUpdate) { + when(wait(done)) { break; } + when(wait(delay(5))) { + if (numDeleted != lastUpdate) { printf("\r%d...", numDeleted); lastUpdate = numDeleted; } @@ -2240,9 +2356,8 @@ ACTOR Future deleteBackupContainer(const char *name, std::string destinati } printf("\r%d objects deleted\n", numDeleted); printf("The entire container has been deleted.\n"); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -2251,16 +2366,19 @@ ACTOR Future deleteBackupContainer(const char *name, std::string destinati return Void(); } -ACTOR Future describeBackup(const char *name, std::string destinationContainer, bool deep, Optional cx, bool json) { +ACTOR Future describeBackup(const char* name, + std::string destinationContainer, + bool deep, + Optional cx, + bool json) { try { Reference c = openBackupContainer(name, destinationContainer); state BackupDescription desc = wait(c->describeBackup(deep)); - if(cx.present()) + if (cx.present()) wait(desc.resolveVersionTimes(cx.get())); printf("%s\n", (json ? desc.toJSON() : desc.toString()).c_str()); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -2275,10 +2393,9 @@ ACTOR Future listBackup(std::string baseUrl) { for (std::string container : containers) { printf("%s\n", container.c_str()); } - } - catch (Error& e) { + } catch (Error& e) { std::string msg = format("ERROR: %s", e.what()); - if(e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { + if (e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { msg += format(": %s", IBackupContainer::lastOpenError.c_str()); } fprintf(stderr, "%s\n", msg.c_str()); @@ -2299,7 +2416,7 @@ struct BackupModifyOptions { }; ACTOR Future modifyBackup(Database db, std::string tagName, BackupModifyOptions options) { - if(!options.hasChanges()) { + if (!options.hasChanges()) { fprintf(stderr, "No changes were specified, nothing to do!\n"); throw backup_error(); } @@ -2307,18 +2424,21 @@ ACTOR Future modifyBackup(Database db, std::string tagName, BackupModifyOp state KeyBackedTag tag = makeBackupTag(tagName); state Reference bc; - if(options.destURL.present()) { + if (options.destURL.present()) { bc = openBackupContainer(exeBackup.toString().c_str(), options.destURL.get()); try { wait(timeoutError(bc->create(), 30)); - } catch(Error &e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; - fprintf(stderr, "ERROR: Could not create backup container at '%s': %s\n", options.destURL.get().c_str(), e.what()); + fprintf(stderr, + "ERROR: Could not create backup container at '%s': %s\n", + options.destURL.get().c_str(), + e.what()); throw backup_error(); } } - + state Reference tr(new ReadYourWritesTransaction(db)); loop { try { @@ -2327,45 +2447,50 @@ ACTOR Future modifyBackup(Database db, std::string tagName, BackupModifyOp state Optional uidFlag = wait(tag.get(db)); - if(!uidFlag.present()) { + if (!uidFlag.present()) { fprintf(stderr, "No backup exists on tag '%s'\n", tagName.c_str()); throw backup_error(); } - if(uidFlag.get().second) { + if (uidFlag.get().second) { fprintf(stderr, "Cannot modify aborted backup on tag '%s'\n", tagName.c_str()); throw backup_error(); } state BackupConfig config(uidFlag.get().first); EBackupState s = wait(config.stateEnum().getOrThrow(tr, false, backup_invalid_info())); - if(!FileBackupAgent::isRunnable(s)) { + if (!FileBackupAgent::isRunnable(s)) { fprintf(stderr, "Backup on tag '%s' is not runnable.\n", tagName.c_str()); throw backup_error(); } - if(options.verifyUID.present() && options.verifyUID.get() != uidFlag.get().first.toString()) { - fprintf(stderr, "UID verification failed, backup on tag '%s' is '%s' but '%s' was specified.\n", tagName.c_str(), uidFlag.get().first.toString().c_str(), options.verifyUID.get().c_str()); + if (options.verifyUID.present() && options.verifyUID.get() != uidFlag.get().first.toString()) { + fprintf(stderr, + "UID verification failed, backup on tag '%s' is '%s' but '%s' was specified.\n", + tagName.c_str(), + uidFlag.get().first.toString().c_str(), + options.verifyUID.get().c_str()); throw backup_error(); } - if(options.snapshotIntervalSeconds.present()) { + if (options.snapshotIntervalSeconds.present()) { config.snapshotIntervalSeconds().set(tr, options.snapshotIntervalSeconds.get()); } - if(options.activeSnapshotIntervalSeconds.present()) { + if (options.activeSnapshotIntervalSeconds.present()) { Version begin = wait(config.snapshotBeginVersion().getOrThrow(tr, false, backup_error())); - config.snapshotTargetEndVersion().set(tr, begin + ((int64_t)options.activeSnapshotIntervalSeconds.get() * CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); + config.snapshotTargetEndVersion().set(tr, + begin + ((int64_t)options.activeSnapshotIntervalSeconds.get() * + CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); } - if(options.destURL.present()) { + if (options.destURL.present()) { config.backupContainer().set(tr, bc); } wait(tr->commit()); break; - } - catch (Error& e) { + } catch (Error& e) { wait(tr->onError(e)); } } @@ -2373,8 +2498,7 @@ ACTOR Future modifyBackup(Database db, std::string tagName, BackupModifyOp return Void(); } -static std::vector> parseLine(std::string &line, bool& err, bool& partial) -{ +static std::vector> parseLine(std::string& line, bool& err, bool& partial) { err = false; partial = false; @@ -2395,8 +2519,7 @@ static std::vector> parseLine(std::string &line, bool& er buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); ret.push_back(std::move(buf)); offset = i = line.find_first_not_of(' ', i + 1); - } - else + } else i++; break; case '"': @@ -2407,12 +2530,10 @@ static std::vector> parseLine(std::string &line, bool& er break; case ' ': if (!quoted) { - buf.push_back(StringRef((uint8_t *)(line.data() + offset), - i - offset)); + buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); offset = i = line.find_first_not_of(' ', i); forcetoken = false; - } - else + } else i++; break; case '\\': @@ -2435,7 +2556,7 @@ static std::vector> parseLine(std::string &line, bool& er ret.push_back(std::move(buf)); return ret; } - char *pEnd; + char* pEnd; save = line[i + 4]; line[i + 4] = 0; ent = char(strtoul(line.data() + i + 2, &pEnd, 16)); @@ -2469,48 +2590,47 @@ static std::vector> parseLine(std::string &line, bool& er return ret; } -static void addKeyRange(std::string optionValue, Standalone>& keyRanges) -{ - bool err = false, partial = false; - int tokenArray = 0, tokenIndex = 0; +static void addKeyRange(std::string optionValue, Standalone>& keyRanges) { + bool err = false, partial = false; + int tokenArray = 0, tokenIndex = 0; auto parsed = parseLine(optionValue, err, partial); - for (auto tokens : parsed) - { + for (auto tokens : parsed) { tokenArray++; tokenIndex = 0; /* for (auto token : tokens) { - tokenIndex++; + tokenIndex++; - printf("%4d token #%2d: %s\n", tokenArray, tokenIndex, printable(token).c_str()); + printf("%4d token #%2d: %s\n", tokenArray, tokenIndex, printable(token).c_str()); } */ // Process the keys // [end] - switch (tokens.size()) - { + switch (tokens.size()) { // empty case 0: break; // single key range case 1: - keyRanges.push_back_deep(keyRanges.arena(), KeyRangeRef(tokens.at(0), strinc(tokens.at(0)))); + keyRanges.push_back_deep(keyRanges.arena(), KeyRangeRef(tokens.at(0), strinc(tokens.at(0)))); break; // full key range case 2: try { keyRanges.push_back_deep(keyRanges.arena(), KeyRangeRef(tokens.at(0), tokens.at(1))); - } - catch (Error& e) { - fprintf(stderr, "ERROR: Invalid key range `%s %s' reported error %s\n", - tokens.at(0).toString().c_str(), tokens.at(1).toString().c_str(), e.what()); + } catch (Error& e) { + fprintf(stderr, + "ERROR: Invalid key range `%s %s' reported error %s\n", + tokens.at(0).toString().c_str(), + tokens.at(1).toString().c_str(), + e.what()); throw invalid_option_value(); } break; @@ -2526,12 +2646,12 @@ static void addKeyRange(std::string optionValue, StandaloneNext()) { lastError = args->LastError(); - switch (lastError) - { + switch (lastError) { case SO_SUCCESS: break; @@ -2838,284 +2944,278 @@ int main(int argc, char* argv[]) { int optId = args->OptionId(); switch (optId) { - case OPT_HELP: - printUsage(programExe, false); - return FDB_EXIT_SUCCESS; - break; - case OPT_DEVHELP: - printUsage(programExe, true); - return FDB_EXIT_SUCCESS; - break; - case OPT_VERSION: - printVersion(); - return FDB_EXIT_SUCCESS; - break; - case OPT_NOBUFSTDOUT: - setvbuf(stdout, NULL, _IONBF, 0); - setvbuf(stderr, NULL, _IONBF, 0); - break; - case OPT_BUFSTDOUTERR: - setvbuf(stdout, NULL, _IOFBF, BUFSIZ); - setvbuf(stderr, NULL, _IOFBF, BUFSIZ); - break; - case OPT_QUIET: - quietDisplay = true; - break; - case OPT_DRYRUN: - dryRun = true; - break; - case OPT_DELETE_DATA: - deleteData = true; - break; - case OPT_MIN_CLEANUP_SECONDS: - knobs.push_back( std::make_pair( "min_cleanup_seconds", args->OptionArg() ) ); - break; - case OPT_FORCE: - forceAction = true; - break; - case OPT_TRACE: - trace = true; - break; - case OPT_TRACE_DIR: - trace = true; - traceDir = args->OptionArg(); - break; - case OPT_TRACE_FORMAT: - if (!validateTraceFormat(args->OptionArg())) { - fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args->OptionArg()); - } - traceFormat = args->OptionArg(); - break; - case OPT_TRACE_LOG_GROUP: - traceLogGroup = args->OptionArg(); - break; - case OPT_LOCALITY: { - std::string syn = args->OptionSyntax(); - if (!StringRef(syn).startsWith(LiteralStringRef("--locality_"))) { - fprintf(stderr, "ERROR: unable to parse locality key '%s'\n", syn.c_str()); - return FDB_EXIT_ERROR; - } - syn = syn.substr(11); - std::transform(syn.begin(), syn.end(), syn.begin(), ::tolower); - localities.set(Standalone(syn), Standalone(std::string(args->OptionArg()))); - break; - } - case OPT_EXPIRE_BEFORE_DATETIME: - expireDatetime = args->OptionArg(); - break; - case OPT_EXPIRE_RESTORABLE_AFTER_DATETIME: - expireRestorableAfterDatetime = args->OptionArg(); - break; - case OPT_EXPIRE_BEFORE_VERSION: - case OPT_EXPIRE_RESTORABLE_AFTER_VERSION: - case OPT_EXPIRE_MIN_RESTORABLE_DAYS: - case OPT_EXPIRE_DELETE_BEFORE_DAYS: - { - const char* a = args->OptionArg(); - long long ver = 0; - if (!sscanf(a, "%lld", &ver)) { - fprintf(stderr, "ERROR: Could not parse expiration version `%s'\n", a); - printHelpTeaser(argv[0]); - return FDB_EXIT_ERROR; - } + case OPT_HELP: + printUsage(programExe, false); + return FDB_EXIT_SUCCESS; + break; + case OPT_DEVHELP: + printUsage(programExe, true); + return FDB_EXIT_SUCCESS; + break; + case OPT_VERSION: + printVersion(); + return FDB_EXIT_SUCCESS; + break; + case OPT_NOBUFSTDOUT: + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + break; + case OPT_BUFSTDOUTERR: + setvbuf(stdout, NULL, _IOFBF, BUFSIZ); + setvbuf(stderr, NULL, _IOFBF, BUFSIZ); + break; + case OPT_QUIET: + quietDisplay = true; + break; + case OPT_DRYRUN: + dryRun = true; + break; + case OPT_DELETE_DATA: + deleteData = true; + break; + case OPT_MIN_CLEANUP_SECONDS: + knobs.push_back(std::make_pair("min_cleanup_seconds", args->OptionArg())); + break; + case OPT_FORCE: + forceAction = true; + break; + case OPT_TRACE: + trace = true; + break; + case OPT_TRACE_DIR: + trace = true; + traceDir = args->OptionArg(); + break; + case OPT_TRACE_FORMAT: + if (!validateTraceFormat(args->OptionArg())) { + fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args->OptionArg()); + } + traceFormat = args->OptionArg(); + break; + case OPT_TRACE_LOG_GROUP: + traceLogGroup = args->OptionArg(); + break; + case OPT_LOCALITY: { + std::string syn = args->OptionSyntax(); + if (!StringRef(syn).startsWith(LiteralStringRef("--locality_"))) { + fprintf(stderr, "ERROR: unable to parse locality key '%s'\n", syn.c_str()); + return FDB_EXIT_ERROR; + } + syn = syn.substr(11); + std::transform(syn.begin(), syn.end(), syn.begin(), ::tolower); + localities.set(Standalone(syn), Standalone(std::string(args->OptionArg()))); + break; + } + case OPT_EXPIRE_BEFORE_DATETIME: + expireDatetime = args->OptionArg(); + break; + case OPT_EXPIRE_RESTORABLE_AFTER_DATETIME: + expireRestorableAfterDatetime = args->OptionArg(); + break; + case OPT_EXPIRE_BEFORE_VERSION: + case OPT_EXPIRE_RESTORABLE_AFTER_VERSION: + case OPT_EXPIRE_MIN_RESTORABLE_DAYS: + case OPT_EXPIRE_DELETE_BEFORE_DAYS: { + const char* a = args->OptionArg(); + long long ver = 0; + if (!sscanf(a, "%lld", &ver)) { + fprintf(stderr, "ERROR: Could not parse expiration version `%s'\n", a); + printHelpTeaser(argv[0]); + return FDB_EXIT_ERROR; + } - // Interpret the value as days worth of versions relative to now (negative) - if(optId == OPT_EXPIRE_MIN_RESTORABLE_DAYS || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS) { - ver = -ver * 24 * 60 * 60 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND; - } + // Interpret the value as days worth of versions relative to now (negative) + if (optId == OPT_EXPIRE_MIN_RESTORABLE_DAYS || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS) { + ver = -ver * 24 * 60 * 60 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND; + } - if(optId == OPT_EXPIRE_BEFORE_VERSION || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS) - expireVersion = ver; - else - expireRestorableAfterVersion = ver; - break; + if (optId == OPT_EXPIRE_BEFORE_VERSION || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS) + expireVersion = ver; + else + expireRestorableAfterVersion = ver; + break; + } + case OPT_RESTORE_TIMESTAMP: + restoreTimestamp = args->OptionArg(); + break; + case OPT_BASEURL: + baseUrl = args->OptionArg(); + break; + case OPT_RESTORE_CLUSTERFILE_DEST: + restoreClusterFileDest = args->OptionArg(); + break; + case OPT_RESTORE_CLUSTERFILE_ORIG: + restoreClusterFileOrig = args->OptionArg(); + break; + case OPT_CLUSTERFILE: + clusterFile = args->OptionArg(); + break; + case OPT_DEST_CLUSTER: + clusterFile = args->OptionArg(); + break; + case OPT_SOURCE_CLUSTER: + sourceClusterFile = args->OptionArg(); + break; + case OPT_CLEANUP: + partial = false; + break; + case OPT_KNOB: { + std::string syn = args->OptionSyntax(); + if (!StringRef(syn).startsWith(LiteralStringRef("--knob_"))) { + fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str()); + return FDB_EXIT_ERROR; } - case OPT_RESTORE_TIMESTAMP: - restoreTimestamp = args->OptionArg(); - break; - case OPT_BASEURL: - baseUrl = args->OptionArg(); - break; - case OPT_RESTORE_CLUSTERFILE_DEST: - restoreClusterFileDest = args->OptionArg(); - break; - case OPT_RESTORE_CLUSTERFILE_ORIG: - restoreClusterFileOrig = args->OptionArg(); - break; - case OPT_CLUSTERFILE: - clusterFile = args->OptionArg(); - break; - case OPT_DEST_CLUSTER: - clusterFile = args->OptionArg(); - break; - case OPT_SOURCE_CLUSTER: - sourceClusterFile = args->OptionArg(); - break; - case OPT_CLEANUP: - partial = false; - break; - case OPT_KNOB: { - std::string syn = args->OptionSyntax(); - if (!StringRef(syn).startsWith(LiteralStringRef("--knob_"))) { - fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str()); - return FDB_EXIT_ERROR; - } - syn = syn.substr(7); - knobs.push_back( std::make_pair( syn, args->OptionArg() ) ); - break; - } - case OPT_BACKUPKEYS: - try { - addKeyRange(args->OptionArg(), backupKeys); - } - catch (Error &) { - printHelpTeaser(argv[0]); - return FDB_EXIT_ERROR; - } - break; - case OPT_DESTCONTAINER: - destinationContainer = args->OptionArg(); - // If the url starts with '/' then prepend "file://" for backwards compatibility - if(StringRef(destinationContainer).startsWith(LiteralStringRef("/"))) - destinationContainer = std::string("file://") + destinationContainer; - modifyOptions.destURL = destinationContainer; - break; - case OPT_SNAPSHOTINTERVAL: - case OPT_MOD_ACTIVE_INTERVAL: - { - const char* a = args->OptionArg(); - int seconds; - if (!sscanf(a, "%d", &seconds)) { - fprintf(stderr, "ERROR: Could not parse snapshot interval `%s'\n", a); - printHelpTeaser(argv[0]); - return FDB_EXIT_ERROR; - } - if(optId == OPT_SNAPSHOTINTERVAL) { - snapshotIntervalSeconds = seconds; - modifyOptions.snapshotIntervalSeconds = seconds; - } - else if(optId == OPT_MOD_ACTIVE_INTERVAL) { - modifyOptions.activeSnapshotIntervalSeconds = seconds; - } - break; + syn = syn.substr(7); + knobs.push_back(std::make_pair(syn, args->OptionArg())); + break; + } + case OPT_BACKUPKEYS: + try { + addKeyRange(args->OptionArg(), backupKeys); + } catch (Error&) { + printHelpTeaser(argv[0]); + return FDB_EXIT_ERROR; } - case OPT_MOD_VERIFY_UID: - modifyOptions.verifyUID = args->OptionArg(); - break; - case OPT_WAITFORDONE: - waitForDone = true; - break; - case OPT_NOSTOPWHENDONE: - stopWhenDone = false; - break; - case OPT_RESTORECONTAINER: - restoreContainer = args->OptionArg(); - // If the url starts with '/' then prepend "file://" for backwards compatibility - if(StringRef(restoreContainer).startsWith(LiteralStringRef("/"))) - restoreContainer = std::string("file://") + restoreContainer; - break; - case OPT_DESCRIBE_DEEP: - describeDeep = true; - break; - case OPT_DESCRIBE_TIMESTAMPS: - describeTimestamps = true; - break; - case OPT_PREFIX_ADD: - addPrefix = args->OptionArg(); - break; - case OPT_PREFIX_REMOVE: - removePrefix = args->OptionArg(); - break; - case OPT_ERRORLIMIT: { - const char* a = args->OptionArg(); - if (!sscanf(a, "%d", &maxErrors)) { - fprintf(stderr, "ERROR: Could not parse max number of errors `%s'\n", a); - printHelpTeaser(argv[0]); - return FDB_EXIT_ERROR; - } - break; + break; + case OPT_DESTCONTAINER: + destinationContainer = args->OptionArg(); + // If the url starts with '/' then prepend "file://" for backwards compatibility + if (StringRef(destinationContainer).startsWith(LiteralStringRef("/"))) + destinationContainer = std::string("file://") + destinationContainer; + modifyOptions.destURL = destinationContainer; + break; + case OPT_SNAPSHOTINTERVAL: + case OPT_MOD_ACTIVE_INTERVAL: { + const char* a = args->OptionArg(); + int seconds; + if (!sscanf(a, "%d", &seconds)) { + fprintf(stderr, "ERROR: Could not parse snapshot interval `%s'\n", a); + printHelpTeaser(argv[0]); + return FDB_EXIT_ERROR; } - case OPT_RESTORE_VERSION: { - const char* a = args->OptionArg(); - long long ver = 0; - if (!sscanf(a, "%lld", &ver)) { - fprintf(stderr, "ERROR: Could not parse database version `%s'\n", a); - printHelpTeaser(argv[0]); - return FDB_EXIT_ERROR; - } - restoreVersion = ver; - break; + if (optId == OPT_SNAPSHOTINTERVAL) { + snapshotIntervalSeconds = seconds; + modifyOptions.snapshotIntervalSeconds = seconds; + } else if (optId == OPT_MOD_ACTIVE_INTERVAL) { + modifyOptions.activeSnapshotIntervalSeconds = seconds; } - #ifdef _WIN32 - case OPT_PARENTPID: { - auto pid_str = args->OptionArg(); - int parent_pid = atoi(pid_str); - auto pHandle = OpenProcess( SYNCHRONIZE, FALSE, parent_pid ); - if( !pHandle ) { - TraceEvent("ParentProcessOpenError").GetLastError(); - fprintf(stderr, "Could not open parent process at pid %d (error %d)", parent_pid, GetLastError()); - throw platform_error(); - } - startThread(&parentWatcher, pHandle); - break; + break; + } + case OPT_MOD_VERIFY_UID: + modifyOptions.verifyUID = args->OptionArg(); + break; + case OPT_WAITFORDONE: + waitForDone = true; + break; + case OPT_NOSTOPWHENDONE: + stopWhenDone = false; + break; + case OPT_RESTORECONTAINER: + restoreContainer = args->OptionArg(); + // If the url starts with '/' then prepend "file://" for backwards compatibility + if (StringRef(restoreContainer).startsWith(LiteralStringRef("/"))) + restoreContainer = std::string("file://") + restoreContainer; + break; + case OPT_DESCRIBE_DEEP: + describeDeep = true; + break; + case OPT_DESCRIBE_TIMESTAMPS: + describeTimestamps = true; + break; + case OPT_PREFIX_ADD: + addPrefix = args->OptionArg(); + break; + case OPT_PREFIX_REMOVE: + removePrefix = args->OptionArg(); + break; + case OPT_ERRORLIMIT: { + const char* a = args->OptionArg(); + if (!sscanf(a, "%d", &maxErrors)) { + fprintf(stderr, "ERROR: Could not parse max number of errors `%s'\n", a); + printHelpTeaser(argv[0]); + return FDB_EXIT_ERROR; } - #endif - case OPT_TAGNAME: - tagName = args->OptionArg(); - tagProvided = true; - break; - case OPT_CRASHONERROR: - g_crashOnError = true; - break; - case OPT_MEMLIMIT: - ti = parse_with_suffix(args->OptionArg(), "MiB"); - if (!ti.present()) { - fprintf(stderr, "ERROR: Could not parse memory limit from `%s'\n", args->OptionArg()); - printHelpTeaser(argv[0]); - flushAndExit(FDB_EXIT_ERROR); - } - memLimit = ti.get(); - break; - case OPT_BLOB_CREDENTIALS: - blobCredentials.push_back(args->OptionArg()); - break; -#ifndef TLS_DISABLED - case TLSConfig::OPT_TLS_PLUGIN: - args->OptionArg(); - break; - case TLSConfig::OPT_TLS_CERTIFICATES: - tlsCertPath = args->OptionArg(); - break; - case TLSConfig::OPT_TLS_PASSWORD: - tlsPassword = args->OptionArg(); - break; - case TLSConfig::OPT_TLS_CA_FILE: - tlsCAPath = args->OptionArg(); - break; - case TLSConfig::OPT_TLS_KEY: - tlsKeyPath = args->OptionArg(); - break; - case TLSConfig::OPT_TLS_VERIFY_PEERS: - tlsVerifyPeers = args->OptionArg(); - break; + break; + } + case OPT_RESTORE_VERSION: { + const char* a = args->OptionArg(); + long long ver = 0; + if (!sscanf(a, "%lld", &ver)) { + fprintf(stderr, "ERROR: Could not parse database version `%s'\n", a); + printHelpTeaser(argv[0]); + return FDB_EXIT_ERROR; + } + restoreVersion = ver; + break; + } +#ifdef _WIN32 + case OPT_PARENTPID: { + auto pid_str = args->OptionArg(); + int parent_pid = atoi(pid_str); + auto pHandle = OpenProcess(SYNCHRONIZE, FALSE, parent_pid); + if (!pHandle) { + TraceEvent("ParentProcessOpenError").GetLastError(); + fprintf(stderr, "Could not open parent process at pid %d (error %d)", parent_pid, GetLastError()); + throw platform_error(); + } + startThread(&parentWatcher, pHandle); + break; + } #endif - case OPT_DUMP_BEGIN: - dumpBegin = parseVersion(args->OptionArg()); - break; - case OPT_DUMP_END: - dumpEnd = parseVersion(args->OptionArg()); - break; - case OPT_JSON: - jsonOutput = true; - break; + case OPT_TAGNAME: + tagName = args->OptionArg(); + tagProvided = true; + break; + case OPT_CRASHONERROR: + g_crashOnError = true; + break; + case OPT_MEMLIMIT: + ti = parse_with_suffix(args->OptionArg(), "MiB"); + if (!ti.present()) { + fprintf(stderr, "ERROR: Could not parse memory limit from `%s'\n", args->OptionArg()); + printHelpTeaser(argv[0]); + flushAndExit(FDB_EXIT_ERROR); + } + memLimit = ti.get(); + break; + case OPT_BLOB_CREDENTIALS: + blobCredentials.push_back(args->OptionArg()); + break; +#ifndef TLS_DISABLED + case TLSConfig::OPT_TLS_PLUGIN: + args->OptionArg(); + break; + case TLSConfig::OPT_TLS_CERTIFICATES: + tlsCertPath = args->OptionArg(); + break; + case TLSConfig::OPT_TLS_PASSWORD: + tlsPassword = args->OptionArg(); + break; + case TLSConfig::OPT_TLS_CA_FILE: + tlsCAPath = args->OptionArg(); + break; + case TLSConfig::OPT_TLS_KEY: + tlsKeyPath = args->OptionArg(); + break; + case TLSConfig::OPT_TLS_VERIFY_PEERS: + tlsVerifyPeers = args->OptionArg(); + break; +#endif + case OPT_DUMP_BEGIN: + dumpBegin = parseVersion(args->OptionArg()); + break; + case OPT_DUMP_END: + dumpEnd = parseVersion(args->OptionArg()); + break; + case OPT_JSON: + jsonOutput = true; + break; } } // Process the extra arguments - for (int argLoop = 0; argLoop < args->FileCount(); argLoop++) - { - switch (programExe) - { + for (int argLoop = 0; argLoop < args->FileCount(); argLoop++) { + switch (programExe) { case EXE_AGENT: fprintf(stderr, "ERROR: Backup Agent does not support argument value `%s'\n", args->File(argLoop)); printHelpTeaser(argv[0]); @@ -3134,8 +3234,7 @@ int main(int argc, char* argv[]) { else { try { addKeyRange(args->File(argLoop), backupKeys); - } - catch (Error& ) { + } catch (Error&) { printHelpTeaser(argv[0]); return FDB_EXIT_ERROR; } @@ -3165,8 +3264,7 @@ int main(int argc, char* argv[]) { else { try { addKeyRange(args->File(argLoop), backupKeys); - } - catch (Error& ) { + } catch (Error&) { printHelpTeaser(argv[0]); return FDB_EXIT_ERROR; } @@ -3180,8 +3278,7 @@ int main(int argc, char* argv[]) { } // Delete the simple option object, if defined - if (args) - { + if (args) { delete args; args = NULL; } @@ -3194,29 +3291,34 @@ int main(int argc, char* argv[]) { ClientKnobs* clientKnobs = new ClientKnobs(true); CLIENT_KNOBS = clientKnobs; - for(auto k=knobs.begin(); k!=knobs.end(); ++k) { + for (auto k = knobs.begin(); k != knobs.end(); ++k) { try { - if (!flowKnobs->setKnob( k->first, k->second ) && - !clientKnobs->setKnob( k->first, k->second )) - { + if (!flowKnobs->setKnob(k->first, k->second) && !clientKnobs->setKnob(k->first, k->second)) { fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", k->first.c_str()); TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(k->first)); } } catch (Error& e) { if (e.code() == error_code_invalid_option_value) { - fprintf(stderr, "WARNING: Invalid value '%s' for knob option '%s'\n", k->second.c_str(), k->first.c_str()); - TraceEvent(SevWarnAlways, "InvalidKnobValue").detail("Knob", printable(k->first)).detail("Value", printable(k->second)); - } - else { + fprintf(stderr, + "WARNING: Invalid value '%s' for knob option '%s'\n", + k->second.c_str(), + k->first.c_str()); + TraceEvent(SevWarnAlways, "InvalidKnobValue") + .detail("Knob", printable(k->first)) + .detail("Value", printable(k->second)); + } else { fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", k->first.c_str(), e.what()); - TraceEvent(SevError, "FailedToSetKnob").detail("Knob", printable(k->first)).detail("Value", printable(k->second)).error(e); + TraceEvent(SevError, "FailedToSetKnob") + .detail("Knob", printable(k->first)) + .detail("Value", printable(k->second)) + .error(e); throw; } } } if (trace) { - if(!traceLogGroup.empty()) + if (!traceLogGroup.empty()) setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(traceLogGroup)); if (traceDir.empty()) @@ -3235,8 +3337,7 @@ int main(int argc, char* argv[]) { if (tlsCertPath.size()) { try { setNetworkOption(FDBNetworkOptions::TLS_CERT_PATH, tlsCertPath); - } - catch (Error& e) { + } catch (Error& e) { fprintf(stderr, "ERROR: cannot set TLS certificate path to `%s' (%s)\n", tlsCertPath.c_str(), e.what()); return 1; } @@ -3245,8 +3346,7 @@ int main(int argc, char* argv[]) { if (tlsCAPath.size()) { try { setNetworkOption(FDBNetworkOptions::TLS_CA_PATH, tlsCAPath); - } - catch (Error& e) { + } catch (Error& e) { fprintf(stderr, "ERROR: cannot set TLS CA path to `%s' (%s)\n", tlsCAPath.c_str(), e.what()); return 1; } @@ -3257,8 +3357,7 @@ int main(int argc, char* argv[]) { setNetworkOption(FDBNetworkOptions::TLS_PASSWORD, tlsPassword); setNetworkOption(FDBNetworkOptions::TLS_KEY_PATH, tlsKeyPath); - } - catch (Error& e) { + } catch (Error& e) { fprintf(stderr, "ERROR: cannot set TLS key path to `%s' (%s)\n", tlsKeyPath.c_str(), e.what()); return 1; } @@ -3266,27 +3365,26 @@ int main(int argc, char* argv[]) { if (tlsVerifyPeers.size()) { try { setNetworkOption(FDBNetworkOptions::TLS_VERIFY_PEERS, tlsVerifyPeers); - } - catch (Error& e) { - fprintf(stderr, "ERROR: cannot set TLS peer verification to `%s' (%s)\n", tlsVerifyPeers.c_str(), e.what()); + } catch (Error& e) { + fprintf( + stderr, "ERROR: cannot set TLS peer verification to `%s' (%s)\n", tlsVerifyPeers.c_str(), e.what()); return 1; } } Error::init(); - std::set_new_handler( &platform::outOfMemory ); - setMemoryQuota( memLimit ); + std::set_new_handler(&platform::outOfMemory); + setMemoryQuota(memLimit); int total = 0; - for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) + for (auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) total += i->second; if (total) printf("%d errors:\n", total); - for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) + for (auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) if (i->second > 0) printf(" %d: %d %s\n", i->second, i->first, Error::fromCode(i->first).what()); - Reference ccf; Database db; Reference sourceCcf; @@ -3299,42 +3397,43 @@ int main(int argc, char* argv[]) { try { setupNetwork(0, true); - } - catch (Error& e) { + } catch (Error& e) { fprintf(stderr, "ERROR: %s\n", e.what()); return FDB_EXIT_ERROR; } TraceEvent("ProgramStart") - .setMaxEventLength(12000) - .detail("SourceVersion", getHGVersion()) - .detail("Version", FDB_VT_VERSION ) - .detail("PackageName", FDB_VT_PACKAGE_NAME) - .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL)) - .setMaxFieldLength(10000) - .detail("CommandLine", commandLine) - .setMaxFieldLength(0) - .detail("MemoryLimit", memLimit) - .trackLatest("ProgramStart"); + .setMaxEventLength(12000) + .detail("SourceVersion", getHGVersion()) + .detail("Version", FDB_VT_VERSION) + .detail("PackageName", FDB_VT_PACKAGE_NAME) + .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL)) + .setMaxFieldLength(10000) + .detail("CommandLine", commandLine) + .setMaxFieldLength(0) + .detail("MemoryLimit", memLimit) + .trackLatest("ProgramStart"); - // Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents are logged. This thread will eventually run the network, so call it now. + // Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents + // are logged. This thread will eventually run the network, so call it now. TraceEvent::setNetworkThread(); // Add blob credentials files from the environment to the list collected from the command line. - const char *blobCredsFromENV = getenv("FDB_BLOB_CREDENTIALS"); - if(blobCredsFromENV != nullptr) { + const char* blobCredsFromENV = getenv("FDB_BLOB_CREDENTIALS"); + if (blobCredsFromENV != nullptr) { StringRef t((uint8_t*)blobCredsFromENV, strlen(blobCredsFromENV)); do { StringRef file = t.eat(":"); - if(file.size() != 0) - blobCredentials.push_back(file.toString()); - } while(t.size() != 0); + if (file.size() != 0) + blobCredentials.push_back(file.toString()); + } while (t.size() != 0); } // Update the global blob credential files list - std::vector *pFiles = (std::vector *)g_network->global(INetwork::enBlobCredentialFiles); - if(pFiles != nullptr) { - for(auto &f : blobCredentials) { + std::vector* pFiles = + (std::vector*)g_network->global(INetwork::enBlobCredentialFiles); + if (pFiles != nullptr) { + for (auto& f : blobCredentials) { pFiles->push_back(f); } } @@ -3343,7 +3442,7 @@ int main(int argc, char* argv[]) { // For most modes, initCluster() will open a trace file, but some fdbbackup operations do not require // a cluster so they should use this instead. auto initTraceFile = [&]() { - if(trace) + if (trace) openTraceFile(NetworkAddress(), traceRollSize, traceMaxLogsSize, traceDir, "trace", traceLogGroup); }; @@ -3351,17 +3450,15 @@ int main(int argc, char* argv[]) { auto resolvedClusterFile = ClusterConnectionFile::lookupClusterFileName(clusterFile); try { ccf = Reference(new ClusterConnectionFile(resolvedClusterFile.first)); - } - catch (Error& e) { - if(!quiet) + } catch (Error& e) { + if (!quiet) fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedClusterFile, e).c_str()); return false; } try { db = Database::createDatabase(ccf, -1, true, localities); - } - catch (Error& e) { + } catch (Error& e) { fprintf(stderr, "ERROR: %s\n", e.what()); fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", ccf->getFilename().c_str()); return false; @@ -3370,130 +3467,144 @@ int main(int argc, char* argv[]) { return true; }; - if(sourceClusterFile.size()) { + if (sourceClusterFile.size()) { auto resolvedSourceClusterFile = ClusterConnectionFile::lookupClusterFileName(sourceClusterFile); try { - sourceCcf = Reference(new ClusterConnectionFile(resolvedSourceClusterFile.first)); - } - catch (Error& e) { + sourceCcf = + Reference(new ClusterConnectionFile(resolvedSourceClusterFile.first)); + } catch (Error& e) { fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedSourceClusterFile, e).c_str()); return FDB_EXIT_ERROR; } try { sourceDb = Database::createDatabase(sourceCcf, -1, true, localities); - } - catch (Error& e) { + } catch (Error& e) { fprintf(stderr, "ERROR: %s\n", e.what()); fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", sourceCcf->getFilename().c_str()); return FDB_EXIT_ERROR; } } - switch (programExe) - { + switch (programExe) { case EXE_AGENT: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; f = stopAfter(runAgent(db)); break; case EXE_BACKUP: - switch (backupType) - { - case BACKUP_START: - { - if(!initCluster()) + switch (backupType) { + case BACKUP_START: { + if (!initCluster()) return FDB_EXIT_ERROR; // Test out the backup url to make sure it parses. Doesn't test to make sure it's actually writeable. openBackupContainer(argv[0], destinationContainer); - f = stopAfter( submitBackup(db, destinationContainer, snapshotIntervalSeconds, backupKeys, tagName, dryRun, waitForDone, stopWhenDone) ); + f = stopAfter(submitBackup(db, + destinationContainer, + snapshotIntervalSeconds, + backupKeys, + tagName, + dryRun, + waitForDone, + stopWhenDone)); break; } - case BACKUP_MODIFY: - { - if(!initCluster()) + case BACKUP_MODIFY: { + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( modifyBackup(db, tagName, modifyOptions) ); + f = stopAfter(modifyBackup(db, tagName, modifyOptions)); break; } case BACKUP_STATUS: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( statusBackup(db, tagName, true, jsonOutput) ); + f = stopAfter(statusBackup(db, tagName, true, jsonOutput)); break; case BACKUP_ABORT: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( abortBackup(db, tagName) ); + f = stopAfter(abortBackup(db, tagName)); break; case BACKUP_CLEANUP: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( cleanupMutations(db, deleteData) ); + f = stopAfter(cleanupMutations(db, deleteData)); break; case BACKUP_WAIT: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( waitBackup(db, tagName, stopWhenDone) ); + f = stopAfter(waitBackup(db, tagName, stopWhenDone)); break; case BACKUP_DISCONTINUE: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( discontinueBackup(db, tagName, waitForDone) ); + f = stopAfter(discontinueBackup(db, tagName, waitForDone)); break; case BACKUP_PAUSE: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( changeBackupResumed(db, true) ); + f = stopAfter(changeBackupResumed(db, true)); break; case BACKUP_RESUME: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( changeBackupResumed(db, false) ); + f = stopAfter(changeBackupResumed(db, false)); break; case BACKUP_EXPIRE: initTraceFile(); // Must have a usable cluster if either expire DateTime options were used - if(!expireDatetime.empty() || !expireRestorableAfterDatetime.empty()) { - if(!initCluster()) + if (!expireDatetime.empty() || !expireRestorableAfterDatetime.empty()) { + if (!initCluster()) return FDB_EXIT_ERROR; } - f = stopAfter( expireBackupData(argv[0], destinationContainer, expireVersion, expireDatetime, db, forceAction, expireRestorableAfterVersion, expireRestorableAfterDatetime) ); + f = stopAfter(expireBackupData(argv[0], + destinationContainer, + expireVersion, + expireDatetime, + db, + forceAction, + expireRestorableAfterVersion, + expireRestorableAfterDatetime)); break; case BACKUP_DELETE: initTraceFile(); - f = stopAfter( deleteBackupContainer(argv[0], destinationContainer) ); + f = stopAfter(deleteBackupContainer(argv[0], destinationContainer)); break; case BACKUP_DESCRIBE: initTraceFile(); // If timestamp lookups are desired, require a cluster file - if(describeTimestamps && !initCluster()) + if (describeTimestamps && !initCluster()) return FDB_EXIT_ERROR; - // Only pass database optionDatabase Describe will lookup version timestamps if a cluster file was given, but quietly skip them if not. - f = stopAfter( describeBackup(argv[0], destinationContainer, describeDeep, describeTimestamps ? Optional(db) : Optional(), jsonOutput) ); + // Only pass database optionDatabase Describe will lookup version timestamps if a cluster file was + // given, but quietly skip them if not. + f = stopAfter(describeBackup(argv[0], + destinationContainer, + describeDeep, + describeTimestamps ? Optional(db) : Optional(), + jsonOutput)); break; case BACKUP_LIST: initTraceFile(); - f = stopAfter( listBackup(baseUrl) ); + f = stopAfter(listBackup(baseUrl)); break; case BACKUP_DUMP: initTraceFile(); - f = stopAfter( dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd) ); + f = stopAfter(dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd)); break; case BACKUP_UNDEFINED: @@ -3506,87 +3617,104 @@ int main(int argc, char* argv[]) { break; case EXE_RESTORE: - if(dryRun) { - if(restoreType != RESTORE_START) { + if (dryRun) { + if (restoreType != RESTORE_START) { fprintf(stderr, "Restore dry run only works for 'start' command\n"); return FDB_EXIT_ERROR; } // Must explicitly call trace file options handling if not calling Database::createDatabase() initTraceFile(); - } - else { - if(restoreClusterFileDest.empty()) { + } else { + if (restoreClusterFileDest.empty()) { fprintf(stderr, "Restore destination cluster file must be specified explicitly.\n"); return FDB_EXIT_ERROR; } - if(!fileExists(restoreClusterFileDest)) { - fprintf(stderr, "Restore destination cluster file '%s' does not exist.\n", restoreClusterFileDest.c_str()); + if (!fileExists(restoreClusterFileDest)) { + fprintf(stderr, + "Restore destination cluster file '%s' does not exist.\n", + restoreClusterFileDest.c_str()); return FDB_EXIT_ERROR; } try { db = Database::createDatabase(restoreClusterFileDest, Database::API_VERSION_LATEST); - } catch(Error &e) { - fprintf(stderr, "Restore destination cluster file '%s' invalid: %s\n", restoreClusterFileDest.c_str(), e.what()); + } catch (Error& e) { + fprintf(stderr, + "Restore destination cluster file '%s' invalid: %s\n", + restoreClusterFileDest.c_str(), + e.what()); return FDB_EXIT_ERROR; } } - switch(restoreType) { - case RESTORE_START: - f = stopAfter( runRestore(db, restoreClusterFileOrig, tagName, restoreContainer, backupKeys, restoreVersion, restoreTimestamp, !dryRun, !quietDisplay, waitForDone, addPrefix, removePrefix) ); - break; - case RESTORE_WAIT: - f = stopAfter( success(ba.waitRestore(db, KeyRef(tagName), true)) ); - break; - case RESTORE_ABORT: - f = stopAfter( map(ba.abortRestore(db, KeyRef(tagName)), [tagName](FileBackupAgent::ERestoreState s) -> Void { - printf("Tag: %s State: %s\n", tagName.c_str(), FileBackupAgent::restoreStateText(s).toString().c_str()); - return Void(); - }) ); - break; - case RESTORE_STATUS: - // If no tag is specifically provided then print all tag status, don't just use "default" - if(tagProvided) - tag = tagName; - f = stopAfter( map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void { - printf("%s\n", s.c_str()); - return Void(); - }) ); - break; - default: - throw restore_error(); + switch (restoreType) { + case RESTORE_START: + f = stopAfter(runRestore(db, + restoreClusterFileOrig, + tagName, + restoreContainer, + backupKeys, + restoreVersion, + restoreTimestamp, + !dryRun, + !quietDisplay, + waitForDone, + addPrefix, + removePrefix)); + break; + case RESTORE_WAIT: + f = stopAfter(success(ba.waitRestore(db, KeyRef(tagName), true))); + break; + case RESTORE_ABORT: + f = stopAfter( + map(ba.abortRestore(db, KeyRef(tagName)), [tagName](FileBackupAgent::ERestoreState s) -> Void { + printf("Tag: %s State: %s\n", + tagName.c_str(), + FileBackupAgent::restoreStateText(s).toString().c_str()); + return Void(); + })); + break; + case RESTORE_STATUS: + // If no tag is specifically provided then print all tag status, don't just use "default" + if (tagProvided) + tag = tagName; + f = stopAfter(map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void { + printf("%s\n", s.c_str()); + return Void(); + })); + break; + default: + throw restore_error(); } break; case EXE_DR_AGENT: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( runDBAgent(sourceDb, db) ); + f = stopAfter(runDBAgent(sourceDb, db)); break; case EXE_DB_BACKUP: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - switch (dbType) - { + switch (dbType) { case DB_START: - f = stopAfter( submitDBBackup(sourceDb, db, backupKeys, tagName) ); + f = stopAfter(submitDBBackup(sourceDb, db, backupKeys, tagName)); break; case DB_STATUS: - f = stopAfter( statusDBBackup(sourceDb, db, tagName, maxErrors) ); + f = stopAfter(statusDBBackup(sourceDb, db, tagName, maxErrors)); break; case DB_SWITCH: - f = stopAfter( switchDBBackup(sourceDb, db, backupKeys, tagName, forceAction) ); + f = stopAfter(switchDBBackup(sourceDb, db, backupKeys, tagName, forceAction)); break; case DB_ABORT: - f = stopAfter( abortDBBackup(sourceDb, db, tagName, partial) ); + f = stopAfter(abortDBBackup(sourceDb, db, tagName, partial)); break; case DB_PAUSE: - f = stopAfter( changeDBBackupResumed(sourceDb, db, true) ); + f = stopAfter(changeDBBackupResumed(sourceDb, db, true)); break; case DB_RESUME: - f = stopAfter( changeDBBackupResumed(sourceDb, db, false) ); + f = stopAfter(changeDBBackupResumed(sourceDb, db, false)); break; case DB_UNDEFINED: default: @@ -3603,34 +3731,28 @@ int main(int argc, char* argv[]) { runNetwork(); - if(f.isValid() && f.isReady() && !f.isError() && !f.get().present()) { + if (f.isValid() && f.isReady() && !f.isError() && !f.get().present()) { status = FDB_EXIT_ERROR; } - if(fstatus.isValid() && fstatus.isReady() && !fstatus.isError() && fstatus.get().present()) { + if (fstatus.isValid() && fstatus.isReady() && !fstatus.isError() && fstatus.get().present()) { status = fstatus.get().get(); } - #ifdef ALLOC_INSTRUMENTATION +#ifdef ALLOC_INSTRUMENTATION { - cout << "Page Counts: " - << FastAllocator<16>::pageCount << " " - << FastAllocator<32>::pageCount << " " - << FastAllocator<64>::pageCount << " " - << FastAllocator<128>::pageCount << " " - << FastAllocator<256>::pageCount << " " - << FastAllocator<512>::pageCount << " " - << FastAllocator<1024>::pageCount << " " - << FastAllocator<2048>::pageCount << " " - << FastAllocator<4096>::pageCount << " " - << FastAllocator<8192>::pageCount << endl; + cout << "Page Counts: " << FastAllocator<16>::pageCount << " " << FastAllocator<32>::pageCount << " " + << FastAllocator<64>::pageCount << " " << FastAllocator<128>::pageCount << " " + << FastAllocator<256>::pageCount << " " << FastAllocator<512>::pageCount << " " + << FastAllocator<1024>::pageCount << " " << FastAllocator<2048>::pageCount << " " + << FastAllocator<4096>::pageCount << " " << FastAllocator<8192>::pageCount << endl; - vector< std::pair > typeNames; - for( auto i = allocInstr.begin(); i != allocInstr.end(); ++i ) { + vector> typeNames; + for (auto i = allocInstr.begin(); i != allocInstr.end(); ++i) { std::string s; #ifdef __linux__ - char *demangled = abi::__cxa_demangle(i->first, NULL, NULL, NULL); + char* demangled = abi::__cxa_demangle(i->first, NULL, NULL, NULL); if (demangled) { s = demangled; if (StringRef(s).startsWith(LiteralStringRef("(anonymous namespace)::"))) @@ -3648,19 +3770,24 @@ int main(int argc, char* argv[]) { s = s.substr(LiteralStringRef("struct ").size()); #endif - typeNames.push_back( std::make_pair(s, i->first) ); + typeNames.push_back(std::make_pair(s, i->first)); } std::sort(typeNames.begin(), typeNames.end()); - for(int i=0; i { - std::string prompt; - ThreadReturnPromise> result; + struct Read : TypedAction { + std::string prompt; + ThreadReturnPromise> result; - virtual double getTimeEstimate() { return 0.0; } - explicit Read(std::string const& prompt) : prompt(prompt) {} - }; + virtual double getTimeEstimate() { return 0.0; } + explicit Read(std::string const& prompt) : prompt(prompt) {} + }; - void action(Read& r) { - try { - r.result.send( read(r.prompt) ); - } catch (Error& e) { - r.result.sendError(e); - } catch (...) { - r.result.sendError(unknown_error()); - } - } + void action(Read& r) { + try { + r.result.send(read(r.prompt)); + } catch (Error& e) { + r.result.sendError(e); + } catch (...) { + r.result.sendError(unknown_error()); + } + } private: - Optional read(std::string const& prompt) { - #if HAVE_LINENOISE - errno = 0; - char* line = linenoise(prompt.c_str()); - if (line) { - std::string s(line); - free(line); - return s; - } else { - if (errno == EAGAIN) // Ctrl-C - return std::string(); - return Optional(); - } - #else - std::string line; - std::fputs( prompt.c_str(), stdout ); - if (!std::getline( std::cin, line ).eof()) { - return line; - } else - return Optional(); - #endif - } + Optional read(std::string const& prompt) { +#if HAVE_LINENOISE + errno = 0; + char* line = linenoise(prompt.c_str()); + if (line) { + std::string s(line); + free(line); + return s; + } else { + if (errno == EAGAIN) // Ctrl-C + return std::string(); + return Optional(); + } +#else + std::string line; + std::fputs(prompt.c_str(), stdout); + if (!std::getline(std::cin, line).eof()) { + return line; + } else + return Optional(); +#endif + } }; -LineNoise::LineNoise( - std::function< void(std::string const&, std::vector&) > _completion_callback, - std::function< Hint(std::string const&) > _hint_callback, - int maxHistoryLines, - bool multiline ) - : threadPool( createGenericThreadPool() ) -{ - reader = new LineNoiseReader(); +LineNoise::LineNoise(std::function&)> _completion_callback, + std::function _hint_callback, + int maxHistoryLines, + bool multiline) + : threadPool(createGenericThreadPool()) { + reader = new LineNoiseReader(); - #if HAVE_LINENOISE - // It should be OK to call these functions from this thread, since read() can't be called yet - // The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely invoke the callbacks we've been given +#if HAVE_LINENOISE + // It should be OK to call these functions from this thread, since read() can't be called yet + // The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely + // invoke the callbacks we've been given - // linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables - static std::function< void(std::string const&, std::vector&) > completion_callback; - static std::function< Hint(std::string const&) > hint_callback; - completion_callback = _completion_callback; - hint_callback = _hint_callback; + // linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables + static std::function&)> completion_callback; + static std::function hint_callback; + completion_callback = _completion_callback; + hint_callback = _hint_callback; - linenoiseHistorySetMaxLen( maxHistoryLines ); - linenoiseSetMultiLine( multiline ); - linenoiseSetCompletionCallback( [](const char* line, linenoiseCompletions* lc) { - // This code will run in the thread pool - std::vector completions; - onMainThread( [line, &completions]() -> Future { - completion_callback(line, completions); - return Void(); - }).getBlocking(); - for( auto const& c : completions ) - linenoiseAddCompletion( lc, c.c_str() ); - }); - /*linenoiseSetHintsCallback( [](const char* line, int* color, int*bold) -> const char* { - Hint h = onMainThread( [line]() -> Future { - return hint_callback(line); - }).getBlocking(); - if (!h.valid) return NULL; - *color = h.color; - *bold = h.bold; - return strdup( h.text.c_str() ); - }); - linenoiseSetFreeHintsCallback( free );*/ - #endif + linenoiseHistorySetMaxLen(maxHistoryLines); + linenoiseSetMultiLine(multiline); + linenoiseSetCompletionCallback([](const char* line, linenoiseCompletions* lc) { + // This code will run in the thread pool + std::vector completions; + onMainThread([line, &completions]() -> Future { + completion_callback(line, completions); + return Void(); + }).getBlocking(); + for (auto const& c : completions) + linenoiseAddCompletion(lc, c.c_str()); + }); + /*linenoiseSetHintsCallback( [](const char* line, int* color, int*bold) -> const char* { + Hint h = onMainThread( [line]() -> Future { + return hint_callback(line); + }).getBlocking(); + if (!h.valid) return NULL; + *color = h.color; + *bold = h.bold; + return strdup( h.text.c_str() ); + }); + linenoiseSetFreeHintsCallback( free );*/ +#endif - threadPool->addThread(reader); + threadPool->addThread(reader); } LineNoise::~LineNoise() { - threadPool.clear(); + threadPool.clear(); } -Future> LineNoise::read( std::string const& prompt ) { - auto r = new LineNoiseReader::Read(prompt); - auto f = r->result.getFuture(); - threadPool->post(r); - return f; +Future> LineNoise::read(std::string const& prompt) { + auto r = new LineNoiseReader::Read(prompt); + auto f = r->result.getFuture(); + threadPool->post(r); + return f; } ACTOR Future waitKeyboardInterrupt(boost::asio::io_service* ios) { - state boost::asio::signal_set signals(*ios, SIGINT); - Promise result; - signals.async_wait([result](const boost::system::error_code& error, int signal_number) { - if (error) { - result.sendError(io_error()); - } else { - result.send(Void()); - } - }); + state boost::asio::signal_set signals(*ios, SIGINT); + Promise result; + signals.async_wait([result](const boost::system::error_code& error, int signal_number) { + if (error) { + result.sendError(io_error()); + } else { + result.send(Void()); + } + }); - wait(result.getFuture()); - return Void(); + wait(result.getFuture()); + return Void(); } Future LineNoise::onKeyboardInterrupt() { - boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService); - if (!ios) return Never(); - return waitKeyboardInterrupt(ios); + boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService); + if (!ios) + return Never(); + return waitKeyboardInterrupt(ios); } -void LineNoise::historyAdd( std::string const& line ) { - #if HAVE_LINENOISE - linenoiseHistoryAdd( line.c_str() ); - #endif +void LineNoise::historyAdd(std::string const& line) { +#if HAVE_LINENOISE + linenoiseHistoryAdd(line.c_str()); +#endif } -void LineNoise::historyLoad( std::string const& filename ) { - #if HAVE_LINENOISE - if(linenoiseHistoryLoad(filename.c_str()) != 0) { - throw io_error(); - } - #endif +void LineNoise::historyLoad(std::string const& filename) { +#if HAVE_LINENOISE + if (linenoiseHistoryLoad(filename.c_str()) != 0) { + throw io_error(); + } +#endif } -void LineNoise::historySave( std::string const& filename ) { - #if HAVE_LINENOISE - if(linenoiseHistorySave(filename.c_str()) != 0) { - throw io_error(); - } - #endif +void LineNoise::historySave(std::string const& filename) { +#if HAVE_LINENOISE + if (linenoiseHistorySave(filename.c_str()) != 0) { + throw io_error(); + } +#endif } diff --git a/fdbcli/FlowLineNoise.h b/fdbcli/FlowLineNoise.h index b81ce88939..87f88c69de 100644 --- a/fdbcli/FlowLineNoise.h +++ b/fdbcli/FlowLineNoise.h @@ -26,39 +26,37 @@ #include struct LineNoise : NonCopyable { - // Wraps the linenoise library so that it can be called from asynchronous Flow code - // Only create one of these at a time; the linenoise library only supports one history - // - // The current implementation does not support calling read concurrently with any other - // function (or itself). + // Wraps the linenoise library so that it can be called from asynchronous Flow code + // Only create one of these at a time; the linenoise library only supports one history + // + // The current implementation does not support calling read concurrently with any other + // function (or itself). - struct Hint { - std::string text; - int color; - bool bold; - bool valid; - Hint() : text(), color(), bold(), valid() {} - Hint( std::string const& text, int color, bool bold ) : text(text), color(color), bold(bold), valid(true) {} - }; + struct Hint { + std::string text; + int color; + bool bold; + bool valid; + Hint() : text(), color(), bold(), valid() {} + Hint(std::string const& text, int color, bool bold) : text(text), color(color), bold(bold), valid(true) {} + }; - LineNoise( - std::function< void(std::string const&, std::vector&) > completion_callback, - std::function< Hint(std::string const&) > hint_callback, - int maxHistoryLines, - bool multiline - ); - ~LineNoise(); + LineNoise(std::function&)> completion_callback, + std::function hint_callback, + int maxHistoryLines, + bool multiline); + ~LineNoise(); - Future< Optional > read( std::string const& prompt ); // Returns "nothing" on EOF - void historyAdd( std::string const& line ); + Future> read(std::string const& prompt); // Returns "nothing" on EOF + void historyAdd(std::string const& line); - void historyLoad( std::string const& filename ); - void historySave( std::string const& filename ); + void historyLoad(std::string const& filename); + void historySave(std::string const& filename); - static Future onKeyboardInterrupt(); // Returns when Ctrl-C is next pressed (i.e. SIGINT) + static Future onKeyboardInterrupt(); // Returns when Ctrl-C is next pressed (i.e. SIGINT) - Reference threadPool; - struct LineNoiseReader* reader; + Reference threadPool; + struct LineNoiseReader* reader; }; #endif \ No newline at end of file diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index f34534f7b3..e89776efc5 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -53,7 +53,7 @@ #include "versions.h" #endif -#include "flow/actorcompiler.h" // This must be the last #include. +#include "flow/actorcompiler.h" // This must be the last #include. extern const char* getHGVersion(); @@ -97,7 +97,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP }, TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS }; + SO_END_OF_OPTIONS }; void printAtCol(const char* text, int col) { const char* iter = text; @@ -106,12 +106,15 @@ void printAtCol(const char* text, int col) { do { iter++; - if (*iter == '\n' || *iter == ' ' || *iter == '\0') space = iter; + if (*iter == '\n' || *iter == ' ' || *iter == '\0') + space = iter; if (*iter == '\n' || *iter == '\0' || (iter - start == col)) { - if (!space) space = iter; + if (!space) + space = iter; printf("%.*s\n", (int)(space - start), start); start = space; - if (*start == ' ' || *start == '\n') start++; + if (*start == ' ' || *start == '\n') + start++; space = NULL; } } while (*iter); @@ -124,12 +127,15 @@ std::string lineWrap(const char* text, int col) { std::string out = ""; do { iter++; - if (*iter == '\n' || *iter == ' ' || *iter == '\0') space = iter; + if (*iter == '\n' || *iter == ' ' || *iter == '\0') + space = iter; if (*iter == '\n' || *iter == '\0' || (iter - start == col)) { - if (!space) space = iter; + if (!space) + space = iter; out += format("%.*s\n", (int)(space - start), start); start = space; - if (*start == ' '/* || *start == '\n'*/) start++; + if (*start == ' ' /* || *start == '\n'*/) + start++; space = NULL; } } while (*iter); @@ -138,78 +144,82 @@ std::string lineWrap(const char* text, int col) { class FdbOptions { public: - //Prints an error and throws invalid_option or invalid_option_value if the option could not be set - void setOption(Reference tr, StringRef optionStr, bool enabled, Optional arg, bool intrans) { + // Prints an error and throws invalid_option or invalid_option_value if the option could not be set + void setOption(Reference tr, + StringRef optionStr, + bool enabled, + Optional arg, + bool intrans) { auto transactionItr = transactionOptions.legalOptions.find(optionStr.toString()); - if(transactionItr != transactionOptions.legalOptions.end()) + if (transactionItr != transactionOptions.legalOptions.end()) setTransactionOption(tr, transactionItr->second, enabled, arg, intrans); else { - printf("ERROR: invalid option '%s'. Try `help options' for a list of available options.\n", optionStr.toString().c_str()); + printf("ERROR: invalid option '%s'. Try `help options' for a list of available options.\n", + optionStr.toString().c_str()); throw invalid_option(); } } - //Applies all enabled transaction options to the given transaction + // Applies all enabled transaction options to the given transaction void apply(Reference tr) { - for(auto itr = transactionOptions.options.begin(); itr != transactionOptions.options.end(); ++itr) + for (auto itr = transactionOptions.options.begin(); itr != transactionOptions.options.end(); ++itr) tr->setOption(itr->first, itr->second.castTo()); } - //Returns true if any options have been set - bool hasAnyOptionsEnabled() { - return !transactionOptions.options.empty(); - } + // Returns true if any options have been set + bool hasAnyOptionsEnabled() { return !transactionOptions.options.empty(); } - //Prints a list of enabled options, along with their parameters (if any) + // Prints a list of enabled options, along with their parameters (if any) void print() { bool found = false; found = found || transactionOptions.print(); - if(!found) + if (!found) printf("There are no options enabled\n"); } - //Returns a vector of the names of all documented options - std::vector getValidOptions() { - return transactionOptions.getValidOptions(); - } + // Returns a vector of the names of all documented options + std::vector getValidOptions() { return transactionOptions.getValidOptions(); } - //Prints the help string obtained by invoking `help options' - void printHelpString() { - transactionOptions.printHelpString(); - } + // Prints the help string obtained by invoking `help options' + void printHelpString() { transactionOptions.printHelpString(); } private: - //Sets a transaction option. If intrans == true, then this option is also applied to the passed in transaction. - void setTransactionOption(Reference tr, FDBTransactionOptions::Option option, bool enabled, Optional arg, bool intrans) { - if(enabled && arg.present() != FDBTransactionOptions::optionInfo.getMustExist(option).hasParameter) { + // Sets a transaction option. If intrans == true, then this option is also applied to the passed in transaction. + void setTransactionOption(Reference tr, + FDBTransactionOptions::Option option, + bool enabled, + Optional arg, + bool intrans) { + if (enabled && arg.present() != FDBTransactionOptions::optionInfo.getMustExist(option).hasParameter) { printf("ERROR: option %s a parameter\n", arg.present() ? "did not expect" : "expected"); throw invalid_option_value(); } - if(intrans) + if (intrans) tr->setOption(option, arg); transactionOptions.setOption(option, enabled, arg.castTo()); } - //A group of enabled options (of type T::Option) as well as a legal options map from string to T::Option + // A group of enabled options (of type T::Option) as well as a legal options map from string to T::Option template struct OptionGroup { std::map>> options; std::map legalOptions; - OptionGroup() { } - OptionGroup(OptionGroup &base) : options(base.options.begin(), base.options.end()), legalOptions(base.legalOptions) { } + OptionGroup() {} + OptionGroup(OptionGroup& base) + : options(base.options.begin(), base.options.end()), legalOptions(base.legalOptions) {} - //Enable or disable an option. Returns true if option value changed + // Enable or disable an option. Returns true if option value changed bool setOption(typename T::Option option, bool enabled, Optional arg) { auto optionItr = options.find(option); - if(enabled && (optionItr == options.end() || Optional>(optionItr->second).castTo< StringRef >() != arg)) { + if (enabled && (optionItr == options.end() || + Optional>(optionItr->second).castTo() != arg)) { options[option] = arg.castTo>(); return true; - } - else if(!enabled && optionItr != options.end()) { + } else if (!enabled && optionItr != options.end()) { options.erase(optionItr); return true; } @@ -217,14 +227,14 @@ private: return false; } - //Prints a list of all enabled options in this group + // Prints a list of all enabled options in this group bool print() { bool found = false; - for(auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) { + for (auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) { auto optionItr = options.find(itr->second); - if(optionItr != options.end()) { - if(optionItr->second.present()) + if (optionItr != options.end()) { + if (optionItr->second.present()) printf("%s: `%s'\n", itr->first.c_str(), formatStringRef(optionItr->second.get()).c_str()); else printf("%s\n", itr->first.c_str()); @@ -236,7 +246,7 @@ private: return found; } - //Returns true if the specified option is documented + // Returns true if the specified option is documented bool isDocumented(typename T::Option option) { FDBOptionInfo info = T::optionInfo.getMustExist(option); @@ -244,25 +254,25 @@ private: return !info.comment.empty() && info.comment.substr(0, deprecatedStr.size()) != deprecatedStr; } - //Returns a vector of the names of all documented options + // Returns a vector of the names of all documented options std::vector getValidOptions() { std::vector ret; for (auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) - if(isDocumented(itr->second)) + if (isDocumented(itr->second)) ret.push_back(itr->first); return ret; } - //Prints a help string for each option in this group. Any options with no comment - //are excluded from this help string. Lines are wrapped to 80 characters. + // Prints a help string for each option in this group. Any options with no comment + // are excluded from this help string. Lines are wrapped to 80 characters. void printHelpString() { - for(auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) { - if(isDocumented(itr->second)) { + for (auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) { + if (isDocumented(itr->second)) { FDBOptionInfo info = T::optionInfo.getMustExist(itr->second); std::string helpStr = info.name + " - " + info.comment; - if(info.hasParameter) + if (info.hasParameter) helpStr += " " + info.parameterComment; helpStr += "\n"; @@ -276,15 +286,15 @@ private: public: FdbOptions() { - for(auto itr = FDBTransactionOptions::optionInfo.begin(); itr != FDBTransactionOptions::optionInfo.end(); ++itr) + for (auto itr = FDBTransactionOptions::optionInfo.begin(); itr != FDBTransactionOptions::optionInfo.end(); + ++itr) transactionOptions.legalOptions[itr->second.name] = itr->first; } - FdbOptions(FdbOptions &base) : transactionOptions(base.transactionOptions) { } + FdbOptions(FdbOptions& base) : transactionOptions(base.transactionOptions) {} }; -static std::string formatStringRef(StringRef item, bool fullEscaping = false) -{ +static std::string formatStringRef(StringRef item, bool fullEscaping = false) { std::string ret; for (int i = 0; i < item.size(); i++) { @@ -303,17 +313,14 @@ static std::string formatStringRef(StringRef item, bool fullEscaping = false) return ret; } -static bool tokencmp(StringRef token, const char *command) -{ +static bool tokencmp(StringRef token, const char* command) { if (token.size() != strlen(command)) return false; return !memcmp(token.begin(), command, token.size()); } - -static std::vector> parseLine(std::string& line, bool& err, bool& partial) -{ +static std::vector> parseLine(std::string& line, bool& err, bool& partial) { err = false; partial = false; @@ -328,69 +335,69 @@ static std::vector> parseLine(std::string& line, bool& er while (i <= line.length()) { switch (line[i]) { - case ';': - if (!quoted) { - if (i > offset || (forcetoken && i == offset)) - buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); - ret.push_back(std::move(buf)); - offset = i = line.find_first_not_of(' ', i+1); - forcetoken = false; - } else - i++; - break; + case ';': + if (!quoted) { + if (i > offset || (forcetoken && i == offset)) + buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); + ret.push_back(std::move(buf)); + offset = i = line.find_first_not_of(' ', i + 1); + forcetoken = false; + } else + i++; + break; + case '"': + quoted = !quoted; + line.erase(i, 1); + forcetoken = true; + break; + case ' ': + if (!quoted) { + if (i > offset || (forcetoken && i == offset)) + buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); + offset = i = line.find_first_not_of(' ', i); + forcetoken = false; + } else + i++; + break; + case '\\': + if (i + 2 > line.length()) { + err = true; + ret.push_back(std::move(buf)); + return ret; + } + switch (line[i + 1]) { + char ent, save; case '"': - quoted = !quoted; - line.erase(i, 1); - forcetoken = true; - break; - case ' ': - if (!quoted) { - if (i > offset || (forcetoken && i == offset)) - buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); - offset = i = line.find_first_not_of(' ', i); - forcetoken = false; - } else - i++; - break; case '\\': - if (i + 2 > line.length()) { + case ' ': + case ';': + line.erase(i, 1); + break; + case 'x': + if (i + 4 > line.length()) { err = true; ret.push_back(std::move(buf)); return ret; } - switch (line[i+1]) { - char ent, save; - case '"': - case '\\': - case ' ': - case ';': - line.erase(i, 1); - break; - case 'x': - if (i + 4 > line.length()) { - err = true; - ret.push_back(std::move(buf)); - return ret; - } - char *pEnd; - save = line[i + 4]; - line[i + 4] = 0; - ent = char(strtoul(line.data() + i + 2, &pEnd, 16)); - if (*pEnd) { - err = true; - ret.push_back(std::move(buf)); - return ret; - } - line[i + 4] = save; - line.replace(i, 4, 1, ent); - break; - default: - err = true; - ret.push_back(std::move(buf)); - return ret; + char* pEnd; + save = line[i + 4]; + line[i + 4] = 0; + ent = char(strtoul(line.data() + i + 2, &pEnd, 16)); + if (*pEnd) { + err = true; + ret.push_back(std::move(buf)); + return ret; } + line[i + 4] = save; + line.replace(i, 4, 1, ent); + break; default: - i++; + err = true; + ret.push_back(std::move(buf)); + return ret; + } + default: + i++; } } @@ -408,12 +415,14 @@ static std::vector> parseLine(std::string& line, bool& er static void printProgramUsage(const char* name) { printf("FoundationDB CLI " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n" - "usage: %s [OPTIONS]\n" - "\n", name); + "usage: %s [OPTIONS]\n" + "\n", + name); printf(" -C CONNFILE The path of a file containing the connection string for the\n" - " FoundationDB cluster. The default is first the value of the\n" - " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" - " then `%s'.\n", platform::getDefaultClusterFilePath().c_str()); + " FoundationDB cluster. The default is first the value of the\n" + " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" + " then `%s'.\n", + platform::getDefaultClusterFilePath().c_str()); printf(" --log Enables trace file logging for the CLI session.\n" " --log-dir PATH Specifes the output directory for trace files. If\n" " unspecified, defaults to the current directory. Has\n" @@ -430,13 +439,12 @@ static void printProgramUsage(const char* name) { #endif " --knob_KNOBNAME KNOBVALUE\n" " Changes a knob option. KNOBNAME should be lowercase.\n" - " --debug-tls Prints the TLS configuration and certificate chain, then exits.\n" - " Useful in reporting and diagnosing TLS issues.\n" + " --debug-tls Prints the TLS configuration and certificate chain, then exits.\n" + " Useful in reporting and diagnosing TLS issues.\n" " -v, --version Print FoundationDB CLI version information and exit.\n" " -h, --help Display this help and exit.\n"); } - struct CommandHelp { std::string usage; std::string short_desc; @@ -452,137 +460,190 @@ std::set hiddenCommands; #define ESCAPINGKV "\n\nFor information on escaping keys and values, type `help escaping'." void initHelp() { - helpMap["begin"] = CommandHelp( - "begin", - "begin a new transaction", - "By default, the fdbcli operates in autocommit mode. All operations are performed in their own transaction, and are automatically committed for you. By explicitly beginning a transaction, successive operations are all performed as part of a single transaction.\n\nTo commit the transaction, use the commit command. To discard the transaction, use the reset command."); - helpMap["commit"] = CommandHelp( - "commit", - "commit the current transaction", - "Any sets or clears executed after the start of the current transaction will be committed to the database. On success, the committed version number is displayed. If commit fails, the error is displayed and the transaction must be retried."); + helpMap["begin"] = + CommandHelp("begin", + "begin a new transaction", + "By default, the fdbcli operates in autocommit mode. All operations are performed in their own " + "transaction, and are automatically committed for you. By explicitly beginning a transaction, " + "successive operations are all performed as part of a single transaction.\n\nTo commit the " + "transaction, use the commit command. To discard the transaction, use the reset command."); + helpMap["commit"] = CommandHelp("commit", + "commit the current transaction", + "Any sets or clears executed after the start of the current transaction will be " + "committed to the database. On success, the committed version number is displayed. " + "If commit fails, the error is displayed and the transaction must be retried."); helpMap["clear"] = CommandHelp( - "clear ", - "clear a key from the database", - "Clear succeeds even if the specified key is not present, but may fail because of conflicts." ESCAPINGK); + "clear ", + "clear a key from the database", + "Clear succeeds even if the specified key is not present, but may fail because of conflicts." ESCAPINGK); helpMap["clearrange"] = CommandHelp( - "clearrange ", - "clear a range of keys from the database", - "All keys between BEGINKEY (inclusive) and ENDKEY (exclusive) are cleared from the database. This command will succeed even if the specified range is empty, but may fail because of conflicts." ESCAPINGK); + "clearrange ", + "clear a range of keys from the database", + "All keys between BEGINKEY (inclusive) and ENDKEY (exclusive) are cleared from the database. This command will " + "succeed even if the specified range is empty, but may fail because of conflicts." ESCAPINGK); helpMap["configure"] = CommandHelp( - "configure [new] |logs=|resolvers=>*", - "change the database configuration", - "The `new' option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When used, both a redundancy mode and a storage engine must be specified.\n\nRedundancy mode:\n single - one copy of the data. Not fault tolerant.\n double - two copies of data (survive one failure).\n triple - three copies of data (survive two failures).\n three_data_hall - See the Admin Guide.\n three_datacenter - See the Admin Guide.\n\nStorage engine:\n ssd - B-Tree storage engine optimized for solid state disks.\n memory - Durable in-memory storage engine for small datasets.\n\nproxies=: Sets the desired number of proxies in the cluster. Must be at least 1, or set to -1 which restores the number of proxies to the default value.\n\nlogs=: Sets the desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of logs to the default value.\n\nresolvers=: Sets the desired number of resolvers in the cluster. Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\nSee the FoundationDB Administration Guide for more information."); + "configure [new] " + "|logs=|resolvers=<" + "RESOLVERS>>*", + "change the database configuration", + "The `new' option, if present, initializes a new database with the given configuration rather than changing " + "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be " + "specified.\n\nRedundancy mode:\n single - one copy of the data. Not fault tolerant.\n double - two copies " + "of data (survive one failure).\n triple - three copies of data (survive two failures).\n three_data_hall - " + "See the Admin Guide.\n three_datacenter - See the Admin Guide.\n\nStorage engine:\n ssd - B-Tree storage " + "engine optimized for solid state disks.\n memory - Durable in-memory storage engine for small " + "datasets.\n\nproxies=: Sets the desired number of proxies in the cluster. Must be at least 1, or set " + "to -1 which restores the number of proxies to the default value.\n\nlogs=: Sets the desired number of " + "log servers in the cluster. Must be at least 1, or set to -1 which restores the number of logs to the default " + "value.\n\nresolvers=: Sets the desired number of resolvers in the cluster. Must be at least 1, or " + "set to -1 which restores the number of resolvers to the default value.\n\nSee the FoundationDB Administration " + "Guide for more information."); helpMap["fileconfigure"] = CommandHelp( - "fileconfigure [new] ", - "change the database configuration from a file", - "The `new' option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. Load a JSON document from the provided file, and change the database configuration to match the contents of the JSON document. The format should be the same as the value of the \"configuration\" entry in status JSON without \"excluded_servers\" or \"coordinators_count\"."); + "fileconfigure [new] ", + "change the database configuration from a file", + "The `new' option, if present, initializes a new database with the given configuration rather than changing " + "the configuration of an existing one. Load a JSON document from the provided file, and change the database " + "configuration to match the contents of the JSON document. The format should be the same as the value of the " + "\"configuration\" entry in status JSON without \"excluded_servers\" or \"coordinators_count\"."); helpMap["coordinators"] = CommandHelp( - "coordinators auto|
+ [description=new_cluster_description]", - "change cluster coordinators or description", - "If 'auto' is specified, coordinator addresses will be choosen automatically to support the configured redundancy level. (If the current set of coordinators are healthy and already support the redundancy level, nothing will be changed.)\n\nOtherwise, sets the coordinators to the list of IP:port pairs specified by
+. An fdbserver process must be running on each of the specified addresses.\n\ne.g. coordinators 10.0.0.1:4000 10.0.0.2:4000 10.0.0.3:4000\n\nIf 'description=desc' is specified then the description field in the cluster\nfile is changed to desc, which must match [A-Za-z0-9_]+."); + "coordinators auto|
+ [description=new_cluster_description]", + "change cluster coordinators or description", + "If 'auto' is specified, coordinator addresses will be choosen automatically to support the configured " + "redundancy level. (If the current set of coordinators are healthy and already support the redundancy level, " + "nothing will be changed.)\n\nOtherwise, sets the coordinators to the list of IP:port pairs specified by " + "
+. An fdbserver process must be running on each of the specified addresses.\n\ne.g. coordinators " + "10.0.0.1:4000 10.0.0.2:4000 10.0.0.3:4000\n\nIf 'description=desc' is specified then the description field in " + "the cluster\nfile is changed to desc, which must match [A-Za-z0-9_]+."); helpMap["exclude"] = - CommandHelp("exclude [no_wait]
*", "exclude servers from the database", + CommandHelp("exclude [no_wait]
*", + "exclude servers from the database", "If no addresses are specified, lists the set of excluded servers.\n\nFor each IP address or " "IP:port pair in
*, adds the address to the set of excluded servers then waits until all " "database state has been safely moved away from the specified servers. If 'no_wait' is set, the " "command returns \nimmediately without checking if the exclusions have completed successfully."); - helpMap["include"] = CommandHelp( - "include all|
*", - "permit previously-excluded servers to rejoin the database", - "If `all' is specified, the excluded servers list is cleared.\n\nFor each IP address or IP:port pair in
*, removes any matching exclusions from the excluded servers list. (A specified IP will match all IP:* exclusion entries)"); - helpMap["setclass"] = CommandHelp( - "setclass
", - "change the class of a process", - "If no address and class are specified, lists the classes of all servers.\n\nSetting the class to `default' resets the process class to the class specified on the command line."); - helpMap["status"] = CommandHelp( - "status [minimal] [details] [json]", - "get the status of a FoundationDB cluster", - "If the cluster is down, this command will print a diagnostic which may be useful in figuring out what is wrong. If the cluster is running, this command will print cluster statistics.\n\nSpecifying 'minimal' will provide a minimal description of the status of your database.\n\nSpecifying 'details' will provide load information for individual workers.\n\nSpecifying 'json' will provide status information in a machine readable JSON format."); + helpMap["include"] = + CommandHelp("include all|
*", + "permit previously-excluded servers to rejoin the database", + "If `all' is specified, the excluded servers list is cleared.\n\nFor each IP address or IP:port " + "pair in
*, removes any matching exclusions from the excluded servers list. (A specified " + "IP will match all IP:* exclusion entries)"); + helpMap["setclass"] = + CommandHelp("setclass
", + "change the class of a process", + "If no address and class are specified, lists the classes of all servers.\n\nSetting the class to " + "`default' resets the process class to the class specified on the command line."); + helpMap["status"] = + CommandHelp("status [minimal] [details] [json]", + "get the status of a FoundationDB cluster", + "If the cluster is down, this command will print a diagnostic which may be useful in figuring out " + "what is wrong. If the cluster is running, this command will print cluster " + "statistics.\n\nSpecifying 'minimal' will provide a minimal description of the status of your " + "database.\n\nSpecifying 'details' will provide load information for individual " + "workers.\n\nSpecifying 'json' will provide status information in a machine readable JSON format."); helpMap["exit"] = CommandHelp("exit", "exit the CLI", ""); helpMap["quit"] = CommandHelp(); helpMap["waitconnected"] = CommandHelp(); helpMap["waitopen"] = CommandHelp(); - helpMap["sleep"] = CommandHelp( - "sleep ", - "sleep for a period of time", - ""); - helpMap["get"] = CommandHelp( - "get ", - "fetch the value for a given key", - "Displays the value of KEY in the database, or `not found' if KEY is not present." ESCAPINGK); - helpMap["getrange"] = CommandHelp( - "getrange [ENDKEY] [LIMIT]", - "fetch key/value pairs in a range of keys", - "Displays up to LIMIT keys and values for keys between BEGINKEY (inclusive) and ENDKEY (exclusive). If ENDKEY is omitted, then the range will include all keys starting with BEGINKEY. LIMIT defaults to 25 if omitted." ESCAPINGK); + helpMap["sleep"] = CommandHelp("sleep ", "sleep for a period of time", ""); + helpMap["get"] = + CommandHelp("get ", + "fetch the value for a given key", + "Displays the value of KEY in the database, or `not found' if KEY is not present." ESCAPINGK); + helpMap["getrange"] = + CommandHelp("getrange [ENDKEY] [LIMIT]", + "fetch key/value pairs in a range of keys", + "Displays up to LIMIT keys and values for keys between BEGINKEY (inclusive) and ENDKEY " + "(exclusive). If ENDKEY is omitted, then the range will include all keys starting with BEGINKEY. " + "LIMIT defaults to 25 if omitted." ESCAPINGK); helpMap["getrangekeys"] = CommandHelp( - "getrangekeys [ENDKEY] [LIMIT]", - "fetch keys in a range of keys", - "Displays up to LIMIT keys for keys between BEGINKEY (inclusive) and ENDKEY (exclusive). If ENDKEY is omitted, then the range will include all keys starting with BEGINKEY. LIMIT defaults to 25 if omitted." ESCAPINGK); + "getrangekeys [ENDKEY] [LIMIT]", + "fetch keys in a range of keys", + "Displays up to LIMIT keys for keys between BEGINKEY (inclusive) and ENDKEY (exclusive). If ENDKEY is omitted, " + "then the range will include all keys starting with BEGINKEY. LIMIT defaults to 25 if omitted." ESCAPINGK); helpMap["getversion"] = - CommandHelp("getversion", "Fetch the current read version", + CommandHelp("getversion", + "Fetch the current read version", "Displays the current read version of the database or currently running transaction."); helpMap["advanceversion"] = CommandHelp( - "advanceversion ", "Force the cluster to recover at the specified version", + "advanceversion ", + "Force the cluster to recover at the specified version", "Forces the cluster to recover at the specified version. If the specified version is larger than the current " "version of the cluster, the cluster version is advanced " "to the specified version via a forced recovery."); - helpMap["reset"] = CommandHelp( - "reset", - "reset the current transaction", - "Any sets or clears executed after the start of the active transaction will be discarded."); - helpMap["rollback"] = CommandHelp( - "rollback", - "rolls back the current transaction", - "The active transaction will be discarded, including any sets or clears executed since the transaction was started."); - helpMap["set"] = CommandHelp( - "set ", - "set a value for a given key", - "If KEY is not already present in the database, it will be created." ESCAPINGKV); + helpMap["reset"] = + CommandHelp("reset", + "reset the current transaction", + "Any sets or clears executed after the start of the active transaction will be discarded."); + helpMap["rollback"] = CommandHelp("rollback", + "rolls back the current transaction", + "The active transaction will be discarded, including any sets or clears executed " + "since the transaction was started."); + helpMap["set"] = CommandHelp("set ", + "set a value for a given key", + "If KEY is not already present in the database, it will be created." ESCAPINGKV); helpMap["option"] = CommandHelp( - "option