From cc7fed6ed466b7216d8b08e6a1d8bebf94852965 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Wed, 3 Mar 2021 22:35:53 +0000 Subject: [PATCH 1/4] Change default for TLS_SERVER_HANDSHAKE_THREADS to 0 --- .../sphinx/source/release-notes/release-notes-630.rst | 4 ++++ flow/Knobs.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/documentation/sphinx/source/release-notes/release-notes-630.rst b/documentation/sphinx/source/release-notes/release-notes-630.rst index 7ca1a2e721..4157325497 100644 --- a/documentation/sphinx/source/release-notes/release-notes-630.rst +++ b/documentation/sphinx/source/release-notes/release-notes-630.rst @@ -4,6 +4,10 @@ Release Notes ############# +6.3.12 +====== +* Change the default for --knob_tls_server_handshake_threads to 0. The previous was 1000. This avoids starting 1000 threads by default, but may adversely affect recovery time for large clusters using tls. Users with large tls clusters should consider explicitly setting this knob in their foundationdb.conf file. `(PR #4421) `_ + 6.3.11 ====== * Added a hint field in the trace event when all replicas of some data are lost. `(PR #4209) `_ diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index e4d72a7f58..bb32483eb2 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -92,7 +92,7 @@ void FlowKnobs::initialize(bool randomize, bool isSimulated) { init( TLS_SERVER_CONNECTION_THROTTLE_ATTEMPTS, 1 ); init( TLS_CLIENT_CONNECTION_THROTTLE_ATTEMPTS, 1 ); init( TLS_CLIENT_HANDSHAKE_THREADS, 0 ); - init( TLS_SERVER_HANDSHAKE_THREADS, 1000 ); + init( TLS_SERVER_HANDSHAKE_THREADS, 0 ); init( TLS_HANDSHAKE_THREAD_STACKSIZE, 64 * 1024 ); init( TLS_MALLOC_ARENA_MAX, 6 ); init( TLS_HANDSHAKE_LIMIT, 1000 ); From 23b85256ae9a378e8e97f480bcb05c3e05385979 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Thu, 4 Mar 2021 20:51:24 +0000 Subject: [PATCH 2/4] Change default to 64, and only start threads if using TLS --- documentation/sphinx/source/release-notes/release-notes-630.rst | 2 +- flow/Knobs.cpp | 2 +- flow/Net2.actor.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/sphinx/source/release-notes/release-notes-630.rst b/documentation/sphinx/source/release-notes/release-notes-630.rst index 4157325497..3cd90e61dc 100644 --- a/documentation/sphinx/source/release-notes/release-notes-630.rst +++ b/documentation/sphinx/source/release-notes/release-notes-630.rst @@ -6,7 +6,7 @@ Release Notes 6.3.12 ====== -* Change the default for --knob_tls_server_handshake_threads to 0. The previous was 1000. This avoids starting 1000 threads by default, but may adversely affect recovery time for large clusters using tls. Users with large tls clusters should consider explicitly setting this knob in their foundationdb.conf file. `(PR #4421) `_ +* Change the default for --knob_tls_server_handshake_threads to 64. The previous was 1000. This avoids starting 1000 threads by default, but may adversely affect recovery time for large clusters using tls. Users with large tls clusters should consider explicitly setting this knob in their foundationdb.conf file. `(PR #4421) `_ 6.3.11 ====== diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index bb32483eb2..6db7034dfe 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -92,7 +92,7 @@ void FlowKnobs::initialize(bool randomize, bool isSimulated) { init( TLS_SERVER_CONNECTION_THROTTLE_ATTEMPTS, 1 ); init( TLS_CLIENT_CONNECTION_THROTTLE_ATTEMPTS, 1 ); init( TLS_CLIENT_HANDSHAKE_THREADS, 0 ); - init( TLS_SERVER_HANDSHAKE_THREADS, 0 ); + init( TLS_SERVER_HANDSHAKE_THREADS, 64 ); init( TLS_HANDSHAKE_THREAD_STACKSIZE, 64 * 1024 ); init( TLS_MALLOC_ARENA_MAX, 6 ); init( TLS_HANDSHAKE_LIMIT, 1000 ); diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index f043ae4114..1f34749dff 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -1601,8 +1601,8 @@ bool Net2::isAddressOnThisHost( NetworkAddress const& addr ) { Reference Net2::listen( NetworkAddress localAddr ) { try { #ifndef TLS_DISABLED - initTLS(ETLSInitState::LISTEN); if ( localAddr.isTLS() ) { + initTLS(ETLSInitState::LISTEN); return Reference(new SSLListener( reactor.ios, &this->sslContextVar, localAddr )); } #endif From 8a8c488ede29c0a24c2dc9841d08fa33dd1e5d5c Mon Sep 17 00:00:00 2001 From: FDB Formatster Date: Fri, 5 Mar 2021 18:13:38 -0600 Subject: [PATCH 3/4] apply clang-format to *.c, *.cpp, *.h, *.hpp files --- .clang-format | 8 +- FDBLibTLS/FDBLibTLSPlugin.cpp | 7 +- FDBLibTLS/FDBLibTLSPlugin.h | 2 +- FDBLibTLS/FDBLibTLSPolicy.cpp | 71 +- FDBLibTLS/FDBLibTLSPolicy.h | 10 +- FDBLibTLS/FDBLibTLSSession.cpp | 106 +- FDBLibTLS/FDBLibTLSSession.h | 15 +- FDBLibTLS/FDBLibTLSVerify.cpp | 43 +- FDBLibTLS/FDBLibTLSVerify.h | 20 +- FDBLibTLS/plugin-test.cpp | 1138 +- FDBLibTLS/verify-test.cpp | 140 +- bindings/c/ThreadCleanup.cpp | 10 +- bindings/c/fdb_c.cpp | 740 +- bindings/c/foundationdb/ClientWorkload.h | 3 +- bindings/c/test/mako/mako.c | 162 +- bindings/c/test/performance_test.c | 336 +- bindings/c/test/ryw_benchmark.c | 156 +- bindings/c/test/test.h | 109 +- bindings/c/test/txn_size_test.c | 16 +- bindings/c/test/workloads/SimpleWorkload.cpp | 22 +- bindings/c/test/workloads/workloads.cpp | 20 +- bindings/c/test/workloads/workloads.h | 10 +- bindings/flow/DirectoryLayer.actor.cpp | 901 +- bindings/flow/DirectoryLayer.h | 154 +- bindings/flow/DirectoryPartition.h | 50 +- bindings/flow/DirectorySubspace.cpp | 165 +- bindings/flow/DirectorySubspace.h | 64 +- bindings/flow/FDBLoanerTypes.h | 576 +- .../flow/HighContentionAllocator.actor.cpp | 152 +- bindings/flow/HighContentionAllocator.h | 21 +- bindings/flow/IDirectory.h | 54 +- bindings/flow/Node.actor.cpp | 73 +- bindings/flow/Subspace.cpp | 117 +- bindings/flow/Subspace.h | 98 +- bindings/flow/Tuple.cpp | 1147 +- bindings/flow/Tuple.h | 143 +- bindings/flow/fdb_flow.actor.cpp | 692 +- bindings/flow/fdb_flow.h | 206 +- .../flow/tester/DirectoryTester.actor.cpp | 201 +- bindings/flow/tester/Tester.actor.cpp | 479 +- bindings/flow/tester/Tester.actor.h | 84 +- bindings/java/JavaWorkload.cpp | 36 +- bindings/java/fdbJNI.cpp | 948 +- contrib/monitoring/actor_flamegraph.cpp | 7 +- fdbbackup/FileConverter.actor.cpp | 49 +- fdbbackup/FileConverter.h | 4 +- fdbbackup/FileDecoder.actor.cpp | 31 +- fdbbackup/backup.actor.cpp | 3028 ++-- fdbcli/FlowLineNoise.actor.cpp | 235 +- fdbcli/FlowLineNoise.h | 52 +- fdbcli/fdbcli.actor.cpp | 2436 +-- fdbcli/linenoise/linenoise.c | 1604 +- fdbcli/linenoise/linenoise.h | 28 +- fdbclient/AsyncFileBlobStore.actor.cpp | 30 +- fdbclient/AsyncFileBlobStore.actor.h | 138 +- fdbclient/Atomic.h | 140 +- fdbclient/AutoPublicAddress.cpp | 3 +- fdbclient/BackupAgent.actor.h | 545 +- fdbclient/BackupAgentBase.actor.cpp | 474 +- fdbclient/BackupContainer.actor.cpp | 1025 +- fdbclient/BackupContainer.h | 78 +- fdbclient/BlobStore.actor.cpp | 712 +- fdbclient/BlobStore.h | 174 +- fdbclient/ClientLogEvents.h | 739 +- fdbclient/ClientWorkerInterface.h | 28 +- fdbclient/ClusterInterface.h | 103 +- fdbclient/CommitTransaction.h | 47 +- fdbclient/CoordinationInterface.h | 63 +- fdbclient/DatabaseBackupAgent.actor.cpp | 3932 +++-- fdbclient/DatabaseConfiguration.cpp | 372 +- fdbclient/DatabaseConfiguration.h | 52 +- fdbclient/DatabaseContext.h | 130 +- fdbclient/EventTypes.actor.h | 13 +- fdbclient/FDBOptions.h | 43 +- fdbclient/FDBTypes.h | 605 +- fdbclient/FileBackupAgent.actor.cpp | 7231 +++++---- fdbclient/HTTP.actor.cpp | 875 +- fdbclient/HTTP.h | 56 +- fdbclient/IClientApi.h | 41 +- fdbclient/JSONDoc.h | 140 +- fdbclient/JsonBuilder.cpp | 65 +- fdbclient/JsonBuilder.h | 225 +- fdbclient/KeyBackedTypes.h | 239 +- fdbclient/KeyRangeMap.actor.cpp | 204 +- fdbclient/KeyRangeMap.h | 313 +- fdbclient/Knobs.cpp | 2 +- fdbclient/Knobs.h | 13 +- fdbclient/ManagementAPI.actor.cpp | 1350 +- fdbclient/ManagementAPI.actor.h | 114 +- fdbclient/MasterProxyInterface.h | 224 +- fdbclient/MetricLogger.actor.cpp | 271 +- fdbclient/MetricLogger.h | 2 +- fdbclient/MonitorLeader.actor.cpp | 443 +- fdbclient/MonitorLeader.h | 43 +- fdbclient/MultiVersionAssignmentVars.h | 157 +- fdbclient/MultiVersionTransaction.actor.cpp | 941 +- fdbclient/MultiVersionTransaction.h | 250 +- fdbclient/MutationList.h | 108 +- fdbclient/NativeAPI.actor.cpp | 3341 ++-- fdbclient/NativeAPI.actor.h | 151 +- fdbclient/Notified.h | 3 +- fdbclient/RYWIterator.cpp | 385 +- fdbclient/RYWIterator.h | 31 +- fdbclient/ReadYourWrites.actor.cpp | 1629 +- fdbclient/ReadYourWrites.h | 102 +- fdbclient/RestoreWorkerInterface.actor.h | 105 +- fdbclient/RunTransaction.actor.h | 29 +- fdbclient/SnapshotCache.h | 230 +- fdbclient/SpecialKeySpace.actor.cpp | 63 +- fdbclient/SpecialKeySpace.actor.h | 36 +- fdbclient/Status.h | 33 +- fdbclient/StatusClient.actor.cpp | 480 +- fdbclient/StorageServerInterface.h | 199 +- fdbclient/Subspace.cpp | 9 +- fdbclient/Subspace.h | 4 +- fdbclient/SystemData.cpp | 511 +- fdbclient/SystemData.h | 158 +- fdbclient/TagThrottle.actor.cpp | 485 +- fdbclient/TagThrottle.h | 83 +- fdbclient/TaskBucket.actor.cpp | 620 +- fdbclient/TaskBucket.h | 233 +- fdbclient/ThreadSafeTransaction.actor.cpp | 309 +- fdbclient/ThreadSafeTransaction.h | 73 +- fdbclient/Tuple.cpp | 105 +- fdbclient/Tuple.h | 2 +- fdbclient/VersionedMap.actor.h | 23 +- fdbclient/VersionedMap.cpp | 6 +- fdbclient/VersionedMap.h | 1235 +- fdbclient/WriteMap.h | 666 +- .../json_spirit/json_spirit_error_position.h | 56 +- .../json_spirit/json_spirit_reader_template.h | 1122 +- fdbclient/json_spirit/json_spirit_value.h | 955 +- .../json_spirit/json_spirit_writer_options.h | 36 +- .../json_spirit/json_spirit_writer_template.h | 590 +- fdbclient/libb64/cdecode.c | 51 +- fdbclient/libb64/cdecode.h | 8 +- fdbclient/libb64/cencode.c | 44 +- fdbclient/libb64/cencode.h | 8 +- fdbclient/libb64/decode.h | 84 +- fdbclient/libb64/encode.h | 106 +- fdbclient/md5/md5.c | 62 +- fdbclient/md5/md5.h | 8 +- fdbclient/rapidjson/allocators.h | 332 +- fdbclient/rapidjson/document.h | 4133 ++--- fdbclient/rapidjson/encodedstream.h | 432 +- fdbclient/rapidjson/encodings.h | 1016 +- fdbclient/rapidjson/error/en.h | 79 +- fdbclient/rapidjson/error/error.h | 89 +- fdbclient/rapidjson/filereadstream.h | 116 +- fdbclient/rapidjson/filewritestream.h | 122 +- fdbclient/rapidjson/fwd.h | 68 +- fdbclient/rapidjson/internal/biginteger.h | 449 +- fdbclient/rapidjson/internal/diyfp.h | 346 +- fdbclient/rapidjson/internal/dtoa.h | 388 +- fdbclient/rapidjson/internal/ieee754.h | 86 +- fdbclient/rapidjson/internal/itoa.h | 514 +- fdbclient/rapidjson/internal/meta.h | 201 +- fdbclient/rapidjson/internal/pow10.h | 57 +- fdbclient/rapidjson/internal/regex.h | 1052 +- fdbclient/rapidjson/internal/stack.h | 312 +- fdbclient/rapidjson/internal/strfunc.h | 47 +- fdbclient/rapidjson/internal/strtod.h | 384 +- fdbclient/rapidjson/internal/swap.h | 8 +- fdbclient/rapidjson/istreamwrapper.h | 103 +- fdbclient/rapidjson/memorybuffer.h | 43 +- fdbclient/rapidjson/memorystream.h | 57 +- fdbclient/rapidjson/msinttypes/inttypes.h | 403 +- fdbclient/rapidjson/msinttypes/stdint.h | 260 +- fdbclient/rapidjson/ostreamwrapper.h | 55 +- fdbclient/rapidjson/pointer.h | 2043 +-- fdbclient/rapidjson/prettywriter.h | 376 +- fdbclient/rapidjson/rapidjson.h | 185 +- fdbclient/rapidjson/reader.h | 2923 ++-- fdbclient/rapidjson/schema.h | 3086 ++-- fdbclient/rapidjson/stream.h | 121 +- fdbclient/rapidjson/stringbuffer.h | 94 +- fdbclient/rapidjson/writer.h | 943 +- fdbclient/rapidxml/rapidxml.hpp | 5449 ++++--- fdbclient/rapidxml/rapidxml_iterators.hpp | 225 +- fdbclient/rapidxml/rapidxml_print.hpp | 743 +- fdbclient/rapidxml/rapidxml_utils.hpp | 164 +- fdbclient/sha1/SHA1.cpp | 390 +- fdbclient/sha1/SHA1.h | 37 +- fdbclient/zipf.c | 91 +- fdbclient/zipf.h | 1 - fdbmonitor/ConvertUTF.h | 109 +- fdbmonitor/SimpleIni.h | 4721 +++--- fdbmonitor/fdbmonitor.cpp | 869 +- fdbrpc/ActorFuzz.actor.cpp | 830 +- fdbrpc/ActorFuzz.h | 10 +- fdbrpc/AsyncFileCached.actor.cpp | 154 +- fdbrpc/AsyncFileCached.actor.h | 269 +- fdbrpc/AsyncFileEIO.actor.h | 408 +- fdbrpc/AsyncFileKAIO.actor.h | 464 +- fdbrpc/AsyncFileNonDurable.actor.cpp | 11 +- fdbrpc/AsyncFileNonDurable.actor.h | 591 +- fdbrpc/AsyncFileReadAhead.actor.h | 95 +- fdbrpc/AsyncFileWinASIO.actor.h | 124 +- fdbrpc/AsyncFileWriteChecker.h | 64 +- fdbrpc/ContinuousSample.h | 44 +- fdbrpc/FailureMonitor.actor.cpp | 28 +- fdbrpc/FailureMonitor.h | 3 +- fdbrpc/FlowTests.actor.cpp | 354 +- fdbrpc/FlowTransport.actor.cpp | 785 +- fdbrpc/FlowTransport.h | 87 +- fdbrpc/HealthMonitor.h | 1 + fdbrpc/IAsyncFile.actor.cpp | 76 +- fdbrpc/IAsyncFile.h | 75 +- fdbrpc/IRateControl.h | 20 +- fdbrpc/LoadBalance.actor.h | 367 +- fdbrpc/LoadPlugin.h | 10 +- fdbrpc/Locality.cpp | 37 +- fdbrpc/Locality.h | 131 +- fdbrpc/MultiInterface.h | 166 +- fdbrpc/Net2FileSystem.cpp | 47 +- fdbrpc/Net2FileSystem.h | 15 +- fdbrpc/PerfMetric.h | 26 +- fdbrpc/QueueModel.cpp | 55 +- fdbrpc/QueueModel.h | 2 +- fdbrpc/RangeMap.h | 148 +- fdbrpc/Replication.h | 426 +- fdbrpc/ReplicationPolicy.cpp | 312 +- fdbrpc/ReplicationPolicy.h | 27 +- fdbrpc/ReplicationTypes.cpp | 2 +- fdbrpc/ReplicationTypes.h | 132 +- fdbrpc/ReplicationUtils.cpp | 747 +- fdbrpc/ReplicationUtils.h | 90 +- fdbrpc/Smoother.h | 42 +- fdbrpc/Stats.actor.cpp | 26 +- fdbrpc/Stats.h | 100 +- fdbrpc/TraceFileIO.cpp | 140 +- fdbrpc/TraceFileIO.h | 11 +- fdbrpc/batcher.actor.h | 64 +- fdbrpc/dsltest.actor.cpp | 713 +- fdbrpc/fdbrpc.h | 193 +- fdbrpc/genericactors.actor.cpp | 8 +- fdbrpc/genericactors.actor.h | 120 +- fdbrpc/libcoroutine/386-ucontext.h | 72 +- fdbrpc/libcoroutine/Common.c | 214 +- fdbrpc/libcoroutine/Common.h | 107 +- fdbrpc/libcoroutine/Coro.c | 493 +- fdbrpc/libcoroutine/Coro.h | 74 +- fdbrpc/libcoroutine/amd64-ucontext.h | 99 +- fdbrpc/libcoroutine/context.c | 53 +- fdbrpc/libcoroutine/power-ucontext.h | 43 +- fdbrpc/libcoroutine/taskimpl.h | 124 +- fdbrpc/libeio/ecb.h | 533 +- fdbrpc/libeio/eio.c | 3621 +++-- fdbrpc/libeio/eio.h | 453 +- fdbrpc/libeio/xthread.h | 139 +- fdbrpc/linux_kaio.h | 21 +- fdbrpc/networksender.actor.h | 12 +- fdbrpc/sim2.actor.cpp | 1683 ++- fdbrpc/sim_validation.cpp | 74 +- fdbrpc/simulator.h | 252 +- fdbrpc/zlib/adler32.c | 280 +- fdbrpc/zlib/crc32.c | 527 +- fdbrpc/zlib/crc32.h | 693 +- fdbrpc/zlib/deflate.c | 2769 ++-- fdbrpc/zlib/deflate.h | 413 +- fdbrpc/zlib/gzclose.c | 15 +- fdbrpc/zlib/gzguts.h | 220 +- fdbrpc/zlib/gzlib.c | 845 +- fdbrpc/zlib/gzread.c | 892 +- fdbrpc/zlib/gzwrite.c | 947 +- fdbrpc/zlib/infback.c | 983 +- fdbrpc/zlib/inffast.c | 485 +- fdbrpc/zlib/inffixed.h | 181 +- fdbrpc/zlib/inflate.c | 2205 +-- fdbrpc/zlib/inflate.h | 146 +- fdbrpc/zlib/inftrees.c | 496 +- fdbrpc/zlib/inftrees.h | 23 +- fdbrpc/zlib/trees.c | 1545 +- fdbrpc/zlib/trees.h | 211 +- fdbrpc/zlib/zconf.h | 638 +- fdbrpc/zlib/zlib.h | 400 +- fdbrpc/zlib/zutil.c | 392 +- fdbrpc/zlib/zutil.h | 272 +- fdbserver/ApplyMetadataMutation.cpp | 460 +- fdbserver/ApplyMetadataMutation.h | 34 +- fdbserver/ArtMutationBuffer.h | 208 +- fdbserver/BackupInterface.h | 10 +- fdbserver/BackupProgress.actor.cpp | 18 +- fdbserver/BackupProgress.actor.h | 14 +- fdbserver/BackupWorker.actor.cpp | 104 +- fdbserver/ClusterController.actor.cpp | 2309 +-- fdbserver/CompactMap.cpp | 422 +- fdbserver/ConflictSet.h | 13 +- fdbserver/CoordinatedState.actor.cpp | 337 +- fdbserver/CoordinatedState.h | 16 +- fdbserver/Coordination.actor.cpp | 452 +- fdbserver/CoordinationInterface.h | 65 +- fdbserver/CoroFlow.actor.cpp | 144 +- fdbserver/CoroFlow.h | 18 +- fdbserver/DBCoreState.h | 67 +- fdbserver/DataDistribution.actor.cpp | 1970 +-- fdbserver/DataDistribution.actor.h | 117 +- fdbserver/DataDistributionQueue.actor.cpp | 1263 +- fdbserver/DataDistributionTracker.actor.cpp | 656 +- fdbserver/DataDistributorInterface.h | 39 +- fdbserver/DeltaTree.h | 12 +- fdbserver/DiskQueue.actor.cpp | 1022 +- fdbserver/FDBExecHelper.actor.cpp | 56 +- fdbserver/FDBExecHelper.actor.h | 12 +- fdbserver/IDiskQueue.h | 67 +- fdbserver/IKeyValueStore.h | 103 +- .../KeyValueStoreCompressTestData.actor.cpp | 96 +- fdbserver/KeyValueStoreMemory.actor.cpp | 356 +- fdbserver/KeyValueStoreRocksDB.actor.cpp | 64 +- fdbserver/KeyValueStoreSQLite.actor.cpp | 1319 +- fdbserver/Knobs.cpp | 9 +- fdbserver/Knobs.h | 88 +- fdbserver/LatencyBandConfig.cpp | 44 +- fdbserver/LeaderElection.actor.cpp | 188 +- fdbserver/LeaderElection.h | 36 +- fdbserver/LogProtocolMessage.h | 11 +- fdbserver/LogRouter.actor.cpp | 349 +- fdbserver/LogSystem.h | 554 +- fdbserver/LogSystemConfig.h | 195 +- fdbserver/LogSystemDiskQueueAdapter.actor.cpp | 134 +- fdbserver/LogSystemDiskQueueAdapter.h | 44 +- fdbserver/LogSystemPeekCursor.actor.cpp | 677 +- fdbserver/MasterInterface.h | 62 +- fdbserver/MasterProxyServer.actor.cpp | 1405 +- fdbserver/MoveKeys.actor.cpp | 858 +- fdbserver/MoveKeys.actor.h | 26 +- fdbserver/NetworkTest.h | 14 +- fdbserver/OldTLogServer_4_6.actor.cpp | 2733 ++-- fdbserver/OldTLogServer_6_0.actor.cpp | 1916 ++- fdbserver/OldTLogServer_6_2.actor.cpp | 2116 +-- fdbserver/Orderer.actor.h | 48 +- fdbserver/QuietDatabase.actor.cpp | 412 +- fdbserver/QuietDatabase.h | 27 +- fdbserver/Ratekeeper.actor.cpp | 829 +- fdbserver/RatekeeperInterface.h | 29 +- fdbserver/RecoveryState.h | 64 +- fdbserver/Resolver.actor.cpp | 254 +- fdbserver/ResolverInterface.h | 42 +- fdbserver/RestoreApplier.actor.cpp | 110 +- fdbserver/RestoreApplier.actor.h | 13 +- fdbserver/RestoreCommon.actor.cpp | 57 +- fdbserver/RestoreCommon.actor.h | 54 +- fdbserver/RestoreController.actor.cpp | 113 +- fdbserver/RestoreController.actor.h | 9 +- fdbserver/RestoreLoader.actor.cpp | 182 +- fdbserver/RestoreLoader.actor.h | 14 +- fdbserver/RestoreRoleCommon.actor.cpp | 8 +- fdbserver/RestoreRoleCommon.actor.h | 8 +- fdbserver/RestoreWorker.actor.cpp | 50 +- fdbserver/RestoreWorker.actor.h | 7 +- fdbserver/ServerDBInfo.actor.h | 56 +- fdbserver/SimulatedCluster.actor.cpp | 1139 +- fdbserver/SimulatedCluster.h | 6 +- fdbserver/SkipList.cpp | 214 +- fdbserver/Status.actor.cpp | 1362 +- fdbserver/Status.h | 15 +- fdbserver/StorageCache.actor.cpp | 677 +- fdbserver/StorageMetrics.actor.h | 327 +- fdbserver/StorageMetrics.h | 4 +- fdbserver/TLogInterface.h | 122 +- fdbserver/TLogServer.actor.cpp | 2080 +-- fdbserver/TagPartitionedLogSystem.actor.cpp | 2292 +-- fdbserver/TesterInterface.actor.h | 53 +- fdbserver/VFSAsync.cpp | 898 +- fdbserver/VFSAsync.h | 86 +- fdbserver/VersionedBTree.actor.cpp | 1289 +- fdbserver/WaitFailure.actor.cpp | 52 +- fdbserver/WaitFailure.h | 19 +- fdbserver/WorkerInterface.actor.h | 461 +- fdbserver/art.h | 432 +- fdbserver/art_impl.h | 2651 ++-- fdbserver/fdbserver.actor.cpp | 796 +- fdbserver/masterserver.actor.cpp | 1249 +- fdbserver/networktest.actor.cpp | 225 +- fdbserver/pubsub.actor.cpp | 181 +- fdbserver/pubsub.h | 3 +- fdbserver/sqlite/btree.c | 12581 ++++++++-------- fdbserver/sqlite/btree.h | 189 +- fdbserver/sqlite/hash.h | 35 +- fdbserver/sqlite/shell.c | 4699 +++--- fdbserver/sqlite/sqlite3.h | 1722 +-- fdbserver/sqlite/sqlite3ext.h | 755 +- fdbserver/sqlite/sqliteInt.h | 1182 +- fdbserver/sqlite/sqliteLimit.h | 69 +- fdbserver/storageserver.actor.cpp | 2933 ++-- fdbserver/template_fdb.h | 3180 ++-- fdbserver/tester.actor.cpp | 959 +- fdbserver/worker.actor.cpp | 1233 +- fdbserver/workloads/ApiCorrectness.actor.cpp | 484 +- fdbserver/workloads/ApiWorkload.actor.cpp | 200 +- fdbserver/workloads/ApiWorkload.h | 313 +- fdbserver/workloads/AsyncFile.actor.h | 82 +- fdbserver/workloads/AsyncFile.cpp | 78 +- .../workloads/AsyncFileCorrectness.actor.cpp | 394 +- fdbserver/workloads/AsyncFileRead.actor.cpp | 220 +- fdbserver/workloads/AsyncFileWrite.actor.cpp | 113 +- fdbserver/workloads/AtomicOps.actor.cpp | 141 +- .../AtomicOpsApiCorrectness.actor.cpp | 367 +- fdbserver/workloads/AtomicRestore.actor.cpp | 55 +- .../workloads/AtomicSwitchover.actor.cpp | 102 +- .../workloads/BackgroundSelectors.actor.cpp | 157 +- ...kupAndParallelRestoreCorrectness.actor.cpp | 169 +- .../workloads/BackupCorrectness.actor.cpp | 520 +- fdbserver/workloads/BackupToDBAbort.actor.cpp | 33 +- .../workloads/BackupToDBCorrectness.actor.cpp | 449 +- .../workloads/BackupToDBUpgrade.actor.cpp | 264 +- fdbserver/workloads/BulkLoad.actor.cpp | 78 +- fdbserver/workloads/BulkSetup.actor.h | 235 +- fdbserver/workloads/ChangeConfig.actor.cpp | 71 +- ...entTransactionProfileCorrectness.actor.cpp | 326 +- fdbserver/workloads/CommitBugCheck.actor.cpp | 91 +- .../workloads/ConfigureDatabase.actor.cpp | 213 +- fdbserver/workloads/ConflictRange.actor.cpp | 334 +- .../workloads/ConsistencyCheck.actor.cpp | 1187 +- fdbserver/workloads/CpuProfiler.actor.cpp | 106 +- fdbserver/workloads/Cycle.actor.cpp | 164 +- fdbserver/workloads/DDBalance.actor.cpp | 200 +- fdbserver/workloads/DDMetrics.actor.cpp | 39 +- .../workloads/DDMetricsExclude.actor.cpp | 46 +- .../DataDistributionMetrics.actor.cpp | 14 +- .../DifferentClustersSameRV.actor.cpp | 6 +- fdbserver/workloads/DiskDurability.actor.cpp | 96 +- .../workloads/DiskDurabilityTest.actor.cpp | 123 +- fdbserver/workloads/Downgrade.actor.cpp | 6 +- fdbserver/workloads/DummyWorkload.actor.cpp | 18 +- .../workloads/ExternalWorkload.actor.cpp | 8 +- .../workloads/FastTriggeredWatches.actor.cpp | 117 +- fdbserver/workloads/FileSystem.actor.cpp | 285 +- fdbserver/workloads/Fuzz.cpp | 16 +- .../workloads/FuzzApiCorrectness.actor.cpp | 737 +- fdbserver/workloads/Increment.actor.cpp | 127 +- fdbserver/workloads/IndexScan.actor.cpp | 88 +- fdbserver/workloads/Inventory.actor.cpp | 178 +- fdbserver/workloads/KVStoreTest.actor.cpp | 49 +- fdbserver/workloads/KillRegion.actor.cpp | 81 +- fdbserver/workloads/LocalRatekeeper.actor.cpp | 29 +- fdbserver/workloads/LockDatabase.actor.cpp | 84 +- .../LockDatabaseFrequently.actor.cpp | 4 +- fdbserver/workloads/LogMetrics.actor.cpp | 55 +- fdbserver/workloads/LowLatency.actor.cpp | 45 +- .../workloads/MachineAttrition.actor.cpp | 211 +- fdbserver/workloads/Mako.actor.cpp | 322 +- fdbserver/workloads/MemoryKeyValueStore.cpp | 72 +- fdbserver/workloads/MemoryKeyValueStore.h | 24 +- fdbserver/workloads/MemoryLifetime.actor.cpp | 129 +- fdbserver/workloads/MetricLogging.actor.cpp | 59 +- fdbserver/workloads/ParallelRestore.actor.cpp | 4 +- fdbserver/workloads/Performance.actor.cpp | 173 +- fdbserver/workloads/Ping.actor.cpp | 211 +- fdbserver/workloads/PopulateTPCC.actor.cpp | 7 +- fdbserver/workloads/PubSubMultiples.actor.cpp | 87 +- fdbserver/workloads/QueuePush.actor.cpp | 92 +- fdbserver/workloads/RYWDisable.actor.cpp | 86 +- fdbserver/workloads/RYWPerformance.actor.cpp | 293 +- fdbserver/workloads/RandomClogging.actor.cpp | 82 +- fdbserver/workloads/RandomMoveKeys.actor.cpp | 148 +- fdbserver/workloads/RandomSelector.actor.cpp | 465 +- .../workloads/ReadHotDetection.actor.cpp | 11 +- fdbserver/workloads/ReadWrite.actor.cpp | 592 +- .../workloads/RemoveServersSafely.actor.cpp | 428 +- .../workloads/ReportConflictingKeys.actor.cpp | 27 +- fdbserver/workloads/Rollback.actor.cpp | 76 +- fdbserver/workloads/RyowCorrectness.actor.cpp | 308 +- fdbserver/workloads/SaveAndKill.actor.cpp | 85 +- .../workloads/SelectorCorrectness.actor.cpp | 202 +- fdbserver/workloads/Serializability.actor.cpp | 396 +- fdbserver/workloads/Sideband.actor.cpp | 123 +- .../workloads/SlowTaskWorkload.actor.cpp | 36 +- fdbserver/workloads/SnapTest.actor.cpp | 24 +- .../SpecialKeySpaceCorrectness.actor.cpp | 44 +- fdbserver/workloads/StatusWorkload.actor.cpp | 99 +- fdbserver/workloads/Storefront.actor.cpp | 224 +- fdbserver/workloads/StreamingRead.actor.cpp | 116 +- .../workloads/SuspendProcesses.actor.cpp | 6 +- fdbserver/workloads/TPCC.actor.cpp | 96 +- fdbserver/workloads/TPCCWorkload.h | 427 +- fdbserver/workloads/TagThrottleApi.actor.cpp | 137 +- fdbserver/workloads/TargetedKill.actor.cpp | 86 +- .../workloads/TaskBucketCorrectness.actor.cpp | 185 +- fdbserver/workloads/ThreadSafety.actor.cpp | 142 +- fdbserver/workloads/Throttling.actor.cpp | 61 +- fdbserver/workloads/Throughput.actor.cpp | 324 +- .../workloads/TimeKeeperCorrectness.actor.cpp | 56 +- fdbserver/workloads/TriggerRecovery.actor.cpp | 14 +- fdbserver/workloads/UnitPerf.actor.cpp | 20 +- fdbserver/workloads/UnitTests.actor.cpp | 21 +- fdbserver/workloads/Unreadable.actor.cpp | 158 +- fdbserver/workloads/VersionStamp.actor.cpp | 142 +- fdbserver/workloads/WatchAndWait.actor.cpp | 91 +- fdbserver/workloads/Watches.actor.cpp | 171 +- fdbserver/workloads/WorkerErrors.actor.cpp | 39 +- fdbserver/workloads/WriteBandwidth.actor.cpp | 86 +- fdbserver/workloads/WriteDuringRead.actor.cpp | 1445 +- fdbserver/workloads/workloads.actor.h | 139 +- fdbservice/FDBService.cpp | 645 +- fdbservice/ServiceBase.cpp | 478 +- fdbservice/ServiceBase.h | 143 +- fdbservice/ThreadPool.h | 80 +- flow/ActorCollection.actor.cpp | 27 +- flow/ActorCollection.h | 38 +- flow/Arena.cpp | 12 +- flow/Arena.h | 428 +- flow/AsioReactor.h | 36 +- flow/CompressedInt.actor.cpp | 85 +- flow/CompressedInt.h | 92 +- flow/Deque.cpp | 4 +- flow/Deque.h | 81 +- flow/DeterministicRandom.cpp | 21 +- flow/Error.cpp | 44 +- flow/Error.h | 208 +- flow/EventTypes.actor.h | 15 +- flow/FastAlloc.cpp | 223 +- flow/FastAlloc.h | 121 +- flow/FastRef.h | 70 +- flow/FaultInjection.cpp | 2 +- flow/FaultInjection.h | 17 +- flow/FileIdentifier.h | 3 +- flow/FileTraceLogWriter.cpp | 59 +- flow/FileTraceLogWriter.h | 10 +- flow/Hash3.c | 1716 ++- flow/Hash3.h | 12 +- flow/Histogram.cpp | 8 +- flow/IDispatched.h | 29 +- flow/IRandom.h | 54 +- flow/IThreadPool.cpp | 58 +- flow/IThreadPool.h | 43 +- flow/IndexedSet.actor.h | 25 +- flow/IndexedSet.cpp | 166 +- flow/IndexedSet.h | 658 +- flow/Knobs.cpp | 40 +- flow/Knobs.h | 47 +- flow/MetricSample.h | 40 +- flow/Net2.actor.cpp | 928 +- flow/Net2Packet.cpp | 60 +- flow/Net2Packet.h | 30 +- flow/ObjectSerializer.h | 29 +- flow/Platform.cpp | 1720 ++- flow/Profiler.actor.cpp | 134 +- flow/Profiler.h | 2 +- flow/RadixTree.h | 235 +- flow/SignalSafeUnwind.cpp | 12 +- flow/SimpleOpt.h | 1327 +- flow/SystemMonitor.cpp | 133 +- flow/SystemMonitor.h | 31 +- flow/TDMetric.actor.h | 729 +- flow/TDMetric.cpp | 169 +- flow/TLSConfig.actor.cpp | 211 +- flow/TLSConfig.actor.h | 152 +- flow/ThreadHelper.actor.h | 289 +- flow/ThreadHelper.cpp | 2 +- flow/ThreadPrimitives.cpp | 12 +- flow/ThreadPrimitives.h | 26 +- flow/ThreadSafeQueue.h | 68 +- flow/Trace.cpp | 573 +- flow/Trace.h | 333 +- flow/TreeBenchmark.h | 5 +- flow/UnitTest.cpp | 3 +- flow/UnitTest.h | 37 +- flow/Util.h | 24 +- flow/XmlTraceLogFormatter.cpp | 37 +- flow/XmlTraceLogFormatter.h | 5 +- flow/actorcompiler.h | 20 +- flow/crc32c-generated-constants.cpp | 729 +- flow/crc32c.cpp | 426 +- flow/crc32c.h | 6 +- flow/flat_buffers.cpp | 33 +- flow/flat_buffers.h | 51 +- flow/flow.cpp | 87 +- flow/flow.h | 431 +- flow/genericactors.actor.cpp | 105 +- flow/genericactors.actor.h | 1115 +- flow/network.cpp | 41 +- flow/network.h | 162 +- flow/serialize.cpp | 8 +- flow/serialize.h | 366 +- flow/sse2neon.h | 3129 ++-- flow/stacktrace.amalgamation.cpp | 2011 ++- flow/stacktrace.h | 33 +- flow/xxhash.h | 418 +- 579 files changed, 124569 insertions(+), 108331 deletions(-) mode change 100755 => 100644 bindings/flow/DirectoryLayer.h mode change 100755 => 100644 bindings/flow/DirectorySubspace.cpp mode change 100755 => 100644 bindings/flow/DirectorySubspace.h mode change 100755 => 100644 bindings/flow/FDBLoanerTypes.h mode change 100755 => 100644 bindings/flow/IDirectory.h mode change 100755 => 100644 bindings/flow/Node.actor.cpp mode change 100755 => 100644 bindings/flow/Subspace.cpp mode change 100755 => 100644 bindings/flow/Subspace.h mode change 100755 => 100644 bindings/flow/Tuple.cpp mode change 100755 => 100644 bindings/flow/Tuple.h mode change 100755 => 100644 fdbclient/JsonBuilder.cpp mode change 100755 => 100644 fdbclient/MetricLogger.h mode change 100755 => 100644 fdbclient/json_spirit/json_spirit_error_position.h mode change 100755 => 100644 fdbclient/json_spirit/json_spirit_reader_template.h mode change 100755 => 100644 fdbclient/json_spirit/json_spirit_value.h mode change 100755 => 100644 fdbclient/json_spirit/json_spirit_writer_options.h mode change 100755 => 100644 fdbclient/json_spirit/json_spirit_writer_template.h mode change 100755 => 100644 fdbclient/libb64/cdecode.c mode change 100755 => 100644 fdbclient/libb64/cdecode.h mode change 100755 => 100644 fdbclient/libb64/cencode.h mode change 100755 => 100644 fdbclient/libb64/decode.h mode change 100755 => 100644 fdbclient/libb64/encode.h mode change 100755 => 100644 fdbclient/rapidjson/allocators.h mode change 100755 => 100644 fdbclient/rapidjson/document.h mode change 100755 => 100644 fdbclient/rapidjson/encodedstream.h mode change 100755 => 100644 fdbclient/rapidjson/encodings.h mode change 100755 => 100644 fdbclient/rapidjson/error/en.h mode change 100755 => 100644 fdbclient/rapidjson/error/error.h mode change 100755 => 100644 fdbclient/rapidjson/filereadstream.h mode change 100755 => 100644 fdbclient/rapidjson/filewritestream.h mode change 100755 => 100644 fdbclient/rapidjson/fwd.h mode change 100755 => 100644 fdbclient/rapidjson/internal/biginteger.h mode change 100755 => 100644 fdbclient/rapidjson/internal/diyfp.h mode change 100755 => 100644 fdbclient/rapidjson/internal/dtoa.h mode change 100755 => 100644 fdbclient/rapidjson/internal/ieee754.h mode change 100755 => 100644 fdbclient/rapidjson/internal/itoa.h mode change 100755 => 100644 fdbclient/rapidjson/internal/meta.h mode change 100755 => 100644 fdbclient/rapidjson/internal/pow10.h mode change 100755 => 100644 fdbclient/rapidjson/internal/regex.h mode change 100755 => 100644 fdbclient/rapidjson/internal/stack.h mode change 100755 => 100644 fdbclient/rapidjson/internal/strfunc.h mode change 100755 => 100644 fdbclient/rapidjson/internal/strtod.h mode change 100755 => 100644 fdbclient/rapidjson/internal/swap.h mode change 100755 => 100644 fdbclient/rapidjson/istreamwrapper.h mode change 100755 => 100644 fdbclient/rapidjson/memorybuffer.h mode change 100755 => 100644 fdbclient/rapidjson/memorystream.h mode change 100755 => 100644 fdbclient/rapidjson/msinttypes/inttypes.h mode change 100755 => 100644 fdbclient/rapidjson/msinttypes/stdint.h mode change 100755 => 100644 fdbclient/rapidjson/ostreamwrapper.h mode change 100755 => 100644 fdbclient/rapidjson/pointer.h mode change 100755 => 100644 fdbclient/rapidjson/prettywriter.h mode change 100755 => 100644 fdbclient/rapidjson/rapidjson.h mode change 100755 => 100644 fdbclient/rapidjson/reader.h mode change 100755 => 100644 fdbclient/rapidjson/schema.h mode change 100755 => 100644 fdbclient/rapidjson/stream.h mode change 100755 => 100644 fdbclient/rapidjson/stringbuffer.h mode change 100755 => 100644 fdbclient/rapidjson/writer.h mode change 100755 => 100644 fdbclient/rapidxml/rapidxml.hpp mode change 100755 => 100644 fdbclient/rapidxml/rapidxml_iterators.hpp mode change 100755 => 100644 fdbclient/rapidxml/rapidxml_print.hpp mode change 100755 => 100644 fdbclient/rapidxml/rapidxml_utils.hpp mode change 100755 => 100644 fdbserver/sqlite/sqlite3.h mode change 100755 => 100644 fdbserver/workloads/DiskDurability.actor.cpp mode change 100755 => 100644 fdbserver/workloads/DiskDurabilityTest.actor.cpp mode change 100755 => 100644 fdbserver/workloads/KVStoreTest.actor.cpp mode change 100755 => 100644 flow/TDMetric.actor.h diff --git a/.clang-format b/.clang-format index c7faa9e0e6..60b24f6172 100644 --- a/.clang-format +++ b/.clang-format @@ -11,14 +11,14 @@ AllowAllParametersOfDeclarationOnNextLine: false AllowShortBlocksOnASingleLine: false AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: Inline -AllowShortIfStatementsOnASingleLine: true -AllowShortLoopsOnASingleLine: true +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false AlwaysBreakAfterDefinitionReturnType: None AlwaysBreakAfterReturnType: None AlwaysBreakBeforeMultilineStrings: false AlwaysBreakTemplateDeclarations: true -BinPackArguments: true -BinPackParameters: true +BinPackArguments: false +BinPackParameters: false BreakBeforeBinaryOperators: None BreakBeforeBraces: Attach ColumnLimit: 120 diff --git a/FDBLibTLS/FDBLibTLSPlugin.cpp b/FDBLibTLS/FDBLibTLSPlugin.cpp index 6bcdb48600..1be61305d1 100644 --- a/FDBLibTLS/FDBLibTLSPlugin.cpp +++ b/FDBLibTLS/FDBLibTLSPlugin.cpp @@ -31,10 +31,9 @@ FDBLibTLSPlugin::FDBLibTLSPlugin() { rc = tls_init(); } -FDBLibTLSPlugin::~FDBLibTLSPlugin() { -} +FDBLibTLSPlugin::~FDBLibTLSPlugin() {} -ITLSPolicy *FDBLibTLSPlugin::create_policy() { +ITLSPolicy* FDBLibTLSPlugin::create_policy() { if (rc < 0) { // Log the failure from tls_init during our constructor. TraceEvent(SevError, "FDBLibTLSInitError").detail("LibTLSErrorMessage", "failed to initialize libtls"); @@ -43,7 +42,7 @@ ITLSPolicy *FDBLibTLSPlugin::create_policy() { return new FDBLibTLSPolicy(Reference::addRef(this)); } -extern "C" BOOST_SYMBOL_EXPORT void *get_tls_plugin(const char *plugin_type_name_and_version) { +extern "C" BOOST_SYMBOL_EXPORT void* get_tls_plugin(const char* plugin_type_name_and_version) { if (strcmp(plugin_type_name_and_version, FDBLibTLSPlugin::get_plugin_type_name_and_version()) == 0) { return new FDBLibTLSPlugin; } diff --git a/FDBLibTLS/FDBLibTLSPlugin.h b/FDBLibTLS/FDBLibTLSPlugin.h index a6ffa03e55..98f11d63e1 100644 --- a/FDBLibTLS/FDBLibTLSPlugin.h +++ b/FDBLibTLS/FDBLibTLSPlugin.h @@ -35,7 +35,7 @@ struct FDBLibTLSPlugin : ITLSPlugin, ReferenceCounted { virtual void addref() { ReferenceCounted::addref(); } virtual void delref() { ReferenceCounted::delref(); } - virtual ITLSPolicy *create_policy(); + virtual ITLSPolicy* create_policy(); int rc; }; diff --git a/FDBLibTLS/FDBLibTLSPolicy.cpp b/FDBLibTLS/FDBLibTLSPolicy.cpp index 1fb9f65277..b2438b9649 100644 --- a/FDBLibTLS/FDBLibTLSPolicy.cpp +++ b/FDBLibTLS/FDBLibTLSPolicy.cpp @@ -37,9 +37,9 @@ #include #include -FDBLibTLSPolicy::FDBLibTLSPolicy(Reference plugin): - plugin(plugin), tls_cfg(NULL), roots(NULL), session_created(false), ca_data_set(false), - cert_data_set(false), key_data_set(false), verify_peers_set(false) { +FDBLibTLSPolicy::FDBLibTLSPolicy(Reference plugin) + : plugin(plugin), tls_cfg(NULL), roots(NULL), session_created(false), ca_data_set(false), cert_data_set(false), + key_data_set(false), verify_peers_set(false) { if ((tls_cfg = tls_config_new()) == NULL) { TraceEvent(SevError, "FDBLibTLSConfigError"); @@ -55,7 +55,13 @@ FDBLibTLSPolicy::~FDBLibTLSPolicy() { tls_config_free(tls_cfg); } -ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid) { +ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, + const char* servername, + TLSSendCallbackFunc send_func, + void* send_ctx, + TLSRecvCallbackFunc recv_func, + void* recv_ctx, + void* uid) { if (is_client) { // If verify peers has been set then there is no point specifying a // servername, since this will be ignored - the servername should be @@ -75,14 +81,21 @@ ITLSSession* FDBLibTLSPolicy::create_session(bool is_client, const char* servern session_created = true; try { - return new FDBLibTLSSession(Reference::addRef(this), is_client, servername, send_func, send_ctx, recv_func, recv_ctx, uid); - } catch ( ... ) { + return new FDBLibTLSSession(Reference::addRef(this), + is_client, + servername, + send_func, + send_ctx, + recv_func, + recv_ctx, + uid); + } catch (...) { return NULL; } } -static int password_cb(char *buf, int size, int rwflag, void *u) { - const char *password = (const char *)u; +static int password_cb(char* buf, int size, int rwflag, void* u) { + const char* password = (const char*)u; int plen; if (size < 0) @@ -102,14 +115,14 @@ static int password_cb(char *buf, int size, int rwflag, void *u) { } struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, size_t cert_pem_len) { - struct stack_st_X509 *certs = NULL; - X509 *cert = NULL; - BIO *bio = NULL; + struct stack_st_X509* certs = NULL; + X509* cert = NULL; + BIO* bio = NULL; int errnum; if (cert_pem_len > INT_MAX) goto err; - if ((bio = BIO_new_mem_buf((void *)cert_pem, cert_pem_len)) == NULL) { + if ((bio = BIO_new_mem_buf((void*)cert_pem, cert_pem_len)) == NULL) { TraceEvent(SevError, "FDBLibTLSOutOfMemory"); goto err; } @@ -145,7 +158,7 @@ struct stack_st_X509* FDBLibTLSPolicy::parse_cert_pem(const uint8_t* cert_pem, s return certs; - err: +err: sk_X509_pop_free(certs, X509_free); X509_free(cert); BIO_free(bio); @@ -200,8 +213,8 @@ bool FDBLibTLSPolicy::set_cert_data(const uint8_t* cert_data, int cert_len) { } bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const char* password) { - EVP_PKEY *key = NULL; - BIO *bio = NULL; + EVP_PKEY* key = NULL; + BIO* bio = NULL; bool rc = false; if (key_data_set) { @@ -214,20 +227,20 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c } if (password != NULL) { - char *data; + char* data; long len; - if ((bio = BIO_new_mem_buf((void *)key_data, key_len)) == NULL) { + if ((bio = BIO_new_mem_buf((void*)key_data, key_len)) == NULL) { TraceEvent(SevError, "FDBLibTLSOutOfMemory"); goto err; } ERR_clear_error(); - if ((key = PEM_read_bio_PrivateKey(bio, NULL, password_cb, (void *)password)) == NULL) { + if ((key = PEM_read_bio_PrivateKey(bio, NULL, password_cb, (void*)password)) == NULL) { int errnum = ERR_peek_error(); char errbuf[256]; if ((ERR_GET_LIB(errnum) == ERR_LIB_PEM && ERR_GET_REASON(errnum) == PEM_R_BAD_DECRYPT) || - (ERR_GET_LIB(errnum) == ERR_LIB_EVP && ERR_GET_REASON(errnum) == EVP_R_BAD_DECRYPT)) { + (ERR_GET_LIB(errnum) == ERR_LIB_EVP && ERR_GET_REASON(errnum) == EVP_R_BAD_DECRYPT)) { TraceEvent(SevError, "FDBLibTLSIncorrectPassword"); } else { ERR_error_string_n(errnum, errbuf, sizeof(errbuf)); @@ -248,7 +261,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c TraceEvent(SevError, "FDBLibTLSOutOfMemory"); goto err; } - if (tls_config_set_key_mem(tls_cfg, (const uint8_t *)data, len) == -1) { + if (tls_config_set_key_mem(tls_cfg, (const uint8_t*)data, len) == -1) { TraceEvent(SevError, "FDBLibTLSKeyError").detail("LibTLSErrorMessage", tls_config_error(tls_cfg)); goto err; } @@ -262,7 +275,7 @@ bool FDBLibTLSPolicy::set_key_data(const uint8_t* key_data, int key_len, const c key_data_set = true; rc = true; - err: +err: BIO_free(bio); EVP_PKEY_free(key); return rc; @@ -287,20 +300,22 @@ bool FDBLibTLSPolicy::set_verify_peers(int count, const uint8_t* verify_peers[], try { std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]); int start = 0; - while(start < verifyString.size()) { + while (start < verifyString.size()) { int split = verifyString.find('|', start); - if(split == std::string::npos) { + if (split == std::string::npos) { break; } - if(split == start || verifyString[split-1] != '\\') { - Reference verify = Reference(new FDBLibTLSVerify(verifyString.substr(start,split-start))); + if (split == start || verifyString[split - 1] != '\\') { + Reference verify = + Reference(new FDBLibTLSVerify(verifyString.substr(start, split - start))); verify_rules.push_back(verify); - start = split+1; + start = split + 1; } } - Reference verify = Reference(new FDBLibTLSVerify(verifyString.substr(start))); + Reference verify = + Reference(new FDBLibTLSVerify(verifyString.substr(start))); verify_rules.push_back(verify); - } catch ( const std::runtime_error& ) { + } catch (const std::runtime_error&) { verify_rules.clear(); std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]); TraceEvent(SevError, "FDBLibTLSVerifyPeersParseError").detail("Config", verifyString); diff --git a/FDBLibTLS/FDBLibTLSPolicy.h b/FDBLibTLS/FDBLibTLSPolicy.h index 268a5f4182..ad874ddd3e 100644 --- a/FDBLibTLS/FDBLibTLSPolicy.h +++ b/FDBLibTLS/FDBLibTLSPolicy.h @@ -32,7 +32,7 @@ #include #include -struct FDBLibTLSPolicy: ITLSPolicy, ReferenceCounted { +struct FDBLibTLSPolicy : ITLSPolicy, ReferenceCounted { FDBLibTLSPolicy(Reference plugin); virtual ~FDBLibTLSPolicy(); @@ -41,7 +41,13 @@ struct FDBLibTLSPolicy: ITLSPolicy, ReferenceCounted { Reference plugin; - virtual ITLSSession* create_session(bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid); + virtual ITLSSession* create_session(bool is_client, + const char* servername, + TLSSendCallbackFunc send_func, + void* send_ctx, + TLSRecvCallbackFunc recv_func, + void* recv_ctx, + void* uid); struct stack_st_X509* parse_cert_pem(const uint8_t* cert_pem, size_t cert_pem_len); void parse_verify(std::string input); diff --git a/FDBLibTLS/FDBLibTLSSession.cpp b/FDBLibTLS/FDBLibTLSSession.cpp index d81dc4e509..dac0301e04 100644 --- a/FDBLibTLS/FDBLibTLSSession.cpp +++ b/FDBLibTLS/FDBLibTLSSession.cpp @@ -36,11 +36,10 @@ #include #include -static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb_arg) -{ - FDBLibTLSSession *session = (FDBLibTLSSession *)cb_arg; +static ssize_t tls_read_func(struct tls* ctx, void* buf, size_t buflen, void* cb_arg) { + FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg; - int rv = session->recv_func(session->recv_ctx, (uint8_t *)buf, buflen); + int rv = session->recv_func(session->recv_ctx, (uint8_t*)buf, buflen); if (rv < 0) return 0; if (rv == 0) @@ -48,11 +47,10 @@ static ssize_t tls_read_func(struct tls *ctx, void *buf, size_t buflen, void *cb return (ssize_t)rv; } -static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, void *cb_arg) -{ - FDBLibTLSSession *session = (FDBLibTLSSession *)cb_arg; +static ssize_t tls_write_func(struct tls* ctx, const void* buf, size_t buflen, void* cb_arg) { + FDBLibTLSSession* session = (FDBLibTLSSession*)cb_arg; - int rv = session->send_func(session->send_ctx, (const uint8_t *)buf, buflen); + int rv = session->send_func(session->send_ctx, (const uint8_t*)buf, buflen); if (rv < 0) return 0; if (rv == 0) @@ -60,11 +58,18 @@ static ssize_t tls_write_func(struct tls *ctx, const void *buf, size_t buflen, v return (ssize_t)rv; } -FDBLibTLSSession::FDBLibTLSSession(Reference policy, bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uidptr) : - tls_ctx(NULL), tls_sctx(NULL), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx), - recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) { +FDBLibTLSSession::FDBLibTLSSession(Reference policy, + bool is_client, + const char* servername, + TLSSendCallbackFunc send_func, + void* send_ctx, + TLSRecvCallbackFunc recv_func, + void* recv_ctx, + void* uidptr) + : tls_ctx(NULL), tls_sctx(NULL), is_client(is_client), policy(policy), send_func(send_func), send_ctx(send_ctx), + recv_func(recv_func), recv_ctx(recv_ctx), handshake_completed(false), lastVerifyFailureLogged(0.0) { if (uidptr) - uid = * (UID*) uidptr; + uid = *(UID*)uidptr; if (is_client) { if ((tls_ctx = tls_client()) == NULL) { @@ -123,12 +128,10 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match if ((entry_utf8_len = ASN1_STRING_to_UTF8(&entry_utf8, entry)) < 1) goto err; if (mt == MatchType::EXACT) { - if (criteria_utf8_len == entry_utf8_len && - memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0) + if (criteria_utf8_len == entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0) rc = true; } else if (mt == MatchType::PREFIX) { - if (criteria_utf8_len <= entry_utf8_len && - memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0) + if (criteria_utf8_len <= entry_utf8_len && memcmp(criteria_utf8, entry_utf8, criteria_utf8_len) == 0) rc = true; } else if (mt == MatchType::SUFFIX) { if (criteria_utf8_len <= entry_utf8_len && @@ -136,15 +139,15 @@ bool match_criteria_entry(const std::string& criteria, ASN1_STRING* entry, Match rc = true; } - err: +err: ASN1_STRING_free(asn_criteria); free(criteria_utf8); free(entry_utf8); return rc; } -bool match_name_criteria(X509_NAME *name, NID nid, const std::string& criteria, MatchType mt) { - X509_NAME_ENTRY *name_entry; +bool match_name_criteria(X509_NAME* name, NID nid, const std::string& criteria, MatchType mt) { + X509_NAME_ENTRY* name_entry; int idx; // If name does not exist, or has multiple of this RDN, refuse to proceed. @@ -158,7 +161,7 @@ bool match_name_criteria(X509_NAME *name, NID nid, const std::string& criteria, return match_criteria_entry(criteria, name_entry->value, mt); } -bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, MatchType mt) { +bool match_extension_criteria(X509* cert, NID nid, const std::string& value, MatchType mt) { if (nid != NID_subject_alt_name && nid != NID_issuer_alt_name) { // I have no idea how other extensions work. return false; @@ -168,28 +171,26 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat return false; } std::string value_gen = value.substr(0, pos); - std::string value_val = value.substr(pos+1, value.npos); + std::string value_val = value.substr(pos + 1, value.npos); STACK_OF(GENERAL_NAME)* sans = reinterpret_cast(X509_get_ext_d2i(cert, nid, NULL, NULL)); if (sans == NULL) { return false; } - int num_sans = sk_GENERAL_NAME_num( sans ); + int num_sans = sk_GENERAL_NAME_num(sans); bool rc = false; - for( int i = 0; i < num_sans && !rc; ++i ) { - GENERAL_NAME* altname = sk_GENERAL_NAME_value( sans, i ); + for (int i = 0; i < num_sans && !rc; ++i) { + GENERAL_NAME* altname = sk_GENERAL_NAME_value(sans, i); std::string matchable; switch (altname->type) { case GEN_OTHERNAME: break; case GEN_EMAIL: - if (value_gen == "EMAIL" && - match_criteria_entry( value_val, altname->d.rfc822Name, mt)) { + if (value_gen == "EMAIL" && match_criteria_entry(value_val, altname->d.rfc822Name, mt)) { rc = true; break; } case GEN_DNS: - if (value_gen == "DNS" && - match_criteria_entry( value_val, altname->d.dNSName, mt )) { + if (value_gen == "DNS" && match_criteria_entry(value_val, altname->d.dNSName, mt)) { rc = true; break; } @@ -198,14 +199,12 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat case GEN_EDIPARTY: break; case GEN_URI: - if (value_gen == "URI" && - match_criteria_entry( value_val, altname->d.uniformResourceIdentifier, mt )) { + if (value_gen == "URI" && match_criteria_entry(value_val, altname->d.uniformResourceIdentifier, mt)) { rc = true; break; } case GEN_IPADD: - if (value_gen == "IP" && - match_criteria_entry( value_val, altname->d.iPAddress, mt )) { + if (value_gen == "IP" && match_criteria_entry(value_val, altname->d.iPAddress, mt)) { rc = true; break; } @@ -217,8 +216,13 @@ bool match_extension_criteria(X509 *cert, NID nid, const std::string& value, Mat return rc; } -bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string& criteria, MatchType mt, X509Location loc) { - switch(loc) { +bool match_criteria(X509* cert, + X509_NAME* subject, + NID nid, + const std::string& criteria, + MatchType mt, + X509Location loc) { + switch (loc) { case X509Location::NAME: { return match_name_criteria(subject, nid, criteria, mt); } @@ -230,8 +234,9 @@ bool match_criteria(X509* cert, X509_NAME* subject, NID nid, const std::string& return false; } -std::tuple FDBLibTLSSession::check_verify(Reference verify, struct stack_st_X509 *certs) { - X509_STORE_CTX *store_ctx = NULL; +std::tuple FDBLibTLSSession::check_verify(Reference verify, + struct stack_st_X509* certs) { + X509_STORE_CTX* store_ctx = NULL; X509_NAME *subject, *issuer; bool rc = false; X509* cert = NULL; @@ -257,7 +262,7 @@ std::tuple FDBLibTLSSession::check_verify(Referenceverify_time) X509_VERIFY_PARAM_set_flags(X509_STORE_CTX_get0_param(store_ctx), X509_V_FLAG_NO_CHECK_TIME); if (X509_verify_cert(store_ctx) <= 0) { - const char *errstr = X509_verify_cert_error_string(X509_STORE_CTX_get_error(store_ctx)); + const char* errstr = X509_verify_cert_error_string(X509_STORE_CTX_get_error(store_ctx)); reason = "Verify cert error: " + std::string(errstr); goto err; } @@ -268,8 +273,9 @@ std::tuple FDBLibTLSSession::check_verify(Referencesubject_criteria) { - if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { + for (auto& pair : verify->subject_criteria) { + if (!match_criteria( + cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { reason = "Cert subject match failure"; goto err; } @@ -280,8 +286,9 @@ std::tuple FDBLibTLSSession::check_verify(Referenceissuer_criteria) { - if (!match_criteria(cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { + for (auto& pair : verify->issuer_criteria) { + if (!match_criteria( + cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { reason = "Cert issuer match failure"; goto err; } @@ -293,8 +300,9 @@ std::tuple FDBLibTLSSession::check_verify(Referenceroot_criteria) { - if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { + for (auto& pair : verify->root_criteria) { + if (!match_criteria( + cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) { reason = "Root subject match failure"; goto err; } @@ -303,15 +311,15 @@ std::tuple FDBLibTLSSession::check_verify(Reference verify_failure_reasons; @@ -331,7 +339,7 @@ bool FDBLibTLSSession::verify_peer() { goto err; // Any matching rule is sufficient. - for (auto &verify_rule: policy->verify_rules) { + for (auto& verify_rule : policy->verify_rules) { std::tie(verify_success, verify_failure_reason) = check_verify(verify_rule, certs); if (verify_success) { rc = true; @@ -344,7 +352,7 @@ bool FDBLibTLSSession::verify_peer() { if (!rc) { // log the various failure reasons - if(now() - lastVerifyFailureLogged > 1.0) { + if (now() - lastVerifyFailureLogged > 1.0) { for (std::string reason : verify_failure_reasons) { lastVerifyFailureLogged = now(); TraceEvent("FDBLibTLSVerifyFailure", uid).suppressFor(1.0).detail("Reason", reason); @@ -352,7 +360,7 @@ bool FDBLibTLSSession::verify_peer() { } } - err: +err: sk_X509_pop_free(certs, X509_free); return rc; diff --git a/FDBLibTLS/FDBLibTLSSession.h b/FDBLibTLS/FDBLibTLSSession.h index d648bd80a7..83f0e4d1c9 100644 --- a/FDBLibTLS/FDBLibTLSSession.h +++ b/FDBLibTLS/FDBLibTLSSession.h @@ -33,14 +33,21 @@ #include struct FDBLibTLSSession : ITLSSession, ReferenceCounted { - FDBLibTLSSession(Reference policy, bool is_client, const char* servername, TLSSendCallbackFunc send_func, void* send_ctx, TLSRecvCallbackFunc recv_func, void* recv_ctx, void* uid); + FDBLibTLSSession(Reference policy, + bool is_client, + const char* servername, + TLSSendCallbackFunc send_func, + void* send_ctx, + TLSRecvCallbackFunc recv_func, + void* recv_ctx, + void* uid); virtual ~FDBLibTLSSession(); virtual void addref() { ReferenceCounted::addref(); } virtual void delref() { ReferenceCounted::delref(); } bool verify_peer(); - std::tuple check_verify(Reference verify, struct stack_st_X509 *certs); + std::tuple check_verify(Reference verify, struct stack_st_X509* certs); virtual int handshake(); virtual int read(uint8_t* data, int length); @@ -50,8 +57,8 @@ struct FDBLibTLSSession : ITLSSession, ReferenceCounted { bool is_client; - struct tls *tls_ctx; - struct tls *tls_sctx; + struct tls* tls_ctx; + struct tls* tls_sctx; TLSSendCallbackFunc send_func; void* send_ctx; diff --git a/FDBLibTLS/FDBLibTLSVerify.cpp b/FDBLibTLS/FDBLibTLSVerify.cpp index 594389916f..2f9a5b678a 100644 --- a/FDBLibTLS/FDBLibTLSVerify.cpp +++ b/FDBLibTLS/FDBLibTLSVerify.cpp @@ -43,24 +43,24 @@ static int hexValue(char c) { static std::string de4514(std::string const& input, int start, int& out_end) { std::string output; - if(input[start] == '#' || input[start] == ' ') { + if (input[start] == '#' || input[start] == ' ') { out_end = start; return output; } int space_count = 0; - for(int p = start; p < input.size();) { - switch(input[p]) { + for (int p = start; p < input.size();) { + switch (input[p]) { case '\\': // Handle escaped sequence // Backslash escaping nothing! - if(p == input.size() - 1) { + if (p == input.size() - 1) { out_end = p; goto FIN; } - switch(input[p+1]) { + switch (input[p + 1]) { case ' ': case '"': case '#': @@ -72,24 +72,24 @@ static std::string de4514(std::string const& input, int start, int& out_end) { case '>': case '|': case '\\': - output += input[p+1]; + output += input[p + 1]; p += 2; space_count = 0; continue; default: // Backslash escaping pair of hex digits requires two characters - if(p == input.size() - 2) { + if (p == input.size() - 2) { out_end = p; goto FIN; } try { - output += hexValue(input[p+1]) * 16 + hexValue(input[p+2]); + output += hexValue(input[p + 1]) * 16 + hexValue(input[p + 2]); p += 3; space_count = 0; continue; - } catch( ... ) { + } catch (...) { out_end = p; goto FIN; } @@ -109,7 +109,7 @@ static std::string de4514(std::string const& input, int start, int& out_end) { default: // Character is what it is output += input[p]; - if(input[p] == ' ') + if (input[p] == ' ') space_count++; else space_count = 0; @@ -119,7 +119,7 @@ static std::string de4514(std::string const& input, int start, int& out_end) { out_end = input.size(); - FIN: +FIN: out_end -= space_count; output.resize(output.size() - space_count); @@ -128,16 +128,17 @@ static std::string de4514(std::string const& input, int start, int& out_end) { static std::pair splitPair(std::string const& input, char c) { int p = input.find_first_of(c); - if(p == input.npos) { + if (p == input.npos) { throw std::runtime_error("splitPair"); } - return std::make_pair(input.substr(0, p), input.substr(p+1, input.size())); + return std::make_pair(input.substr(0, p), input.substr(p + 1, input.size())); } static NID abbrevToNID(std::string const& sn) { NID nid = NID_undef; - if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" || sn == "subjectAltName") + if (sn == "C" || sn == "CN" || sn == "L" || sn == "ST" || sn == "O" || sn == "OU" || sn == "UID" || sn == "DC" || + sn == "subjectAltName") nid = OBJ_sn2nid(sn.c_str()); if (nid == NID_undef) throw std::runtime_error("abbrevToNID"); @@ -158,13 +159,11 @@ static X509Location locationForNID(NID nid) { } } -FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config): - verify_cert(true), verify_time(true) { +FDBLibTLSVerify::FDBLibTLSVerify(std::string verify_config) : verify_cert(true), verify_time(true) { parse_verify(verify_config); } -FDBLibTLSVerify::~FDBLibTLSVerify() { -} +FDBLibTLSVerify::~FDBLibTLSVerify() {} void FDBLibTLSVerify::parse_verify(std::string input) { int s = 0; @@ -176,8 +175,10 @@ void FDBLibTLSVerify::parse_verify(std::string input) { throw std::runtime_error("parse_verify"); MatchType mt = MatchType::EXACT; - if (input[eq-1] == '>') mt = MatchType::PREFIX; - if (input[eq-1] == '<') mt = MatchType::SUFFIX; + if (input[eq - 1] == '>') + mt = MatchType::PREFIX; + if (input[eq - 1] == '<') + mt = MatchType::SUFFIX; std::string term = input.substr(s, eq - s - (mt == MatchType::EXACT ? 0 : 1)); if (term.find("Check.") == 0) { @@ -206,7 +207,7 @@ void FDBLibTLSVerify::parse_verify(std::string input) { s = eq + 3; } else { - std::map< int, Criteria >* criteria = &subject_criteria; + std::map* criteria = &subject_criteria; if (term.find('.') != term.npos) { auto scoped = splitPair(term, '.'); diff --git a/FDBLibTLS/FDBLibTLSVerify.h b/FDBLibTLS/FDBLibTLSVerify.h index 10fb07913e..8cf3330112 100644 --- a/FDBLibTLS/FDBLibTLSVerify.h +++ b/FDBLibTLS/FDBLibTLSVerify.h @@ -47,14 +47,10 @@ enum class X509Location { }; struct Criteria { - Criteria( const std::string& s ) - : criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {} - Criteria( const std::string& s, MatchType mt ) - : criteria(s), match_type(mt), location(X509Location::NAME) {} - Criteria( const std::string& s, X509Location loc) - : criteria(s), match_type(MatchType::EXACT), location(loc) {} - Criteria( const std::string& s, MatchType mt, X509Location loc) - : criteria(s), match_type(mt), location(loc) {} + Criteria(const std::string& s) : criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {} + Criteria(const std::string& s, MatchType mt) : criteria(s), match_type(mt), location(X509Location::NAME) {} + Criteria(const std::string& s, X509Location loc) : criteria(s), match_type(MatchType::EXACT), location(loc) {} + Criteria(const std::string& s, MatchType mt, X509Location loc) : criteria(s), match_type(mt), location(loc) {} std::string criteria; MatchType match_type; @@ -65,7 +61,7 @@ struct Criteria { } }; -struct FDBLibTLSVerify: ReferenceCounted { +struct FDBLibTLSVerify : ReferenceCounted { FDBLibTLSVerify(std::string verify); virtual ~FDBLibTLSVerify(); @@ -77,9 +73,9 @@ struct FDBLibTLSVerify: ReferenceCounted { bool verify_cert; bool verify_time; - std::map< NID, Criteria > subject_criteria; - std::map< NID, Criteria > issuer_criteria; - std::map< NID, Criteria > root_criteria; + std::map subject_criteria; + std::map issuer_criteria; + std::map root_criteria; }; #endif /* FDB_LIBTLS_VERIFY_H */ diff --git a/FDBLibTLS/plugin-test.cpp b/FDBLibTLS/plugin-test.cpp index e12d4b82b6..3567ec83a8 100644 --- a/FDBLibTLS/plugin-test.cpp +++ b/FDBLibTLS/plugin-test.cpp @@ -36,8 +36,7 @@ #define TESTDATA "./testdata/" -static std::string load_file(std::string path) -{ +static std::string load_file(std::string path) { std::ifstream fs(path); std::stringstream ss; @@ -71,8 +70,8 @@ struct FDBLibTLSPluginTest { boost::circular_buffer client_buffer; boost::circular_buffer server_buffer; - int circular_read(boost::circular_buffer *cb, uint8_t* buf, int len); - int circular_write(boost::circular_buffer *cb, const uint8_t* buf, int len); + int circular_read(boost::circular_buffer* cb, uint8_t* buf, int len); + int circular_write(boost::circular_buffer* cb, const uint8_t* buf, int len); int client_read(uint8_t* buf, int len); int client_write(const uint8_t* buf, int len); int server_read(uint8_t* buf, int len); @@ -85,23 +84,18 @@ struct FDBLibTLSPluginTest { void circular_reset(void); void circular_self_test(void); - int client_server_test(const struct client_server_test *cst); + int client_server_test(const struct client_server_test* cst); int set_cert_data_test(void); }; -FDBLibTLSPluginTest::FDBLibTLSPluginTest(Reference plugin) : - plugin(plugin) -{ +FDBLibTLSPluginTest::FDBLibTLSPluginTest(Reference plugin) : plugin(plugin) { circular_reset(); circular_self_test(); } -FDBLibTLSPluginTest::~FDBLibTLSPluginTest() -{ -} +FDBLibTLSPluginTest::~FDBLibTLSPluginTest() {} -int FDBLibTLSPluginTest::circular_read(boost::circular_buffer *cb, uint8_t* buf, int len) -{ +int FDBLibTLSPluginTest::circular_read(boost::circular_buffer* cb, uint8_t* buf, int len) { int n = 0; for (n = 0; n < len; n++) { @@ -114,8 +108,7 @@ int FDBLibTLSPluginTest::circular_read(boost::circular_buffer *cb, uint return n; } -int FDBLibTLSPluginTest::circular_write(boost::circular_buffer *cb, const uint8_t* buf, int len) -{ +int FDBLibTLSPluginTest::circular_write(boost::circular_buffer* cb, const uint8_t* buf, int len) { int n = 0; for (n = 0; n < len; n++) { @@ -127,39 +120,33 @@ int FDBLibTLSPluginTest::circular_write(boost::circular_buffer *cb, con return n; } -int FDBLibTLSPluginTest::client_read(uint8_t* buf, int len) -{ +int FDBLibTLSPluginTest::client_read(uint8_t* buf, int len) { // Read bytes from the server from the client's buffer. return circular_read(&client_buffer, buf, len); } -int FDBLibTLSPluginTest::client_write(const uint8_t* buf, int len) -{ +int FDBLibTLSPluginTest::client_write(const uint8_t* buf, int len) { // Write bytes from the client into the server's buffer. return circular_write(&server_buffer, buf, len); } -int FDBLibTLSPluginTest::server_read(uint8_t* buf, int len) -{ +int FDBLibTLSPluginTest::server_read(uint8_t* buf, int len) { // Read bytes from the client from the server's buffer. return circular_read(&server_buffer, buf, len); } -int FDBLibTLSPluginTest::server_write(const uint8_t* buf, int len) -{ +int FDBLibTLSPluginTest::server_write(const uint8_t* buf, int len) { // Write bytes from the server into the client's buffer. return circular_write(&client_buffer, buf, len); } -void FDBLibTLSPluginTest::circular_reset() -{ +void FDBLibTLSPluginTest::circular_reset() { client_buffer = boost::circular_buffer(1024); server_buffer = boost::circular_buffer(1024); } -void FDBLibTLSPluginTest::circular_self_test() -{ - uint8_t buf[1024] = {1, 2, 3}; +void FDBLibTLSPluginTest::circular_self_test() { + uint8_t buf[1024] = { 1, 2, 3 }; std::cerr << "INFO: running circular buffer self tests...\n"; @@ -198,73 +185,74 @@ void FDBLibTLSPluginTest::circular_self_test() assert(client_read(buf, 1024) == 0); } -Reference FDBLibTLSPluginTest::create_policy(void) -{ +Reference FDBLibTLSPluginTest::create_policy(void) { return Reference(plugin->create_policy()); } static int client_send_func(void* ctx, const uint8_t* buf, int len) { - FDBLibTLSPluginTest *pt = (FDBLibTLSPluginTest *)ctx; + FDBLibTLSPluginTest* pt = (FDBLibTLSPluginTest*)ctx; try { return pt->client_write(buf, len); - } catch ( const std::runtime_error& e ) { + } catch (const std::runtime_error& e) { return -1; } } static int client_recv_func(void* ctx, uint8_t* buf, int len) { - FDBLibTLSPluginTest *pt = (FDBLibTLSPluginTest *)ctx; + FDBLibTLSPluginTest* pt = (FDBLibTLSPluginTest*)ctx; try { return pt->client_read(buf, len); - } catch ( const std::runtime_error& e ) { + } catch (const std::runtime_error& e) { return -1; } } -Reference FDBLibTLSPluginTest::create_client_session(Reference policy, const char* servername) -{ - return Reference(policy->create_session(true, servername, client_send_func, this, client_recv_func, this, NULL)); +Reference FDBLibTLSPluginTest::create_client_session(Reference policy, + const char* servername) { + return Reference( + policy->create_session(true, servername, client_send_func, this, client_recv_func, this, NULL)); } static int server_send_func(void* ctx, const uint8_t* buf, int len) { - FDBLibTLSPluginTest *pt = (FDBLibTLSPluginTest *)ctx; + FDBLibTLSPluginTest* pt = (FDBLibTLSPluginTest*)ctx; try { return pt->server_write(buf, len); - } catch ( const std::runtime_error& e ) { + } catch (const std::runtime_error& e) { return -1; } } static int server_recv_func(void* ctx, uint8_t* buf, int len) { - FDBLibTLSPluginTest *pt = (FDBLibTLSPluginTest *)ctx; + FDBLibTLSPluginTest* pt = (FDBLibTLSPluginTest*)ctx; try { return pt->server_read(buf, len); - } catch ( const std::runtime_error& e ) { + } catch (const std::runtime_error& e) { return -1; } } -Reference FDBLibTLSPluginTest::create_server_session(Reference policy) -{ - return Reference(policy->create_session(false, NULL, server_send_func, this, server_recv_func, this, NULL)); +Reference FDBLibTLSPluginTest::create_server_session(Reference policy) { + return Reference( + policy->create_session(false, NULL, server_send_func, this, server_recv_func, this, NULL)); } #define MAX_VERIFY_RULES 5 -static void convert_verify_peers(const std::vector *verify_rules, const uint8_t *verify_peers[], int verify_peers_len[]) { +static void convert_verify_peers(const std::vector* verify_rules, + const uint8_t* verify_peers[], + int verify_peers_len[]) { if (verify_rules->size() > MAX_VERIFY_RULES) throw std::runtime_error("verify"); int i = 0; - for (auto &verify_rule: *verify_rules) { - verify_peers[i] = (const uint8_t *)&verify_rule[0]; + for (auto& verify_rule : *verify_rules) { + verify_peers[i] = (const uint8_t*)&verify_rule[0]; verify_peers_len[i] = verify_rule.size(); i++; } } -int FDBLibTLSPluginTest::client_server_test(const struct client_server_test* cst) -{ - const uint8_t *verify_peers[MAX_VERIFY_RULES]; +int FDBLibTLSPluginTest::client_server_test(const struct client_server_test* cst) { + const uint8_t* verify_peers[MAX_VERIFY_RULES]; int verify_peers_len[MAX_VERIFY_RULES]; circular_reset(); @@ -395,7 +383,7 @@ int FDBLibTLSPluginTest::client_server_test(const struct client_server_test* cst i = 0; do { if (!client_done) { - rc = client_session->write((const uint8_t*)&client_msg[cn], client_msg.size()-cn); + rc = client_session->write((const uint8_t*)&client_msg[cn], client_msg.size() - cn); if (rc > 0) { cn += rc; if (cn >= client_msg.size()) @@ -446,7 +434,7 @@ int FDBLibTLSPluginTest::client_server_test(const struct client_server_test* cst i = 0; do { if (!server_done) { - rc = server_session->write((const uint8_t*)&server_msg[cn], server_msg.size()-cn); + rc = server_session->write((const uint8_t*)&server_msg[cn], server_msg.size() - cn); if (rc > 0) { cn += rc; if (cn >= server_msg.size()) @@ -490,7 +478,7 @@ int FDBLibTLSPluginTest::client_server_test(const struct client_server_test* cst static void logf(const char* event, void* uid, bool is_error, ...) { va_list args; - std::string log_type ("INFO"); + std::string log_type("INFO"); if (is_error) log_type = "ERROR"; @@ -498,10 +486,10 @@ static void logf(const char* event, void* uid, bool is_error, ...) { va_start(args, is_error); - const char *s = va_arg(args, const char *); + const char* s = va_arg(args, const char*); while (s != NULL) { std::cerr << " " << s; - s = va_arg(args, const char *); + s = va_arg(args, const char*); } std::cerr << "\n"; @@ -512,635 +500,638 @@ static void logf(const char* event, void* uid, bool is_error, ...) { const struct client_server_test client_server_tests[] = { // Single root CA. { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, // Multiple root CAs. { - .ca_path = "test-ca-all.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-4.pem", - .server_password = "fdb123", - .server_verify = {""}, + .ca_path = "test-ca-all.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-4.pem", + .server_password = "fdb123", + .server_verify = { "" }, }, { - .ca_path = "test-ca-all.pem", - .client_success = true, - .client_path = "test-client-4.pem", - .client_password = "fdb321", - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = "fdb123", - .server_verify = {""}, + .ca_path = "test-ca-all.pem", + .client_success = true, + .client_path = "test-client-4.pem", + .client_password = "fdb321", + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = "fdb123", + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-4.pem", - .server_password = "fdb123", - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-4.pem", + .server_password = "fdb123", + .server_verify = { "" }, }, { - .ca_path = "test-ca-2.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = false, - .server_path = "test-server-4.pem", - .server_password = "fdb123", - .server_verify = {""}, + .ca_path = "test-ca-2.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = false, + .server_path = "test-server-4.pem", + .server_password = "fdb123", + .server_verify = { "" }, }, // Expired certificates. { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-3.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-3.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-3.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = false, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-3.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = false, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"Check.Unexpired=0"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-3.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "Check.Unexpired=0" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-3.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-3.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"Check.Unexpired=0"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-3.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "Check.Unexpired=0" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"Check.Valid=0"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-3.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "Check.Valid=0" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-3.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-3.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"Check.Valid=0"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-3.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "Check.Valid=0" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1", "I.CN=FDB LibTLS Plugin Test Intermediate CA 2,Check.Unexpired=0"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-3.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1", + "I.CN=FDB LibTLS Plugin Test Intermediate CA 2,Check.Unexpired=0" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-3.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1,Check.Unexpired=0", "I.CN=FDB LibTLS Plugin Test Intermediate CA 2"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-3.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1,Check.Unexpired=0", + "I.CN=FDB LibTLS Plugin Test Intermediate CA 2" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-3.pem", + .server_password = NULL, + .server_verify = { "" }, }, // Match on specific subject and/or issuer. { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"C=US"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "C=US" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"C=US"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "C=US" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"C=AU"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "C=AU" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"C=US", "C=AU"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "C=US", "C=AU" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"C=US", "C=JP"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "C=US", "C=JP" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\04\\>"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\04\\>" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\81 \\<\\01\\+\\02=\\04\\>"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\81 \\<\\01\\+\\02=\\04\\>" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\04"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\04" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {"CN=FDB LibTLS Plugin Test Client 1"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "CN=FDB LibTLS Plugin Test Client 1" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"CN=FDB LibTLS Plugin Test Client 1"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "CN=FDB LibTLS Plugin Test Client 1" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {""}, - .servername = NULL, - .server_success = false, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"O=Apple Pty Limited,OU=FDC Team"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "" }, + .servername = NULL, + .server_success = false, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "O=Apple Pty Limited,OU=FDC Team" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"O=Apple Inc.,OU=FDB Team"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"O=Apple Pty Limited,OU=FDB Team"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "O=Apple Inc.,OU=FDB Team" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "O=Apple Pty Limited,OU=FDB Team" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"O=Apple Inc.,OU=FDC Team"}, - .servername = NULL, - .server_success = false, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"O=Apple Pty Limited,OU=FDC Team"}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "O=Apple Inc.,OU=FDC Team" }, + .servername = NULL, + .server_success = false, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "O=Apple Pty Limited,OU=FDC Team" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDB Team"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDB Team"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDB Team" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDB Team" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDC Team"}, - .servername = NULL, - .server_success = false, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDC Team"}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDC Team" }, + .servername = NULL, + .server_success = false, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "I.C=US,I.ST=California,I.L=Cupertino,I.O=Apple Inc.,I.OU=FDC Team" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 2"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1"}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 2" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 2"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 2" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin Test Intermediate CA 2"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1,O=Apple Inc.,I.C=US,S.C=US"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin " + "Test Intermediate CA 2" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1,O=Apple Inc.,I.C=US,S.C=US" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin Test Intermediate CA 1"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {"I.CN=FDB LibTLS Plugin Test Intermediate CA 1,O=Apple Inc.,I.C=US,S.C=US"}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "CN=FDB LibTLS Plugin Test Server 2\\, \\80 \\<\\01\\+\\02=\\03\\>,I.CN=FDB LibTLS Plugin " + "Test Intermediate CA 1" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "I.CN=FDB LibTLS Plugin Test Intermediate CA 1,O=Apple Inc.,I.C=US,S.C=US" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 1"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-2.pem", - .server_password = NULL, - .server_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 1"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 1" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-2.pem", + .server_password = NULL, + .server_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 1" }, }, { - .ca_path = "test-ca-all.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 1"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-4.pem", - .server_password = "fdb123", - .server_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 1"}, + .ca_path = "test-ca-all.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 1" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-4.pem", + .server_password = "fdb123", + .server_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 1" }, }, { - .ca_path = "test-ca-all.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 1", "R.CN=FDB LibTLS Plugin Test Root CA 2"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-4.pem", - .server_password = "fdb123", - .server_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 1"}, + .ca_path = "test-ca-all.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 1", "R.CN=FDB LibTLS Plugin Test Root CA 2" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-4.pem", + .server_password = "fdb123", + .server_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 1" }, }, { - .ca_path = "test-ca-all.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 2"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-4.pem", - .server_password = "fdb123", - .server_verify = {"R.CN=FDB LibTLS Plugin Test Root CA 1"}, + .ca_path = "test-ca-all.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 2" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-4.pem", + .server_password = "fdb123", + .server_verify = { "R.CN=FDB LibTLS Plugin Test Root CA 1" }, }, { - .ca_path = "test-ca-all.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {"R.OU=FDB Team"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-4.pem", - .server_password = "fdb123", - .server_verify = {"R.OU=FDB Team"}, + .ca_path = "test-ca-all.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = { "R.OU=FDB Team" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-4.pem", + .server_password = "fdb123", + .server_verify = { "R.OU=FDB Team" }, }, // Client performing name validation via servername. { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {}, - .servername = "test.foundationdb.org", - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = {}, + .servername = "test.foundationdb.org", + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-1.pem", - .client_password = NULL, - .client_verify = {}, - .servername = "www.foundationdb.org", - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {""}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-1.pem", + .client_password = NULL, + .client_verify = {}, + .servername = "www.foundationdb.org", + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "" }, }, // Prefix and Suffix Matching { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"O>=Apple Inc.,OU>=FDB"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"O<=Limited,OU<=Team"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "O>=Apple Inc.,OU>=FDB" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "O<=Limited,OU<=Team" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"O<=Apple Inc.,OU<=FDB"}, - .servername = NULL, - .server_success = false, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"O>=Limited,OU>=Team"}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "O<=Apple Inc.,OU<=FDB" }, + .servername = NULL, + .server_success = false, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "O>=Limited,OU>=Team" }, }, // Subject Alternative Name { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"S.subjectAltName=DNS:test.foundationdb.org"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"Check.Valid=0"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "S.subjectAltName=DNS:test.foundationdb.org" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "Check.Valid=0" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"S.subjectAltName>=DNS:test."}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"Check.Valid=0"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "S.subjectAltName>=DNS:test." }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "Check.Valid=0" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = true, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"S.subjectAltName<=DNS:.org"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"Check.Valid=0"}, + .ca_path = "test-ca-1.pem", + .client_success = true, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "S.subjectAltName<=DNS:.org" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "Check.Valid=0" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"S.subjectAltName<=DNS:.com"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"Check.Valid=0"}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "S.subjectAltName<=DNS:.com" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "Check.Valid=0" }, }, { - .ca_path = "test-ca-1.pem", - .client_success = false, - .client_path = "test-client-2.pem", - .client_password = NULL, - .client_verify = {"S.subjectAltName<=EMAIL:.com"}, - .servername = NULL, - .server_success = true, - .server_path = "test-server-1.pem", - .server_password = NULL, - .server_verify = {"Check.Valid=0"}, + .ca_path = "test-ca-1.pem", + .client_success = false, + .client_path = "test-client-2.pem", + .client_password = NULL, + .client_verify = { "S.subjectAltName<=EMAIL:.com" }, + .servername = NULL, + .server_success = true, + .server_path = "test-server-1.pem", + .server_password = NULL, + .server_verify = { "Check.Valid=0" }, }, }; -int main(int argc, char **argv) -{ - void *pluginSO = NULL; - void *(*getPlugin)(const char*); +int main(int argc, char** argv) { + void* pluginSO = NULL; + void* (*getPlugin)(const char*); int failed = 0; if (argc != 2) { @@ -1154,18 +1145,19 @@ int main(int argc, char **argv) exit(1); } - getPlugin = (void*(*)(const char*))dlsym( pluginSO, "get_plugin" ); + getPlugin = (void* (*)(const char*))dlsym(pluginSO, "get_plugin"); if (getPlugin == NULL) { std::cerr << "plugin '" << argv[1] << "' does not provide get_plugin()\n"; exit(1); } - Reference plugin = Reference((ITLSPlugin *)getPlugin(ITLSPlugin::get_plugin_type_name_and_version())); + Reference plugin = + Reference((ITLSPlugin*)getPlugin(ITLSPlugin::get_plugin_type_name_and_version())); - FDBLibTLSPluginTest *pt = new FDBLibTLSPluginTest(plugin); + FDBLibTLSPluginTest* pt = new FDBLibTLSPluginTest(plugin); int test_num = 1; - for (auto &cst: client_server_tests) { + for (auto& cst : client_server_tests) { std::cerr << "== Test " << test_num++ << " ==\n"; failed |= pt->client_server_test(&cst); } diff --git a/FDBLibTLS/verify-test.cpp b/FDBLibTLS/verify-test.cpp index 1a0cdcbc5c..c1383b5e0b 100644 --- a/FDBLibTLS/verify-test.cpp +++ b/FDBLibTLS/verify-test.cpp @@ -33,11 +33,18 @@ #include "FDBLibTLS/FDBLibTLSPolicy.h" struct FDBLibTLSVerifyTest { - FDBLibTLSVerifyTest(std::string input): - input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}), root_criteria({}) {}; - FDBLibTLSVerifyTest(std::string input, bool verify_cert, bool verify_time, std::map subject, std::map issuer, std::map root): - input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject), issuer_criteria(issuer), root_criteria(root) {}; - ~FDBLibTLSVerifyTest() {}; + FDBLibTLSVerifyTest(std::string input) + : input(input), valid(false), verify_cert(true), verify_time(true), subject_criteria({}), issuer_criteria({}), + root_criteria({}){}; + FDBLibTLSVerifyTest(std::string input, + bool verify_cert, + bool verify_time, + std::map subject, + std::map issuer, + std::map root) + : input(input), valid(true), verify_cert(verify_cert), verify_time(verify_time), subject_criteria(subject), + issuer_criteria(issuer), root_criteria(root){}; + ~FDBLibTLSVerifyTest(){}; int run(); @@ -54,20 +61,21 @@ struct FDBLibTLSVerifyTest { static std::string criteriaToString(std::map const& criteria) { std::string s; - for (auto &pair: criteria) { - s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " + boost::lexical_cast((int)pair.second.match_type) + ", " + boost::lexical_cast((int)pair.second.location) + ")}"; + for (auto& pair : criteria) { + s += "{" + std::to_string(pair.first) + ":(" + printable(pair.second.criteria) + ", " + + boost::lexical_cast((int)pair.second.match_type) + ", " + + boost::lexical_cast((int)pair.second.location) + ")}"; } return "{" + s + "}"; } -static void logf(const char* event, void* uid, bool is_error, ...) { -} +static void logf(const char* event, void* uid, bool is_error, ...) {} int FDBLibTLSVerifyTest::run() { Reference verify; try { verify = Reference(new FDBLibTLSVerify(input)); - } catch ( const std::runtime_error& e ) { + } catch (const std::runtime_error& e) { if (valid) { std::cerr << "FAIL: Verify test failed, but should have succeeded - '" << input << "'\n"; return 1; @@ -87,15 +95,18 @@ int FDBLibTLSVerifyTest::run() { return 1; } if (verify->subject_criteria != subject_criteria) { - std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want " << criteriaToString(subject_criteria) << "\n"; + std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify->subject_criteria) << ", want " + << criteriaToString(subject_criteria) << "\n"; return 1; } if (verify->issuer_criteria != issuer_criteria) { - std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want " << criteriaToString(issuer_criteria) << "\n"; + std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify->issuer_criteria) << ", want " + << criteriaToString(issuer_criteria) << "\n"; return 1; } if (verify->root_criteria != root_criteria) { - std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want " << criteriaToString(root_criteria) << "\n"; + std::cerr << "FAIL: Got root criteria " << criteriaToString(verify->root_criteria) << ", want " + << criteriaToString(root_criteria) << "\n"; return 1; } return 0; @@ -105,7 +116,7 @@ static int policy_verify_test() { Reference plugin = Reference(new FDBLibTLSPlugin()); Reference policy = Reference(new FDBLibTLSPolicy(plugin, (ITLSLogFunc)logf)); - const char *verify_peers[] = { + const char* verify_peers[] = { "S.CN=abc", "I.CN=def", "R.CN=xyz,Check.Unexpired=0", @@ -121,7 +132,7 @@ static int policy_verify_test() { Reference(new FDBLibTLSVerify(std::string(verify_peers[2], verify_peers_len[2]))), }; - if (!policy->set_verify_peers(3, (const uint8_t **)verify_peers, verify_peers_len)) { + if (!policy->set_verify_peers(3, (const uint8_t**)verify_peers, verify_peers_len)) { std::cerr << "FAIL: Policy verify test failed, but should have succeeded\n"; return 1; } @@ -131,25 +142,30 @@ static int policy_verify_test() { } int i = 0; - for (auto &verify_rule: policy->verify_rules) { + for (auto& verify_rule : policy->verify_rules) { if (verify_rule->verify_cert != verify_rules[i]->verify_cert) { - std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want " << verify_rules[i]->verify_cert << "\n"; + std::cerr << "FAIL: Got verify cert " << verify_rule->verify_cert << ", want " + << verify_rules[i]->verify_cert << "\n"; return 1; } if (verify_rule->verify_time != verify_rules[i]->verify_time) { - std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want " << verify_rules[i]->verify_time << "\n"; + std::cerr << "FAIL: Got verify time " << verify_rule->verify_time << ", want " + << verify_rules[i]->verify_time << "\n"; return 1; } if (verify_rule->subject_criteria != verify_rules[i]->subject_criteria) { - std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want " << criteriaToString(verify_rules[i]->subject_criteria) << "\n"; + std::cerr << "FAIL: Got subject criteria " << criteriaToString(verify_rule->subject_criteria) << ", want " + << criteriaToString(verify_rules[i]->subject_criteria) << "\n"; return 1; } if (verify_rule->issuer_criteria != verify_rules[i]->issuer_criteria) { - std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want " << criteriaToString(verify_rules[i]->issuer_criteria) << "\n"; + std::cerr << "FAIL: Got issuer criteria " << criteriaToString(verify_rule->issuer_criteria) << ", want " + << criteriaToString(verify_rules[i]->issuer_criteria) << "\n"; return 1; } if (verify_rule->root_criteria != verify_rules[i]->root_criteria) { - std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want " << criteriaToString(verify_rules[i]->root_criteria) << "\n"; + std::cerr << "FAIL: Got root criteria " << criteriaToString(verify_rule->root_criteria) << ", want " + << criteriaToString(verify_rules[i]->root_criteria) << "\n"; return 1; } i++; @@ -157,8 +173,7 @@ static int policy_verify_test() { return 0; } -int main(int argc, char **argv) -{ +int main(int argc, char** argv) { int failed = 0; #define EXACT(x) Criteria(x, MatchType::EXACT, X509Location::NAME) @@ -173,29 +188,60 @@ int main(int argc, char **argv) FDBLibTLSVerifyTest("Check.Unexpired=0", true, false, {}, {}, {}), FDBLibTLSVerifyTest("Check.Valid=1,Check.Unexpired=0", true, false, {}, {}, {}), FDBLibTLSVerifyTest("Check.Unexpired=0,Check.Valid=0", false, false, {}, {}, {}), - FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC", true, false, - {{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}), - FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC", true, false, - {{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {{NID_countryName, EXACT("US")}}, {}), - FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC", true, false, - {{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp= LLC")}}, {}, {{NID_countryName, EXACT("US")}}), - FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC", true, false, - {{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}), - FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC", true, false, - {{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp=LLC")}}, {{NID_countryName, EXACT("US")}}, {}), - FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC", true, true, - {{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, {{NID_countryName, EXACT("US")}}, {}), - FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc", true, true, - {{NID_countryName, EXACT("US")}, {NID_organizationName, EXACT("XYZCorp, LLC")}}, - {{NID_countryName, EXACT("US")}}, - {{NID_commonName, EXACT("abc")}}), - FDBLibTLSVerifyTest("C=\\,S=abc", true, true, {{NID_countryName, EXACT(",S=abc")}}, {}, {}), - FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, {{NID_commonName, EXACT("abc")}}, {}, {}), - FDBLibTLSVerifyTest("CN=a\\62c", true, true, {{NID_commonName, EXACT("abc")}}, {}, {}), - FDBLibTLSVerifyTest("CN=a\\01c", true, true, {{NID_commonName, EXACT("a\001c")}}, {}, {}), - FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp", true, true, {{NID_subject_alt_name, {"XYZCorp", MatchType::EXACT, X509Location::EXTENSION}}}, {}, {}), - FDBLibTLSVerifyTest("S.O>=XYZ", true, true, {{NID_organizationName, PREFIX("XYZ")}}, {}, {}), - FDBLibTLSVerifyTest("S.O<=LLC", true, true, {{NID_organizationName, SUFFIX("LLC")}}, {}, {}), + FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\, LLC", + true, + false, + { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } }, + { { NID_countryName, EXACT("US") } }, + {}), + FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp\\= LLC", + true, + false, + { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } }, + { { NID_countryName, EXACT("US") } }, + {}), + FDBLibTLSVerifyTest("Check.Unexpired=0,R.C=US,C=US,S.O=XYZCorp\\= LLC", + true, + false, + { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp= LLC") } }, + {}, + { { NID_countryName, EXACT("US") } }), + FDBLibTLSVerifyTest("Check.Unexpired=0,I.C=US,C=US,S.O=XYZCorp=LLC", + true, + false, + { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } }, + { { NID_countryName, EXACT("US") } }, + {}), + FDBLibTLSVerifyTest("I.C=US,C=US,Check.Unexpired=0,S.O=XYZCorp=LLC", + true, + false, + { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp=LLC") } }, + { { NID_countryName, EXACT("US") } }, + {}), + FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC", + true, + true, + { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } }, + { { NID_countryName, EXACT("US") } }, + {}), + FDBLibTLSVerifyTest("I.C=US,C=US,S.O=XYZCorp\\, LLC,R.CN=abc", + true, + true, + { { NID_countryName, EXACT("US") }, { NID_organizationName, EXACT("XYZCorp, LLC") } }, + { { NID_countryName, EXACT("US") } }, + { { NID_commonName, EXACT("abc") } }), + FDBLibTLSVerifyTest("C=\\,S=abc", true, true, { { NID_countryName, EXACT(",S=abc") } }, {}, {}), + FDBLibTLSVerifyTest("CN=\\61\\62\\63", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}), + FDBLibTLSVerifyTest("CN=a\\62c", true, true, { { NID_commonName, EXACT("abc") } }, {}, {}), + FDBLibTLSVerifyTest("CN=a\\01c", true, true, { { NID_commonName, EXACT("a\001c") } }, {}, {}), + FDBLibTLSVerifyTest("S.subjectAltName=XYZCorp", + true, + true, + { { NID_subject_alt_name, { "XYZCorp", MatchType::EXACT, X509Location::EXTENSION } } }, + {}, + {}), + FDBLibTLSVerifyTest("S.O>=XYZ", true, true, { { NID_organizationName, PREFIX("XYZ") } }, {}, {}), + FDBLibTLSVerifyTest("S.O<=LLC", true, true, { { NID_organizationName, SUFFIX("LLC") } }, {}, {}), // Invalid cases. FDBLibTLSVerifyTest("Check.Invalid=0"), @@ -212,7 +258,7 @@ int main(int argc, char **argv) #undef PREFIX #undef SUFFIX - for (auto &test: tests) + for (auto& test : tests) failed |= test.run(); failed |= policy_verify_test(); diff --git a/bindings/c/ThreadCleanup.cpp b/bindings/c/ThreadCleanup.cpp index 966e38b800..86f61ba608 100644 --- a/bindings/c/ThreadCleanup.cpp +++ b/bindings/c/ThreadCleanup.cpp @@ -25,17 +25,17 @@ #include -BOOL WINAPI DllMain( HINSTANCE dll, DWORD reason, LPVOID reserved ) { +BOOL WINAPI DllMain(HINSTANCE dll, DWORD reason, LPVOID reserved) { if (reason == DLL_THREAD_DETACH) releaseAllThreadMagazines(); return TRUE; } -#elif defined( __unixish__ ) +#elif defined(__unixish__) #ifdef __INTEL_COMPILER -#pragma warning ( disable:2415 ) +#pragma warning(disable : 2415) #endif static pthread_key_t threadDestructorKey; @@ -45,13 +45,13 @@ static void threadDestructor(void*) { } void registerThread() { - pthread_setspecific( threadDestructorKey, (const void*)1 ); + pthread_setspecific(threadDestructorKey, (const void*)1); } static int initThreadDestructorKey() { if (!pthread_key_create(&threadDestructorKey, &threadDestructor)) { registerThread(); - setFastAllocatorThreadInitFunction( ®isterThread ); + setFastAllocatorThreadInitFunction(®isterThread); } return 0; diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index 0e53634c40..4cc11130d9 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -52,29 +52,23 @@ int g_api_version = 0; /* This must be true so that we can return the data pointer of a Standalone as an array of FDBKeyValue. */ -static_assert( sizeof(FDBKeyValue) == sizeof(KeyValueRef), - "FDBKeyValue / KeyValueRef size mismatch" ); - +static_assert(sizeof(FDBKeyValue) == sizeof(KeyValueRef), "FDBKeyValue / KeyValueRef size mismatch"); #define TSAV_ERROR(type, error) ((FDBFuture*)(ThreadFuture(error())).extractPtr()) - -extern "C" DLLEXPORT -const char *fdb_get_error( fdb_error_t code ) { - return Error::fromUnvalidatedCode( code ).what(); +extern "C" DLLEXPORT const char* fdb_get_error(fdb_error_t code) { + return Error::fromUnvalidatedCode(code).what(); } -extern "C" DLLEXPORT -fdb_bool_t fdb_error_predicate( int predicate_test, fdb_error_t code ) { - if(predicate_test == FDBErrorPredicates::RETRYABLE) { - return fdb_error_predicate( FDBErrorPredicates::MAYBE_COMMITTED, code ) || - fdb_error_predicate( FDBErrorPredicates::RETRYABLE_NOT_COMMITTED, code ); +extern "C" DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_error_t code) { + if (predicate_test == FDBErrorPredicates::RETRYABLE) { + return fdb_error_predicate(FDBErrorPredicates::MAYBE_COMMITTED, code) || + fdb_error_predicate(FDBErrorPredicates::RETRYABLE_NOT_COMMITTED, code); } - if(predicate_test == FDBErrorPredicates::MAYBE_COMMITTED) { - return code == error_code_commit_unknown_result || - code == error_code_cluster_version_changed; + if (predicate_test == FDBErrorPredicates::MAYBE_COMMITTED) { + return code == error_code_commit_unknown_result || code == error_code_cluster_version_changed; } - if(predicate_test == FDBErrorPredicates::RETRYABLE_NOT_COMMITTED) { + if (predicate_test == FDBErrorPredicates::RETRYABLE_NOT_COMMITTED) { return code == error_code_not_committed || code == error_code_transaction_too_old || code == error_code_future_version || code == error_code_database_locked || code == error_code_proxy_memory_limit_exceeded || code == error_code_batch_transaction_throttled || @@ -83,93 +77,93 @@ fdb_bool_t fdb_error_predicate( int predicate_test, fdb_error_t code ) { return false; } -#define RETURN_ON_ERROR(code_to_run) \ - try { code_to_run } \ - catch( Error& e) { if (e.code() <= 0) return internal_error().code(); else return e.code(); } \ - catch( ... ) { return error_code_unknown_error; } +#define RETURN_ON_ERROR(code_to_run) \ + try { \ + code_to_run \ + } catch (Error & e) { \ + if (e.code() <= 0) \ + return internal_error().code(); \ + else \ + return e.code(); \ + } catch (...) { \ + return error_code_unknown_error; \ + } -#define CATCH_AND_RETURN(code_to_run) \ - RETURN_ON_ERROR(code_to_run); \ +#define CATCH_AND_RETURN(code_to_run) \ + RETURN_ON_ERROR(code_to_run); \ return error_code_success; -#define CATCH_AND_DIE(code_to_run) \ - try { code_to_run } \ - catch ( Error& e ) { fprintf( stderr, "Unexpected FDB error %d\n", e.code() ); abort(); } \ - catch ( ... ) { fprintf( stderr, "Unexpected FDB unknown error\n" ); abort(); } +#define CATCH_AND_DIE(code_to_run) \ + try { \ + code_to_run \ + } catch (Error & e) { \ + fprintf(stderr, "Unexpected FDB error %d\n", e.code()); \ + abort(); \ + } catch (...) { \ + fprintf(stderr, "Unexpected FDB unknown error\n"); \ + abort(); \ + } -extern "C" DLLEXPORT -fdb_error_t fdb_network_set_option( FDBNetworkOption option, - uint8_t const* value, - int value_length ) -{ - CATCH_AND_RETURN( - API->setNetworkOption( (FDBNetworkOptions::Option)option, value ? StringRef( value, value_length ) : Optional() ); ); +extern "C" DLLEXPORT fdb_error_t fdb_network_set_option(FDBNetworkOption option, + uint8_t const* value, + int value_length) { + CATCH_AND_RETURN(API->setNetworkOption((FDBNetworkOptions::Option)option, + value ? StringRef(value, value_length) : Optional());); } fdb_error_t fdb_setup_network_impl() { - CATCH_AND_RETURN( API->setupNetwork(); ); + CATCH_AND_RETURN(API->setupNetwork();); } -fdb_error_t fdb_setup_network_v13( const char* localAddress ) { - fdb_error_t errorCode = fdb_network_set_option( FDB_NET_OPTION_LOCAL_ADDRESS, (uint8_t const*)localAddress, strlen(localAddress) ); - if(errorCode != 0) +fdb_error_t fdb_setup_network_v13(const char* localAddress) { + fdb_error_t errorCode = + fdb_network_set_option(FDB_NET_OPTION_LOCAL_ADDRESS, (uint8_t const*)localAddress, strlen(localAddress)); + if (errorCode != 0) return errorCode; return fdb_setup_network_impl(); } -extern "C" DLLEXPORT -fdb_error_t fdb_run_network() { - CATCH_AND_RETURN( API->runNetwork(); ); +extern "C" DLLEXPORT fdb_error_t fdb_run_network() { + CATCH_AND_RETURN(API->runNetwork();); } -extern "C" DLLEXPORT -fdb_error_t fdb_stop_network() { - CATCH_AND_RETURN( API->stopNetwork(); ); +extern "C" DLLEXPORT fdb_error_t fdb_stop_network() { + CATCH_AND_RETURN(API->stopNetwork();); } -extern "C" DLLEXPORT -fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void *hook_parameter) { - CATCH_AND_RETURN( API->addNetworkThreadCompletionHook(hook, hook_parameter); ); +extern "C" DLLEXPORT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void* hook_parameter) { + CATCH_AND_RETURN(API->addNetworkThreadCompletionHook(hook, hook_parameter);); } -extern "C" DLLEXPORT -void fdb_future_cancel( FDBFuture* f ) { - CATCH_AND_DIE( - TSAVB(f)->addref(); - TSAVB(f)->cancel(); - ); +extern "C" DLLEXPORT void fdb_future_cancel(FDBFuture* f) { + CATCH_AND_DIE(TSAVB(f)->addref(); TSAVB(f)->cancel();); } -extern "C" DLLEXPORT -void fdb_future_release_memory( FDBFuture* f ) { - CATCH_AND_DIE( TSAVB(f)->releaseMemory(); ); +extern "C" DLLEXPORT void fdb_future_release_memory(FDBFuture* f) { + CATCH_AND_DIE(TSAVB(f)->releaseMemory();); } -extern "C" DLLEXPORT -void fdb_future_destroy( FDBFuture* f ) { - CATCH_AND_DIE( TSAVB(f)->cancel(); ); +extern "C" DLLEXPORT void fdb_future_destroy(FDBFuture* f) { + CATCH_AND_DIE(TSAVB(f)->cancel();); } -extern "C" DLLEXPORT -fdb_error_t fdb_future_block_until_ready( FDBFuture* f ) { +extern "C" DLLEXPORT fdb_error_t fdb_future_block_until_ready(FDBFuture* f) { CATCH_AND_RETURN(TSAVB(f)->blockUntilReadyCheckOnMainThread();); } -fdb_bool_t fdb_future_is_error_v22( FDBFuture* f ) { +fdb_bool_t fdb_future_is_error_v22(FDBFuture* f) { return TSAVB(f)->isError(); } -extern "C" DLLEXPORT -fdb_bool_t fdb_future_is_ready( FDBFuture* f ) { +extern "C" DLLEXPORT fdb_bool_t fdb_future_is_ready(FDBFuture* f) { return TSAVB(f)->isReady(); } class CAPICallback : public ThreadCallback { public: - CAPICallback(void (*callbackf)(FDBFuture*, void*), FDBFuture* f, - void* userdata) - : callbackf(callbackf), f(f), userdata(userdata) {} + CAPICallback(void (*callbackf)(FDBFuture*, void*), FDBFuture* f, void* userdata) + : callbackf(callbackf), f(f), userdata(userdata) {} virtual bool canFire(int notMadeActive) { return true; } virtual void fire(const Void& unused, int& userParam) { @@ -187,249 +181,217 @@ private: void* userdata; }; -extern "C" DLLEXPORT -fdb_error_t fdb_future_set_callback( FDBFuture* f, - void (*callbackf)(FDBFuture*, void*), - void* userdata ) { +extern "C" DLLEXPORT fdb_error_t fdb_future_set_callback(FDBFuture* f, + void (*callbackf)(FDBFuture*, void*), + void* userdata) { CAPICallback* cb = new CAPICallback(callbackf, f, userdata); int ignore; - CATCH_AND_RETURN( TSAVB(f)->callOrSetAsCallback( cb, ignore, 0 ); ); + CATCH_AND_RETURN(TSAVB(f)->callOrSetAsCallback(cb, ignore, 0);); } -fdb_error_t fdb_future_get_error_impl( FDBFuture* f ) { +fdb_error_t fdb_future_get_error_impl(FDBFuture* f) { return TSAVB(f)->getErrorCode(); } -fdb_error_t fdb_future_get_error_v22( FDBFuture* f, const char** description ) { - if ( !( TSAVB(f)->isError() ) ) +fdb_error_t fdb_future_get_error_v22(FDBFuture* f, const char** description) { + if (!(TSAVB(f)->isError())) return error_code_future_not_error; if (description) *description = TSAVB(f)->error.what(); return TSAVB(f)->error.code(); } -fdb_error_t fdb_future_get_version_v619( FDBFuture* f, int64_t* out_version ) { - CATCH_AND_RETURN( *out_version = TSAV(Version, f)->get(); ); +fdb_error_t fdb_future_get_version_v619(FDBFuture* f, int64_t* out_version) { + CATCH_AND_RETURN(*out_version = TSAV(Version, f)->get();); } -extern "C" DLLEXPORT -fdb_error_t fdb_future_get_int64( FDBFuture* f, int64_t* out_value ) { - CATCH_AND_RETURN( *out_value = TSAV(int64_t, f)->get(); ); +extern "C" DLLEXPORT fdb_error_t fdb_future_get_int64(FDBFuture* f, int64_t* out_value) { + CATCH_AND_RETURN(*out_value = TSAV(int64_t, f)->get();); } -extern "C" DLLEXPORT -fdb_error_t fdb_future_get_key( FDBFuture* f, uint8_t const** out_key, - int* out_key_length ) { - CATCH_AND_RETURN( - KeyRef key = TSAV(Key, f)->get(); - *out_key = key.begin(); - *out_key_length = key.size(); ); +extern "C" DLLEXPORT fdb_error_t fdb_future_get_key(FDBFuture* f, uint8_t const** out_key, int* out_key_length) { + CATCH_AND_RETURN(KeyRef key = TSAV(Key, f)->get(); *out_key = key.begin(); *out_key_length = key.size();); } -fdb_error_t fdb_future_get_cluster_v609( FDBFuture* f, FDBCluster** out_cluster ) { - CATCH_AND_RETURN( - *out_cluster = (FDBCluster*) - ( (TSAV( char*, f )->get() ) ); ); +fdb_error_t fdb_future_get_cluster_v609(FDBFuture* f, FDBCluster** out_cluster) { + CATCH_AND_RETURN(*out_cluster = (FDBCluster*)((TSAV(char*, f)->get()));); } -fdb_error_t fdb_future_get_database_v609( FDBFuture* f, FDBDatabase** out_database ) { - CATCH_AND_RETURN( - *out_database = (FDBDatabase*) - ( (TSAV( Reference, f )->get() ).extractPtr() ); ); +fdb_error_t fdb_future_get_database_v609(FDBFuture* f, FDBDatabase** out_database) { + CATCH_AND_RETURN(*out_database = (FDBDatabase*)((TSAV(Reference, f)->get()).extractPtr());); } -extern "C" DLLEXPORT -fdb_error_t fdb_future_get_value( FDBFuture* f, fdb_bool_t* out_present, - uint8_t const** out_value, int* out_value_length ) { - CATCH_AND_RETURN( - Optional v = TSAV(Optional, f)->get(); - *out_present = v.present(); - if (*out_present) { - *out_value = v.get().begin(); - *out_value_length = v.get().size(); - } ); +extern "C" DLLEXPORT fdb_error_t fdb_future_get_value(FDBFuture* f, + fdb_bool_t* out_present, + uint8_t const** out_value, + int* out_value_length) { + CATCH_AND_RETURN(Optional v = TSAV(Optional, f)->get(); *out_present = v.present(); + if (*out_present) { + *out_value = v.get().begin(); + *out_value_length = v.get().size(); + }); } -fdb_error_t fdb_future_get_keyvalue_array_impl( - FDBFuture* f, FDBKeyValue const** out_kv, - int* out_count, fdb_bool_t* out_more ) -{ - CATCH_AND_RETURN( - Standalone rrr = TSAV(Standalone, f)->get(); - *out_kv = (FDBKeyValue*)rrr.begin(); - *out_count = rrr.size(); - *out_more = rrr.more; ); +fdb_error_t fdb_future_get_keyvalue_array_impl(FDBFuture* f, + FDBKeyValue const** out_kv, + int* out_count, + fdb_bool_t* out_more) { + CATCH_AND_RETURN(Standalone rrr = TSAV(Standalone, f)->get(); + *out_kv = (FDBKeyValue*)rrr.begin(); + *out_count = rrr.size(); + *out_more = rrr.more;); } -fdb_error_t fdb_future_get_keyvalue_array_v13( - FDBFuture* f, FDBKeyValue const** out_kv, int* out_count) -{ - CATCH_AND_RETURN( - Standalone rrr = TSAV(Standalone, f)->get(); - *out_kv = (FDBKeyValue*)rrr.begin(); - *out_count = rrr.size(); ); +fdb_error_t fdb_future_get_keyvalue_array_v13(FDBFuture* f, FDBKeyValue const** out_kv, int* out_count) { + CATCH_AND_RETURN(Standalone rrr = TSAV(Standalone, f)->get(); + *out_kv = (FDBKeyValue*)rrr.begin(); + *out_count = rrr.size();); } -extern "C" DLLEXPORT -fdb_error_t fdb_future_get_string_array( - FDBFuture* f, const char*** out_strings, int* out_count) -{ - CATCH_AND_RETURN( - Standalone> na = TSAV(Standalone>, f)->get(); - *out_strings = (const char **) na.begin(); - *out_count = na.size(); - ); +extern "C" DLLEXPORT fdb_error_t fdb_future_get_string_array(FDBFuture* f, const char*** out_strings, int* out_count) { + CATCH_AND_RETURN(Standalone> na = TSAV(Standalone>, f)->get(); + *out_strings = (const char**)na.begin(); + *out_count = na.size();); } -FDBFuture* fdb_create_cluster_v609( const char* cluster_file_path ) { - char *path; - if(cluster_file_path) { +FDBFuture* fdb_create_cluster_v609(const char* cluster_file_path) { + char* path; + if (cluster_file_path) { path = new char[strlen(cluster_file_path) + 1]; strcpy(path, cluster_file_path); - } - else { + } else { path = new char[1]; path[0] = '\0'; } return (FDBFuture*)ThreadFuture(path).extractPtr(); } -fdb_error_t fdb_cluster_set_option_v609( FDBCluster* c, - FDBClusterOption option, - uint8_t const* value, - int value_length ) -{ +fdb_error_t fdb_cluster_set_option_v609(FDBCluster* c, + FDBClusterOption option, + uint8_t const* value, + int value_length) { // There are no cluster options return error_code_success; } -void fdb_cluster_destroy_v609( FDBCluster* c ) { - CATCH_AND_DIE( delete[] CLUSTER(c); ); +void fdb_cluster_destroy_v609(FDBCluster* c) { + CATCH_AND_DIE(delete[] CLUSTER(c);); } // This exists so that fdb_cluster_create_database doesn't need to call the public symbol fdb_create_database. // If it does and this is an external client loaded though the multi-version API, then it may inadvertently call // the version of the function in the primary library if it was loaded into the global symbols. -fdb_error_t fdb_create_database_impl( const char* cluster_file_path, FDBDatabase** out_database ) { - CATCH_AND_RETURN( - *out_database = (FDBDatabase*)API->createDatabase( cluster_file_path ? cluster_file_path : "" ).extractPtr(); - ); +fdb_error_t fdb_create_database_impl(const char* cluster_file_path, FDBDatabase** out_database) { + CATCH_AND_RETURN(*out_database = + (FDBDatabase*)API->createDatabase(cluster_file_path ? cluster_file_path : "").extractPtr();); } -FDBFuture* fdb_cluster_create_database_v609( FDBCluster* c, uint8_t const* db_name, - int db_name_length ) -{ - if(strncmp((const char*)db_name, "DB", db_name_length) != 0) { +FDBFuture* fdb_cluster_create_database_v609(FDBCluster* c, uint8_t const* db_name, int db_name_length) { + if (strncmp((const char*)db_name, "DB", db_name_length) != 0) { return (FDBFuture*)ThreadFuture>(invalid_database_name()).extractPtr(); } - FDBDatabase *db; + FDBDatabase* db; fdb_error_t err = fdb_create_database_impl(CLUSTER(c), &db); - if(err) { + if (err) { return (FDBFuture*)ThreadFuture>(Error(err)).extractPtr(); } return (FDBFuture*)ThreadFuture>(Reference(DB(db))).extractPtr(); } -extern "C" DLLEXPORT -fdb_error_t fdb_create_database( const char* cluster_file_path, FDBDatabase** out_database ) { - return fdb_create_database_impl( cluster_file_path, out_database ); +extern "C" DLLEXPORT fdb_error_t fdb_create_database(const char* cluster_file_path, FDBDatabase** out_database) { + return fdb_create_database_impl(cluster_file_path, out_database); } -extern "C" DLLEXPORT -fdb_error_t fdb_database_set_option( FDBDatabase* d, - FDBDatabaseOption option, - uint8_t const* value, - int value_length ) -{ - CATCH_AND_RETURN( - DB(d)->setOption( (FDBDatabaseOptions::Option)option, value ? StringRef( value, value_length ) : Optional() ); ); +extern "C" DLLEXPORT fdb_error_t fdb_database_set_option(FDBDatabase* d, + FDBDatabaseOption option, + uint8_t const* value, + int value_length) { + CATCH_AND_RETURN(DB(d)->setOption((FDBDatabaseOptions::Option)option, + value ? StringRef(value, value_length) : Optional());); } -extern "C" DLLEXPORT -void fdb_database_destroy( FDBDatabase* d ) { - CATCH_AND_DIE( DB(d)->delref(); ); +extern "C" DLLEXPORT void fdb_database_destroy(FDBDatabase* d) { + CATCH_AND_DIE(DB(d)->delref();); } -extern "C" DLLEXPORT -fdb_error_t fdb_database_create_transaction( FDBDatabase* d, - FDBTransaction** out_transaction ) -{ - CATCH_AND_RETURN( - Reference tr = DB(d)->createTransaction(); - if(g_api_version <= 15) - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - *out_transaction = (FDBTransaction*)tr.extractPtr(); ); +extern "C" DLLEXPORT fdb_error_t fdb_database_create_transaction(FDBDatabase* d, FDBTransaction** out_transaction) { + CATCH_AND_RETURN(Reference tr = DB(d)->createTransaction(); + if (g_api_version <= 15) tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + *out_transaction = (FDBTransaction*)tr.extractPtr();); } - -extern "C" DLLEXPORT -void fdb_transaction_destroy( FDBTransaction* tr ) { +extern "C" DLLEXPORT void fdb_transaction_destroy(FDBTransaction* tr) { try { TXN(tr)->delref(); - } catch ( ... ) { } + } catch (...) { + } } -extern "C" DLLEXPORT -void fdb_transaction_cancel( FDBTransaction* tr ) { - CATCH_AND_DIE( TXN(tr)->cancel(); ); +extern "C" DLLEXPORT void fdb_transaction_cancel(FDBTransaction* tr) { + CATCH_AND_DIE(TXN(tr)->cancel();); } -extern "C" DLLEXPORT -void fdb_transaction_set_read_version( FDBTransaction* tr, int64_t version ) { - CATCH_AND_DIE( TXN(tr)->setVersion( version ); ); +extern "C" DLLEXPORT void fdb_transaction_set_read_version(FDBTransaction* tr, int64_t version) { + CATCH_AND_DIE(TXN(tr)->setVersion(version);); } -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_get_read_version( FDBTransaction* tr ) { - return (FDBFuture*)( TXN(tr)->getReadVersion().extractPtr() ); +extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_read_version(FDBTransaction* tr) { + return (FDBFuture*)(TXN(tr)->getReadVersion().extractPtr()); } -FDBFuture* fdb_transaction_get_impl( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length, fdb_bool_t snapshot ) { - return (FDBFuture*) - ( TXN(tr)->get( KeyRef( key_name, key_name_length ), snapshot ).extractPtr() ); +FDBFuture* fdb_transaction_get_impl(FDBTransaction* tr, + uint8_t const* key_name, + int key_name_length, + fdb_bool_t snapshot) { + return (FDBFuture*)(TXN(tr)->get(KeyRef(key_name, key_name_length), snapshot).extractPtr()); } -FDBFuture* fdb_transaction_get_v13( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length ) -{ - return fdb_transaction_get_impl( tr, key_name, key_name_length, 0 ); +FDBFuture* fdb_transaction_get_v13(FDBTransaction* tr, uint8_t const* key_name, int key_name_length) { + return fdb_transaction_get_impl(tr, key_name, key_name_length, 0); } -FDBFuture* fdb_transaction_get_key_impl( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length, fdb_bool_t or_equal, - int offset, fdb_bool_t snapshot ) { - return (FDBFuture*)( TXN(tr)->getKey( KeySelectorRef( - KeyRef( key_name, - key_name_length ), - or_equal, offset ), - snapshot ).extractPtr() ); +FDBFuture* fdb_transaction_get_key_impl(FDBTransaction* tr, + uint8_t const* key_name, + int key_name_length, + fdb_bool_t or_equal, + int offset, + fdb_bool_t snapshot) { + return (FDBFuture*)(TXN(tr) + ->getKey(KeySelectorRef(KeyRef(key_name, key_name_length), or_equal, offset), snapshot) + .extractPtr()); } -FDBFuture* fdb_transaction_get_key_v13( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length, fdb_bool_t or_equal, - int offset ) { - return fdb_transaction_get_key_impl( tr, key_name, key_name_length, - or_equal, offset, false ); +FDBFuture* fdb_transaction_get_key_v13(FDBTransaction* tr, + uint8_t const* key_name, + int key_name_length, + fdb_bool_t or_equal, + int offset) { + return fdb_transaction_get_key_impl(tr, key_name, key_name_length, or_equal, offset, false); } -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_get_addresses_for_key( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length ){ - return (FDBFuture*)( TXN(tr)->getAddressesForKey( KeyRef(key_name, key_name_length) ).extractPtr() ); - +extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_addresses_for_key(FDBTransaction* tr, + uint8_t const* key_name, + int key_name_length) { + return (FDBFuture*)(TXN(tr)->getAddressesForKey(KeyRef(key_name, key_name_length)).extractPtr()); } -FDBFuture* fdb_transaction_get_range_impl( - FDBTransaction* tr, uint8_t const* begin_key_name, - int begin_key_name_length, fdb_bool_t begin_or_equal, int begin_offset, - uint8_t const* end_key_name, int end_key_name_length, - fdb_bool_t end_or_equal, int end_offset, int limit, int target_bytes, - FDBStreamingMode mode, int iteration, fdb_bool_t snapshot, - fdb_bool_t reverse ) -{ +FDBFuture* fdb_transaction_get_range_impl(FDBTransaction* tr, + uint8_t const* begin_key_name, + int begin_key_name_length, + fdb_bool_t begin_or_equal, + int begin_offset, + uint8_t const* end_key_name, + int end_key_name_length, + fdb_bool_t end_or_equal, + int end_offset, + int limit, + int target_bytes, + FDBStreamingMode mode, + int iteration, + fdb_bool_t snapshot, + fdb_bool_t reverse) { /* This method may be called with a runtime API version of 13, in which negative row limits are a reverse range read */ if (g_api_version <= 13 && limit < 0) { @@ -444,7 +406,8 @@ FDBFuture* fdb_transaction_get_range_impl( target_bytes = CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED; /* Unlimited/unlimited with mode _EXACT isn't permitted */ - if (limit == CLIENT_KNOBS->ROW_LIMIT_UNLIMITED && target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED && mode == FDB_STREAMING_MODE_EXACT) + if (limit == CLIENT_KNOBS->ROW_LIMIT_UNLIMITED && target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED && + mode == FDB_STREAMING_MODE_EXACT) return TSAV_ERROR(Standalone, exact_mode_without_limits); /* _ITERATOR mode maps to one of the known streaming modes @@ -453,12 +416,14 @@ FDBFuture* fdb_transaction_get_range_impl( /* The progression used for FDB_STREAMING_MODE_ITERATOR. Goes from small -> medium -> large. Then 1.5 * previous until serial. */ - static const int iteration_progression[] = { 256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000 }; + static const int iteration_progression[] = { + 256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000 + }; /* length(iteration_progression) */ static const int max_iteration = sizeof(iteration_progression) / sizeof(int); - if(mode == FDB_STREAMING_MODE_WANT_ALL) + if (mode == FDB_STREAMING_MODE_WANT_ALL) mode = FDB_STREAMING_MODE_SERIAL; int mode_bytes; @@ -468,180 +433,177 @@ FDBFuture* fdb_transaction_get_range_impl( iteration = std::min(iteration, max_iteration); mode_bytes = iteration_progression[iteration - 1]; - } - else if(mode >= 0 && mode <= FDB_STREAMING_MODE_SERIAL) + } else if (mode >= 0 && mode <= FDB_STREAMING_MODE_SERIAL) mode_bytes = mode_bytes_array[mode]; else return TSAV_ERROR(Standalone, client_invalid_operation); - if(target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED) + if (target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED) target_bytes = mode_bytes; - else if(mode_bytes != CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED) + else if (mode_bytes != CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED) target_bytes = std::min(target_bytes, mode_bytes); - return (FDBFuture*)( TXN(tr)->getRange( - KeySelectorRef( - KeyRef( begin_key_name, - begin_key_name_length ), - begin_or_equal, begin_offset ), - KeySelectorRef( - KeyRef( end_key_name, - end_key_name_length ), - end_or_equal, end_offset ), - GetRangeLimits(limit, target_bytes), - snapshot, reverse ).extractPtr() ); + return ( + FDBFuture*)(TXN(tr) + ->getRange( + KeySelectorRef(KeyRef(begin_key_name, begin_key_name_length), begin_or_equal, begin_offset), + KeySelectorRef(KeyRef(end_key_name, end_key_name_length), end_or_equal, end_offset), + GetRangeLimits(limit, target_bytes), + snapshot, + reverse) + .extractPtr()); } -FDBFuture* fdb_transaction_get_range_selector_v13( - FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, - fdb_bool_t begin_or_equal, int begin_offset, uint8_t const* end_key_name, - int end_key_name_length, fdb_bool_t end_or_equal, int end_offset, int limit ) -{ - return fdb_transaction_get_range_impl( - tr, begin_key_name, begin_key_name_length, begin_or_equal, begin_offset, - end_key_name, end_key_name_length, end_or_equal, end_offset, - limit, 0, FDB_STREAMING_MODE_EXACT, 0, false, false); +FDBFuture* fdb_transaction_get_range_selector_v13(FDBTransaction* tr, + uint8_t const* begin_key_name, + int begin_key_name_length, + fdb_bool_t begin_or_equal, + int begin_offset, + uint8_t const* end_key_name, + int end_key_name_length, + fdb_bool_t end_or_equal, + int end_offset, + int limit) { + return fdb_transaction_get_range_impl(tr, + begin_key_name, + begin_key_name_length, + begin_or_equal, + begin_offset, + end_key_name, + end_key_name_length, + end_or_equal, + end_offset, + limit, + 0, + FDB_STREAMING_MODE_EXACT, + 0, + false, + false); } -FDBFuture* fdb_transaction_get_range_v13( - FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, - uint8_t const* end_key_name, int end_key_name_length, int limit ) -{ +FDBFuture* fdb_transaction_get_range_v13(FDBTransaction* tr, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length, + int limit) { return fdb_transaction_get_range_selector_v13( - tr, - FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(begin_key_name, - begin_key_name_length), - FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(end_key_name, - end_key_name_length), - limit ); + tr, + FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(begin_key_name, begin_key_name_length), + FDB_KEYSEL_FIRST_GREATER_OR_EQUAL(end_key_name, end_key_name_length), + limit); } -extern "C" DLLEXPORT -void fdb_transaction_set( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length, uint8_t const* value, - int value_length ) { +extern "C" DLLEXPORT void fdb_transaction_set(FDBTransaction* tr, + uint8_t const* key_name, + int key_name_length, + uint8_t const* value, + int value_length) { + CATCH_AND_DIE(TXN(tr)->set(KeyRef(key_name, key_name_length), ValueRef(value, value_length));); +} + +extern "C" DLLEXPORT void fdb_transaction_atomic_op(FDBTransaction* tr, + uint8_t const* key_name, + int key_name_length, + uint8_t const* param, + int param_length, + FDBMutationType operation_type) { + CATCH_AND_DIE(TXN(tr)->atomicOp( + KeyRef(key_name, key_name_length), ValueRef(param, param_length), (FDBMutationTypes::Option)operation_type);); +} + +extern "C" DLLEXPORT void fdb_transaction_clear(FDBTransaction* tr, uint8_t const* key_name, int key_name_length) { + CATCH_AND_DIE(TXN(tr)->clear(KeyRef(key_name, key_name_length));); +} + +extern "C" DLLEXPORT void fdb_transaction_clear_range(FDBTransaction* tr, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length) { CATCH_AND_DIE( - TXN(tr)->set( KeyRef( key_name, key_name_length ), - ValueRef( value, value_length ) ); ); + TXN(tr)->clear(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));); } -extern "C" DLLEXPORT -void fdb_transaction_atomic_op( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length, uint8_t const* param, - int param_length, FDBMutationType operation_type ) { - CATCH_AND_DIE( - TXN(tr)->atomicOp( KeyRef( key_name, key_name_length ), - ValueRef( param, param_length ), - (FDBMutationTypes::Option) operation_type ); ); +extern "C" DLLEXPORT FDBFuture* fdb_transaction_watch(FDBTransaction* tr, + uint8_t const* key_name, + int key_name_length) { + return (FDBFuture*)(TXN(tr)->watch(KeyRef(key_name, key_name_length)).extractPtr()); } -extern "C" DLLEXPORT -void fdb_transaction_clear( FDBTransaction* tr, uint8_t const* key_name, - int key_name_length ) { - CATCH_AND_DIE( - TXN(tr)->clear( KeyRef( key_name, key_name_length ) ); ); +extern "C" DLLEXPORT FDBFuture* fdb_transaction_commit(FDBTransaction* tr) { + return (FDBFuture*)(TXN(tr)->commit().extractPtr()); } -extern "C" DLLEXPORT -void fdb_transaction_clear_range( - FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, - uint8_t const* end_key_name, int end_key_name_length ) -{ - CATCH_AND_DIE( - TXN(tr)->clear( KeyRef( begin_key_name, - begin_key_name_length ), - KeyRef( end_key_name, - end_key_name_length ) ); ); +extern "C" DLLEXPORT fdb_error_t fdb_transaction_get_committed_version(FDBTransaction* tr, int64_t* out_version) { + CATCH_AND_RETURN(*out_version = TXN(tr)->getCommittedVersion();); } -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_watch( FDBTransaction *tr, uint8_t const* key_name, - int key_name_length) -{ - return (FDBFuture*)( TXN(tr)->watch(KeyRef(key_name, key_name_length)).extractPtr() ); -} - -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_commit( FDBTransaction* tr ) { - return (FDBFuture*)( TXN(tr)->commit().extractPtr() ); -} - -extern "C" DLLEXPORT -fdb_error_t fdb_transaction_get_committed_version( FDBTransaction* tr, - int64_t* out_version ) -{ - CATCH_AND_RETURN( - *out_version = TXN(tr)->getCommittedVersion(); ); -} - -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* tr) { +extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* tr) { return (FDBFuture*)TXN(tr)->getApproximateSize().extractPtr(); } -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_get_versionstamp( FDBTransaction* tr ) -{ +extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_versionstamp(FDBTransaction* tr) { return (FDBFuture*)(TXN(tr)->getVersionstamp().extractPtr()); } -fdb_error_t fdb_transaction_set_option_impl( FDBTransaction* tr, - FDBTransactionOption option, - uint8_t const* value, - int value_length ) -{ +fdb_error_t fdb_transaction_set_option_impl(FDBTransaction* tr, + FDBTransactionOption option, + uint8_t const* value, + int value_length) { + CATCH_AND_RETURN(TXN(tr)->setOption((FDBTransactionOptions::Option)option, + value ? StringRef(value, value_length) : Optional());); +} + +void fdb_transaction_set_option_v13(FDBTransaction* tr, FDBTransactionOption option) { + fdb_transaction_set_option_impl(tr, option, NULL, 0); +} + +extern "C" DLLEXPORT FDBFuture* fdb_transaction_on_error(FDBTransaction* tr, fdb_error_t error) { + return (FDBFuture*)(TXN(tr)->onError(Error::fromUnvalidatedCode(error)).extractPtr()); +} + +extern "C" DLLEXPORT void fdb_transaction_reset(FDBTransaction* tr) { + CATCH_AND_DIE(TXN(tr)->reset();); +} + +extern "C" DLLEXPORT fdb_error_t fdb_transaction_add_conflict_range(FDBTransaction* tr, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length, + FDBConflictRangeType type) { CATCH_AND_RETURN( - TXN(tr)->setOption( (FDBTransactionOptions::Option)option, value ? StringRef( value, value_length ) : Optional() ); ); + KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); + if (type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_READ) TXN(tr)->addReadConflictRange(range); + else if (type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_WRITE) TXN(tr)->addWriteConflictRange(range); + else return error_code_client_invalid_operation;); } -void fdb_transaction_set_option_v13( FDBTransaction* tr, - FDBTransactionOption option ) -{ - fdb_transaction_set_option_impl( tr, option, NULL, 0 ); -} - -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_on_error( FDBTransaction* tr, fdb_error_t error ) { - return (FDBFuture*)( TXN(tr)->onError( - Error::fromUnvalidatedCode( error ) ).extractPtr() ); -} - -extern "C" DLLEXPORT -void fdb_transaction_reset( FDBTransaction* tr ) { - CATCH_AND_DIE( TXN(tr)->reset(); ); -} - -extern "C" DLLEXPORT -fdb_error_t fdb_transaction_add_conflict_range( FDBTransaction*tr, uint8_t const* begin_key_name, - int begin_key_name_length, uint8_t const* end_key_name, - int end_key_name_length, FDBConflictRangeType type) { - CATCH_AND_RETURN( - KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); - if(type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_READ) - TXN(tr)->addReadConflictRange(range); - else if(type == FDBConflictRangeType::FDB_CONFLICT_RANGE_TYPE_WRITE) - TXN(tr)->addWriteConflictRange(range); - else - return error_code_client_invalid_operation; - ); - -} - -extern "C" DLLEXPORT -FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name, - int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length ) { +extern "C" DLLEXPORT FDBFuture* fdb_transaction_get_estimated_range_size_bytes(FDBTransaction* tr, + uint8_t const* begin_key_name, + int begin_key_name_length, + uint8_t const* end_key_name, + int end_key_name_length) { KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); return (FDBFuture*)(TXN(tr)->getEstimatedRangeSizeBytes(range).extractPtr()); } #include "fdb_c_function_pointers.g.h" -#define FDB_API_CHANGED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) fdb_api_ptr_##func = (void*)&(func##_impl); +#define FDB_API_CHANGED(func, ver) \ + if (header_version < ver) \ + fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); \ + else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) \ + fdb_api_ptr_##func = (void*)&(func##_impl); -#define FDB_API_REMOVED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else fdb_api_ptr_##func = (void*)&fdb_api_ptr_removed; +#define FDB_API_REMOVED(func, ver) \ + if (header_version < ver) \ + fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); \ + else \ + fdb_api_ptr_##func = (void*)&fdb_api_ptr_removed; -extern "C" DLLEXPORT -fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version ) { +extern "C" DLLEXPORT fdb_error_t fdb_select_api_version_impl(int runtime_version, int header_version) { /* Can only call this once */ if (g_api_version != 0) return error_code_api_version_already_set; @@ -658,9 +620,7 @@ fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version if (runtime_version < 13) return error_code_api_version_not_supported; - RETURN_ON_ERROR( - API->selectApiVersion(runtime_version); - ); + RETURN_ON_ERROR(API->selectApiVersion(runtime_version);); g_api_version = runtime_version; @@ -668,52 +628,50 @@ fdb_error_t fdb_select_api_version_impl( int runtime_version, int header_version Error::init(); // Versioned API changes -- descending order by version (new changes at top) - // FDB_API_CHANGED( function, ver ) means there is a new implementation as of ver, and a function function_(ver-1) is the old implementation - // FDB_API_REMOVED( function, ver ) means the function was removed as of ver, and function_(ver-1) is the old implementation + // FDB_API_CHANGED( function, ver ) means there is a new implementation as of ver, and a function function_(ver-1) + // is the old implementation FDB_API_REMOVED( function, ver ) means the function was removed as of ver, and + // function_(ver-1) is the old implementation // - // WARNING: use caution when implementing removed functions by calling public API functions. This can lead to undesired behavior when - // using the multi-version API. Instead, it is better to have both the removed and public functions call an internal implementation function. - // See fdb_create_database_impl for an example. - FDB_API_REMOVED( fdb_future_get_version, 620 ); - FDB_API_REMOVED( fdb_create_cluster, 610 ); - FDB_API_REMOVED( fdb_cluster_create_database, 610 ); - FDB_API_REMOVED( fdb_cluster_set_option, 610 ); - FDB_API_REMOVED( fdb_cluster_destroy, 610 ); - FDB_API_REMOVED( fdb_future_get_cluster, 610 ); - FDB_API_REMOVED( fdb_future_get_database, 610 ); - FDB_API_CHANGED( fdb_future_get_error, 23 ); - FDB_API_REMOVED( fdb_future_is_error, 23 ); - FDB_API_CHANGED( fdb_future_get_keyvalue_array, 14 ); - FDB_API_CHANGED( fdb_transaction_get_key, 14 ); - FDB_API_CHANGED( fdb_transaction_get_range, 14 ); - FDB_API_REMOVED( fdb_transaction_get_range_selector, 14 ); - FDB_API_CHANGED( fdb_transaction_get, 14 ); - FDB_API_CHANGED( fdb_setup_network, 14 ); - FDB_API_CHANGED( fdb_transaction_set_option, 14 ); + // WARNING: use caution when implementing removed functions by calling public API functions. This can lead to + // undesired behavior when using the multi-version API. Instead, it is better to have both the removed and public + // functions call an internal implementation function. See fdb_create_database_impl for an example. + FDB_API_REMOVED(fdb_future_get_version, 620); + FDB_API_REMOVED(fdb_create_cluster, 610); + FDB_API_REMOVED(fdb_cluster_create_database, 610); + FDB_API_REMOVED(fdb_cluster_set_option, 610); + FDB_API_REMOVED(fdb_cluster_destroy, 610); + FDB_API_REMOVED(fdb_future_get_cluster, 610); + FDB_API_REMOVED(fdb_future_get_database, 610); + FDB_API_CHANGED(fdb_future_get_error, 23); + FDB_API_REMOVED(fdb_future_is_error, 23); + FDB_API_CHANGED(fdb_future_get_keyvalue_array, 14); + FDB_API_CHANGED(fdb_transaction_get_key, 14); + FDB_API_CHANGED(fdb_transaction_get_range, 14); + FDB_API_REMOVED(fdb_transaction_get_range_selector, 14); + FDB_API_CHANGED(fdb_transaction_get, 14); + FDB_API_CHANGED(fdb_setup_network, 14); + FDB_API_CHANGED(fdb_transaction_set_option, 14); /* End versioned API changes */ return error_code_success; } -extern "C" DLLEXPORT -int fdb_get_max_api_version() { +extern "C" DLLEXPORT int fdb_get_max_api_version() { return FDB_API_VERSION; } -extern "C" DLLEXPORT -const char* fdb_get_client_version() { +extern "C" DLLEXPORT const char* fdb_get_client_version() { return API->getClientVersion(); } #if defined(__APPLE__) #include -__attribute__((constructor)) -static void initialize() { - //OS X ld doesn't support -z nodelete, so we dlopen to increment the reference count of this module +__attribute__((constructor)) static void initialize() { + // OS X ld doesn't support -z nodelete, so we dlopen to increment the reference count of this module Dl_info info; int ret = dladdr((void*)&fdb_select_api_version_impl, &info); - if(!ret || !info.dli_fname) - return; //If we get here somehow, we face the risk of seg faults if somebody unloads our library + if (!ret || !info.dli_fname) + return; // If we get here somehow, we face the risk of seg faults if somebody unloads our library dlopen(info.dli_fname, RTLD_NOLOAD | RTLD_NODELETE); } diff --git a/bindings/c/foundationdb/ClientWorkload.h b/bindings/c/foundationdb/ClientWorkload.h index edaa35ccdf..0b785e1f31 100644 --- a/bindings/c/foundationdb/ClientWorkload.h +++ b/bindings/c/foundationdb/ClientWorkload.h @@ -44,7 +44,8 @@ enum class FDBSeverity { Debug, Info, Warn, WarnAlways, Error }; class FDBLogger { public: - virtual void trace(FDBSeverity sev, const std::string& name, + virtual void trace(FDBSeverity sev, + const std::string& name, const std::vector>& details) = 0; }; diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c index 6083d1e617..d1f520c171 100644 --- a/bindings/c/test/mako/mako.c +++ b/bindings/c/test/mako/mako.c @@ -139,11 +139,13 @@ int cleanup(FDBTransaction* transaction, mako_args_t* args) { endstr[4] = 0xff; clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); fdb_transaction_clear_range(transaction, (uint8_t*)beginstr, 5, (uint8_t*)endstr, 5); - if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit; + if (commit_transaction(transaction) != FDB_SUCCESS) + goto failExit; fdb_transaction_reset(transaction); clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_end); - fprintf(printme, "INFO: Clear range: %6.3f sec\n", + fprintf(printme, + "INFO: Clear range: %6.3f sec\n", ((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) / 1000000000); return 0; @@ -154,7 +156,11 @@ failExit: } /* populate database */ -int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int thread_id, int thread_tps, +int populate(FDBTransaction* transaction, + mako_args_t* args, + int worker_id, + int thread_id, + int thread_tps, mako_stats_t* stats) { int i; struct timespec timer_start, timer_end; @@ -169,7 +175,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int int tracetimer = 0; keystr = (char*)malloc(sizeof(char) * args->key_length + 1); - if (!keystr) return -1; + if (!keystr) + return -1; valstr = (char*)malloc(sizeof(char) * args->value_length + 1); if (!valstr) { free(keystr); @@ -207,8 +214,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int fdb_error_t err; tracetimer = 0; fprintf(debugme, "DEBUG: txn tracing %s\n", keystr); - err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, - (uint8_t*)keystr, strlen(keystr)); + err = fdb_transaction_set_option( + transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, (uint8_t*)keystr, strlen(keystr)); if (err) { fprintf( stderr, @@ -217,7 +224,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int } err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0); if (err) { - fprintf(stderr, "ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n", + fprintf(stderr, + "ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n", fdb_get_error(err)); } } @@ -238,7 +246,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int /* commit every 100 inserts (default) */ if (i % args->txnspec.ops[OP_INSERT][OP_COUNT] == 0) { - if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit; + if (commit_transaction(transaction) != FDB_SUCCESS) + goto failExit; /* xact latency stats */ clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); @@ -252,7 +261,8 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int } } - if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit; + if (commit_transaction(transaction) != FDB_SUCCESS) + goto failExit; /* xact latency stats */ clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); @@ -261,7 +271,11 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int clock_gettime(CLOCK_MONOTONIC, &timer_end); stats->xacts++; - fprintf(debugme, "DEBUG: Populated %d rows (%d-%d): %6.3f sec\n", end - begin, begin, end, + fprintf(debugme, + "DEBUG: Populated %d rows (%d-%d): %6.3f sec\n", + end - begin, + begin, + end, ((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) / 1000000000); @@ -270,8 +284,10 @@ int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int return 0; failExit: - if (keystr) free(keystr); - if (valstr) free(valstr); + if (keystr) + free(keystr); + if (valstr) + free(valstr); fprintf(stderr, "ERROR: FDB failure in populate()\n"); return -1; } @@ -336,10 +352,15 @@ int run_op_getrange(FDBTransaction* transaction, char* keystr, char* keystr2, ch int out_count; int out_more; - f = fdb_transaction_get_range(transaction, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)), - FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1, 0 /* limit */, - 0 /* target_bytes */, FDB_STREAMING_MODE_WANT_ALL /* FDBStreamingMode */, - 0 /* iteration */, snapshot, reverse /* reverse */); + f = fdb_transaction_get_range(transaction, + FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)), + FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1, + 0 /* limit */, + 0 /* target_bytes */, + FDB_STREAMING_MODE_WANT_ALL /* FDBStreamingMode */, + 0 /* iteration */, + snapshot, + reverse /* reverse */); fdb_wait_and_handle_error(fdb_transaction_get_range, f, transaction); err = fdb_future_get_keyvalue_array(f, &out_kv, &out_count, &out_more); @@ -392,8 +413,12 @@ int run_op_clearrange(FDBTransaction* transaction, char* keystr, char* keystr2) } /* run one transaction */ -int run_one_transaction(FDBTransaction* transaction, mako_args_t* args, mako_stats_t* stats, char* keystr, - char* keystr2, char* valstr) { +int run_one_transaction(FDBTransaction* transaction, + mako_args_t* args, + mako_stats_t* stats, + char* keystr, + char* keystr2, + char* valstr) { int i; int count; int rc; @@ -475,10 +500,13 @@ retryTxn: randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */ randstr(valstr, args->value_length + 1); for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) { - sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]), + sprintf(keystr + KEYPREFIXLEN + randstrlen, + "%0.*d", + digits(args->txnspec.ops[i][OP_RANGE]), rangei); rc = run_op_insert(transaction, keystr, valstr); - if (rc != FDB_SUCCESS) break; + if (rc != FDB_SUCCESS) + break; } docommit = 1; break; @@ -525,7 +553,9 @@ retryTxn: randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */ randstr(valstr, args->value_length + 1); for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) { - sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]), + sprintf(keystr + KEYPREFIXLEN + randstrlen, + "%0.*d", + digits(args->txnspec.ops[i][OP_RANGE]), rangei); if (rangei == 0) { strcpy(keystr2, keystr); @@ -630,8 +660,14 @@ retryTxn: return 0; } -int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps, volatile double* throttle_factor, - int thread_iters, volatile int* signal, mako_stats_t* stats, int dotrace) { +int run_workload(FDBTransaction* transaction, + mako_args_t* args, + int thread_tps, + volatile double* throttle_factor, + int thread_iters, + volatile int* signal, + mako_stats_t* stats, + int dotrace) { int xacts = 0; int64_t total_xacts = 0; int rc = 0; @@ -643,7 +679,8 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps, char* traceid; int tracetimer = 0; - if (thread_tps < 0) return 0; + if (thread_tps < 0) + return 0; if (dotrace) { traceid = (char*)malloc(32); @@ -652,7 +689,8 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps, current_tps = (int)((double)thread_tps * *throttle_factor); keystr = (char*)malloc(sizeof(char) * args->key_length + 1); - if (!keystr) return -1; + if (!keystr) + return -1; keystr2 = (char*)malloc(sizeof(char) * args->key_length + 1); if (!keystr2) { free(keystr); @@ -693,11 +731,13 @@ int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps, tracetimer = 0; snprintf(traceid, 32, "makotrace%019lld", total_xacts); fprintf(debugme, "DEBUG: txn tracing %s\n", traceid); - err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, - (uint8_t*)traceid, strlen(traceid)); + err = fdb_transaction_set_option(transaction, + FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, + (uint8_t*)traceid, + strlen(traceid)); if (err) { - fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", - fdb_get_error(err)); + fprintf( + stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", fdb_get_error(err)); } err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0); if (err) { @@ -769,8 +809,13 @@ void* worker_thread(void* thread_args) { stats->latency_us_total[op] = 0; } - fprintf(debugme, "DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n", worker_id, args->num_processes, thread_id, - args->num_threads, (unsigned int)pthread_self()); + fprintf(debugme, + "DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n", + worker_id, + args->num_processes, + thread_id, + args->num_threads, + (unsigned int)pthread_self()); if (args->tpsmax) { thread_tps = compute_thread_tps(args->tpsmax, worker_id, thread_id, args->num_processes, args->num_threads); @@ -847,7 +892,6 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) { return -1; } - /* enable flatbuffers if specified */ if (args->flatbuffers) { #ifdef FDB_NET_OPTION_USE_FLATBUFFERS @@ -863,7 +907,9 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) { /* enable tracing if specified */ if (args->trace) { - fprintf(debugme, "DEBUG: Enable Tracing in %s (%s)\n", (args->traceformat == 0) ? "XML" : "JSON", + fprintf(debugme, + "DEBUG: Enable Tracing in %s (%s)\n", + (args->traceformat == 0) ? "XML" : "JSON", (args->tracepath[0] == '\0') ? "current directory" : args->tracepath); err = fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE, (uint8_t*)args->tracepath, strlen(args->tracepath)); if (err) { @@ -895,11 +941,10 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) { fprintf(debugme, "DEBUG: fdb_setup_network\n"); err = fdb_setup_network(); if (err) { - fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err)); - return -1; + fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(err)); + return -1; } - /* Each worker process will have its own network thread */ fprintf(debugme, "DEBUG: creating network thread\n"); rc = pthread_create(&network_thread, NULL, fdb_network_thread, (void*)args); @@ -968,8 +1013,10 @@ int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) { } failExit: - if (worker_threads) free(worker_threads); - if (thread_args) free(thread_args); + if (worker_threads) + free(worker_threads); + if (thread_args) + free(thread_args); /* clean up database and cluster */ fdb_database_destroy(process.database); @@ -995,7 +1042,8 @@ failExit: /* initialize the parameters with default values */ int init_args(mako_args_t* args) { int i; - if (!args) return -1; + if (!args) + return -1; memset(args, 0, sizeof(mako_args_t)); /* zero-out everything */ args->api_version = fdb_get_max_api_version(); args->json = 0; @@ -1220,7 +1268,8 @@ int parse_args(int argc, char* argv[], mako_args_t* args) { }; idx = 0; c = getopt_long(argc, argv, short_options, long_options, &idx); - if (c < 0) break; + if (c < 0) + break; switch (c) { case '?': case 'h': @@ -1249,7 +1298,8 @@ int parse_args(int argc, char* argv[], mako_args_t* args) { break; case 'x': rc = parse_transaction(args, optarg); - if (rc < 0) return -1; + if (rc < 0) + return -1; break; case 'v': args->verbose = atoi(optarg); @@ -1458,7 +1508,8 @@ void print_stats_header(mako_args_t* args) { int i; /* header */ - for (i = 0; i <= STATS_TITLE_WIDTH; i++) printf(" "); + for (i = 0; i <= STATS_TITLE_WIDTH; i++) + printf(" "); for (op = 0; op < MAX_OP; op++) { if (args->txnspec.ops[op][OP_COUNT] > 0) { switch (op) { @@ -1504,19 +1555,23 @@ void print_stats_header(mako_args_t* args) { printf("%" STR(STATS_FIELD_WIDTH) "s ", "TPS"); printf("%" STR(STATS_FIELD_WIDTH) "s\n", "Conflicts/s"); - for (i = 0; i < STATS_TITLE_WIDTH; i++) printf("="); + for (i = 0; i < STATS_TITLE_WIDTH; i++) + printf("="); printf(" "); for (op = 0; op < MAX_OP; op++) { if (args->txnspec.ops[op][OP_COUNT] > 0) { - for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); + for (i = 0; i < STATS_FIELD_WIDTH; i++) + printf("="); printf(" "); } } /* TPS */ - for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); + for (i = 0; i < STATS_FIELD_WIDTH; i++) + printf("="); printf(" "); /* Conflicts */ - for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); + for (i = 0; i < STATS_FIELD_WIDTH; i++) + printf("="); printf("\n"); } @@ -1666,7 +1721,8 @@ int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double* usleep(10000); /* 10ms */ } - if (args->verbose >= VERBOSE_DEFAULT) print_stats_header(args); + if (args->verbose >= VERBOSE_DEFAULT) + print_stats_header(args); clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); timer_prev.tv_sec = timer_start.tv_sec; @@ -1708,7 +1764,8 @@ int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double* } } - if (args->verbose >= VERBOSE_DEFAULT) print_stats(args, stats, &timer_now, &timer_prev); + if (args->verbose >= VERBOSE_DEFAULT) + print_stats(args, stats, &timer_now, &timer_prev); timer_prev.tv_sec = timer_now.tv_sec; timer_prev.tv_nsec = timer_now.tv_nsec; } @@ -1750,7 +1807,8 @@ int main(int argc, char* argv[]) { } rc = validate_args(&args); - if (rc < 0) return -1; + if (rc < 0) + return -1; if (args.mode == MODE_CLEAN) { /* cleanup will be done from a single thread */ @@ -1915,9 +1973,11 @@ int main(int argc, char* argv[]) { failExit: - if (worker_pids) free(worker_pids); + if (worker_pids) + free(worker_pids); - if (shm != MAP_FAILED) munmap(shm, shmsize); + if (shm != MAP_FAILED) + munmap(shm, shmsize); if (shmfd) { close(shmfd); diff --git a/bindings/c/test/performance_test.c b/bindings/c/test/performance_test.c index 7a265e7d0f..087920f92d 100644 --- a/bindings/c/test/performance_test.c +++ b/bindings/c/test/performance_test.c @@ -31,38 +31,40 @@ int numKeys = 1000000; int keySize = 16; uint8_t** keys = NULL; int valueSize = 100; -uint8_t *valueStr = NULL; +uint8_t* valueStr = NULL; -fdb_error_t waitError(FDBFuture *f) { +fdb_error_t waitError(FDBFuture* f) { fdb_error_t blockError = fdb_future_block_until_ready(f); - if(!blockError) { + if (!blockError) { return fdb_future_get_error(f); } else { return blockError; } } -struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) { - FDBTransaction *tr = NULL; +struct RunResult run(struct ResultSet* rs, + FDBDatabase* db, + struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) { + FDBTransaction* tr = NULL; fdb_error_t e = fdb_database_create_transaction(db, &tr); checkError(e, "create transaction", rs); - while(1) { + while (1) { struct RunResult r = func(rs, tr); e = r.e; - if(!e) { - FDBFuture *f = fdb_transaction_commit(tr); + if (!e) { + FDBFuture* f = fdb_transaction_commit(tr); e = waitError(f); fdb_future_destroy(f); } - if(e) { - FDBFuture *f = fdb_transaction_on_error(tr, e); + if (e) { + FDBFuture* f = fdb_transaction_on_error(tr, e); fdb_error_t retryE = waitError(f); fdb_future_destroy(f); if (retryE) { fdb_transaction_destroy(tr); - return (struct RunResult) {0, retryE}; + return (struct RunResult){ 0, retryE }; } } else { fdb_transaction_destroy(tr); @@ -73,19 +75,22 @@ struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*f return RES(0, 4100); // internal_error ; we should never get here } -int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) { +int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), + FDBDatabase* db, + struct ResultSet* rs, + const char* kpiName) { int numRuns = 25; - int *results = malloc(sizeof(int)*numRuns); + int* results = malloc(sizeof(int) * numRuns); int i = 0; - for(; i < numRuns; ++i) { + for (; i < numRuns; ++i) { struct RunResult res = run(rs, db, testFxn); - if(res.e) { + if (res.e) { logError(res.e, kpiName, rs); free(results); return 0; } results[i] = res.res; - if(results[i] < 0) { + if (results[i] < 0) { free(results); return -1; } @@ -99,19 +104,22 @@ int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDB return result; } -int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) { +int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), + FDBDatabase* db, + struct ResultSet* rs, + const char* kpiName) { int numRuns = 25; - int *results = malloc(sizeof(int)*numRuns); + int* results = malloc(sizeof(int) * numRuns); int i = 0; - for(; i < numRuns; ++i) { + for (; i < numRuns; ++i) { struct RunResult res = testFxn(rs, db); - if(res.e) { + if (res.e) { logError(res.e, kpiName, rs); free(results); return 0; } results[i] = res.res; - if(results[i] < 0) { + if (results[i] < 0) { free(results); return -1; } @@ -125,139 +133,144 @@ int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBD return result; } - -struct RunResult clearAll(struct ResultSet *rs, FDBTransaction *tr) { +struct RunResult clearAll(struct ResultSet* rs, FDBTransaction* tr) { fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1); return RES(0, 0); } uint32_t start = 0; uint32_t stop = 0; -struct RunResult insertRange(struct ResultSet *rs, FDBTransaction *tr) { +struct RunResult insertRange(struct ResultSet* rs, FDBTransaction* tr) { int i; - for(i = start; i < stop; i++) { + for (i = start; i < stop; i++) { fdb_transaction_set(tr, keys[i], keySize, valueStr, valueSize); } return RES(0, 0); } -void insertData(struct ResultSet *rs, FDBDatabase *db) { +void insertData(struct ResultSet* rs, FDBDatabase* db) { checkError(run(rs, db, &clearAll).e, "clearing database", rs); // TODO: Do this asynchronously. start = 0; - while(start < numKeys) { + while (start < numKeys) { stop = start + 1000; - if(stop > numKeys) stop = numKeys; + if (stop > numKeys) + stop = numKeys; checkError(run(rs, db, &insertRange).e, "inserting data range", rs); start = stop; } } -fdb_error_t setRetryLimit(struct ResultSet *rs, FDBTransaction *tr, uint64_t limit) { +fdb_error_t setRetryLimit(struct ResultSet* rs, FDBTransaction* tr, uint64_t limit) { return fdb_transaction_set_option(tr, FDB_TR_OPTION_RETRY_LIMIT, (const uint8_t*)&limit, sizeof(uint64_t)); } uint32_t FUTURE_LATENCY_COUNT = 100000; -const char *FUTURE_LATENCY_KPI = "C future throughput (local client)"; -struct RunResult futureLatency(struct ResultSet *rs, FDBTransaction *tr) { +const char* FUTURE_LATENCY_KPI = "C future throughput (local client)"; +struct RunResult futureLatency(struct ResultSet* rs, FDBTransaction* tr) { fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); - FDBFuture *f = fdb_transaction_get_read_version(tr); + FDBFuture* f = fdb_transaction_get_read_version(tr); e = waitError(f); fdb_future_destroy(f); maybeLogError(e, "getting initial read version", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); double start = getTime(); int i; - for(i = 0; i < FUTURE_LATENCY_COUNT; i++) { - FDBFuture *f = fdb_transaction_get_read_version(tr); + for (i = 0; i < FUTURE_LATENCY_COUNT; i++) { + FDBFuture* f = fdb_transaction_get_read_version(tr); e = waitError(f); fdb_future_destroy(f); maybeLogError(e, "getting read version", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); } double end = getTime(); - return RES(FUTURE_LATENCY_COUNT/(end - start), 0); + return RES(FUTURE_LATENCY_COUNT / (end - start), 0); } uint32_t CLEAR_COUNT = 100000; -const char *CLEAR_KPI = "C clear throughput (local client)"; -struct RunResult clear(struct ResultSet *rs, FDBTransaction *tr) { +const char* CLEAR_KPI = "C clear throughput (local client)"; +struct RunResult clear(struct ResultSet* rs, FDBTransaction* tr) { double start = getTime(); int i; - for(i = 0; i < CLEAR_COUNT; i++) { + for (i = 0; i < CLEAR_COUNT; i++) { int k = ((uint64_t)rand()) % numKeys; fdb_transaction_clear(tr, keys[k], keySize); } double end = getTime(); fdb_transaction_reset(tr); // Don't actually clear things. - return RES(CLEAR_COUNT/(end - start), 0); + return RES(CLEAR_COUNT / (end - start), 0); } uint32_t CLEAR_RANGE_COUNT = 100000; -const char *CLEAR_RANGE_KPI = "C clear range throughput (local client)"; -struct RunResult clearRange(struct ResultSet *rs, FDBTransaction *tr) { +const char* CLEAR_RANGE_KPI = "C clear range throughput (local client)"; +struct RunResult clearRange(struct ResultSet* rs, FDBTransaction* tr) { double start = getTime(); int i; - for(i = 0; i < CLEAR_RANGE_COUNT; i++) { + for (i = 0; i < CLEAR_RANGE_COUNT; i++) { int k = ((uint64_t)rand()) % (numKeys - 1); - fdb_transaction_clear_range(tr, keys[k], keySize, keys[k+1], keySize); + fdb_transaction_clear_range(tr, keys[k], keySize, keys[k + 1], keySize); } double end = getTime(); fdb_transaction_reset(tr); // Don't actually clear things. - return RES(CLEAR_RANGE_COUNT/(end - start), 0); + return RES(CLEAR_RANGE_COUNT / (end - start), 0); } uint32_t SET_COUNT = 100000; -const char *SET_KPI = "C set throughput (local client)"; -struct RunResult set(struct ResultSet *rs, FDBTransaction *tr) { +const char* SET_KPI = "C set throughput (local client)"; +struct RunResult set(struct ResultSet* rs, FDBTransaction* tr) { double start = getTime(); int i; - for(i = 0; i < SET_COUNT; i++) { + for (i = 0; i < SET_COUNT; i++) { int k = ((uint64_t)rand()) % numKeys; fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize); } double end = getTime(); fdb_transaction_reset(tr); // Don't actually set things. - return RES(SET_COUNT/(end - start), 0); + return RES(SET_COUNT / (end - start), 0); } uint32_t PARALLEL_GET_COUNT = 10000; -const char *PARALLEL_GET_KPI = "C parallel get throughput (local client)"; -struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) { +const char* PARALLEL_GET_KPI = "C parallel get throughput (local client)"; +struct RunResult parallelGet(struct ResultSet* rs, FDBTransaction* tr) { fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); - FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT); + FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT); double start = getTime(); int i; - for(i = 0; i < PARALLEL_GET_COUNT; i++) { + for (i = 0; i < PARALLEL_GET_COUNT; i++) { int k = ((uint64_t)rand()) % numKeys; futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0); } fdb_bool_t present; - uint8_t const *outValue; + uint8_t const* outValue; int outValueLength; - for(i = 0; i < PARALLEL_GET_COUNT; i++) { + for (i = 0; i < PARALLEL_GET_COUNT; i++) { e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs); - if(e) { + if (e) { fdb_future_destroy(futures[i]); return RES(0, e); } - e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs); - if(e) { + e = maybeLogError( + fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs); + if (e) { fdb_future_destroy(futures[i]); return RES(0, e); } @@ -268,39 +281,41 @@ struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) { double end = getTime(); free(futures); - return RES(PARALLEL_GET_COUNT/(end - start), 0); + return RES(PARALLEL_GET_COUNT / (end - start), 0); } uint32_t ALTERNATING_GET_SET_COUNT = 2000; -const char *ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)"; -struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) { +const char* ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)"; +struct RunResult alternatingGetSet(struct ResultSet* rs, FDBTransaction* tr) { fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); - FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT); + FDBFuture** futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT); double start = getTime(); int i; - for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) { + for (i = 0; i < ALTERNATING_GET_SET_COUNT; i++) { int k = ((uint64_t)rand()) % numKeys; fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize); futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0); } fdb_bool_t present; - uint8_t const *outValue; + uint8_t const* outValue; int outValueLength; - for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) { + for (i = 0; i < ALTERNATING_GET_SET_COUNT; i++) { e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs); - if(e) { + if (e) { fdb_future_destroy(futures[i]); return RES(0, e); } - e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs); - if(e) { + e = maybeLogError( + fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs); + if (e) { fdb_future_destroy(futures[i]); return RES(0, e); } @@ -311,38 +326,39 @@ struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) { double end = getTime(); free(futures); - return RES(ALTERNATING_GET_SET_COUNT/(end - start), 0); + return RES(ALTERNATING_GET_SET_COUNT / (end - start), 0); } uint32_t SERIAL_GET_COUNT = 2000; -const char *SERIAL_GET_KPI = "C serial get throughput (local client)"; -struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) { +const char* SERIAL_GET_KPI = "C serial get throughput (local client)"; +struct RunResult serialGet(struct ResultSet* rs, FDBTransaction* tr) { fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); int i; - uint32_t *keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT); + uint32_t* keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT); - if(SERIAL_GET_COUNT > numKeys/2) { - for(i = 0; i < SERIAL_GET_COUNT; i++) { + if (SERIAL_GET_COUNT > numKeys / 2) { + for (i = 0; i < SERIAL_GET_COUNT; i++) { keyIndices[i] = ((uint64_t)rand()) % numKeys; } } else { - for(i = 0; i < SERIAL_GET_COUNT; i++) { - while(1) { + for (i = 0; i < SERIAL_GET_COUNT; i++) { + while (1) { // Yes, this is a linear scan. This happens outside // the part we are measuring. uint32_t index = ((uint64_t)rand()) % numKeys; int j; fdb_bool_t found = 0; - for(j = 0; j < i; j++) { - if(keyIndices[j] == index) { + for (j = 0; j < i; j++) { + if (keyIndices[j] == index) { found = 1; break; } } - if(!found) { + if (!found) { keyIndices[i] = index; break; } @@ -353,13 +369,13 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) { double start = getTime(); fdb_bool_t present; - uint8_t const *outValue; + uint8_t const* outValue; int outValueLength; - for(i = 0; i < SERIAL_GET_COUNT; i++) { - FDBFuture *f = fdb_transaction_get(tr, keys[keyIndices[i]], keySize, 0); + for (i = 0; i < SERIAL_GET_COUNT; i++) { + FDBFuture* f = fdb_transaction_get(tr, keys[keyIndices[i]], keySize, 0); fdb_error_t e = maybeLogError(fdb_future_block_until_ready(f), "getting key in serial", rs); - if(e) { + if (e) { free(keyIndices); fdb_future_destroy(f); return RES(0, e); @@ -367,7 +383,7 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) { e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs); fdb_future_destroy(f); - if(e) { + if (e) { free(keyIndices); return RES(0, e); } @@ -376,66 +392,87 @@ struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) { double end = getTime(); free(keyIndices); - return RES(SERIAL_GET_COUNT/(end - start), 0); + return RES(SERIAL_GET_COUNT / (end - start), 0); } uint32_t GET_RANGE_COUNT = 100000; -const char *GET_RANGE_KPI = "C get range throughput (local client)"; -struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) { +const char* GET_RANGE_KPI = "C get range throughput (local client)"; +struct RunResult getRange(struct ResultSet* rs, FDBTransaction* tr) { fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); uint32_t startKey = ((uint64_t)rand()) % (numKeys - GET_RANGE_COUNT - 1); double start = getTime(); - const FDBKeyValue *outKv; + const FDBKeyValue* outKv; int outCount; fdb_bool_t outMore = 1; int totalOut = 0; int iteration = 0; - FDBFuture *f = fdb_transaction_get_range(tr, - keys[startKey], keySize, 1, 0, - keys[startKey + GET_RANGE_COUNT], keySize, 1, 0, - 0, 0, - FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0); + FDBFuture* f = fdb_transaction_get_range(tr, + keys[startKey], + keySize, + 1, + 0, + keys[startKey + GET_RANGE_COUNT], + keySize, + 1, + 0, + 0, + 0, + FDB_STREAMING_MODE_WANT_ALL, + ++iteration, + 0, + 0); - while(outMore) { + while (outMore) { e = maybeLogError(fdb_future_block_until_ready(f), "getting range", rs); - if(e) { + if (e) { fdb_future_destroy(f); return RES(0, e); } e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading range array", rs); - if(e) { + if (e) { fdb_future_destroy(f); return RES(0, e); } totalOut += outCount; - if(outMore) { - FDBFuture *f2 = fdb_transaction_get_range(tr, - outKv[outCount - 1].key, outKv[outCount - 1].key_length, 1, 1, - keys[startKey + GET_RANGE_COUNT], keySize, 1, 0, - 0, 0, - FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0); + if (outMore) { + FDBFuture* f2 = fdb_transaction_get_range(tr, + outKv[outCount - 1].key, + outKv[outCount - 1].key_length, + 1, + 1, + keys[startKey + GET_RANGE_COUNT], + keySize, + 1, + 0, + 0, + 0, + FDB_STREAMING_MODE_WANT_ALL, + ++iteration, + 0, + 0); fdb_future_destroy(f); f = f2; } } - if(totalOut != GET_RANGE_COUNT) { - char *msg = (char*)malloc((sizeof(char)) * 200); + if (totalOut != GET_RANGE_COUNT) { + char* msg = (char*)malloc((sizeof(char)) * 200); sprintf(msg, "verifying out count (%d != %d)", totalOut, GET_RANGE_COUNT); logError(4100, msg, rs); free(msg); fdb_future_destroy(f); return RES(0, 4100); } - if(outMore) { + if (outMore) { logError(4100, "verifying no more in range", rs); fdb_future_destroy(f); return RES(0, 4100); @@ -444,84 +481,84 @@ struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) { double end = getTime(); - return RES(GET_RANGE_COUNT/(end - start), 0); + return RES(GET_RANGE_COUNT / (end - start), 0); } uint32_t GET_KEY_COUNT = 2000; -const char *GET_KEY_KPI = "C get key throughput (local client)"; -struct RunResult getKey(struct ResultSet *rs, FDBTransaction *tr) { +const char* GET_KEY_KPI = "C get key throughput (local client)"; +struct RunResult getKey(struct ResultSet* rs, FDBTransaction* tr) { fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); double start = getTime(); fdb_bool_t present; - uint8_t const *outValue; + uint8_t const* outValue; int outValueLength; int i; - for(i = 0; i < GET_KEY_COUNT; i++) { + for (i = 0; i < GET_KEY_COUNT; i++) { int key = ((uint64_t)rand()) % numKeys; int offset = (((uint64_t)rand()) % 21) - 10; - FDBFuture *f = fdb_transaction_get_key(tr, keys[key], keySize, 1, offset, 0); + FDBFuture* f = fdb_transaction_get_key(tr, keys[key], keySize, 1, offset, 0); e = maybeLogError(fdb_future_block_until_ready(f), "waiting for get key", rs); - if(e) { + if (e) { fdb_future_destroy(f); return RES(0, e); } e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs); fdb_future_destroy(f); - if(e) { + if (e) { return RES(0, e); } } double end = getTime(); - return RES(GET_KEY_COUNT/(end - start), 0); + return RES(GET_KEY_COUNT / (end - start), 0); } uint32_t GET_SINGLE_KEY_RANGE_COUNT = 2000; -const char *GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)"; -struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) { +const char* GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)"; +struct RunResult getSingleKeyRange(struct ResultSet* rs, FDBTransaction* tr) { fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs); - if(e) return RES(0, e); + if (e) + return RES(0, e); double start = getTime(); - const FDBKeyValue *outKv; + const FDBKeyValue* outKv; int outCount; fdb_bool_t outMore; int i; - for(i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) { + for (i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) { int key = ((uint64_t)rand()) % (numKeys - 1); - FDBFuture *f = fdb_transaction_get_range(tr, - keys[key], keySize, 1, 0, - keys[key + 1], keySize, 1, 0, - 2, 0, - FDB_STREAMING_MODE_EXACT, 1, 0, 0); + FDBFuture* f = fdb_transaction_get_range( + tr, keys[key], keySize, 1, 0, keys[key + 1], keySize, 1, 0, 2, 0, FDB_STREAMING_MODE_EXACT, 1, 0, 0); e = maybeLogError(fdb_future_block_until_ready(f), "waiting for single key range", rs); - if(e) { + if (e) { fdb_future_destroy(f); return RES(0, e); } - e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs); - if(e) { + e = maybeLogError( + fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs); + if (e) { fdb_future_destroy(f); return RES(0, e); } - if(outCount != 1) { + if (outCount != 1) { logError(4100, "more than one key returned in single key range read", rs); fdb_future_destroy(f); return RES(0, 4100); } - if(outMore) { + if (outMore) { logError(4100, "more keys to read in single key range read", rs); fdb_future_destroy(f); return RES(0, 4100); @@ -532,33 +569,34 @@ struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) { double end = getTime(); - return RES(GET_SINGLE_KEY_RANGE_COUNT/(end - start), 0); + return RES(GET_SINGLE_KEY_RANGE_COUNT / (end - start), 0); } -struct RunResult singleKey(struct ResultSet *rs, FDBTransaction *tr) { +struct RunResult singleKey(struct ResultSet* rs, FDBTransaction* tr) { int k = ((uint64_t)rand()) % numKeys; fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize); return RES(0, 0); } uint32_t WRITE_TRANSACTION_COUNT = 1000; -const char *WRITE_TRANSACTION_KPI = "C write_transaction throughput (local client)"; -struct RunResult writeTransaction(struct ResultSet *rs, FDBDatabase *db) { +const char* WRITE_TRANSACTION_KPI = "C write_transaction throughput (local client)"; +struct RunResult writeTransaction(struct ResultSet* rs, FDBDatabase* db) { double start = getTime(); int i; - for(i = 0; i < WRITE_TRANSACTION_COUNT; i++) { + for (i = 0; i < WRITE_TRANSACTION_COUNT; i++) { struct RunResult res = run(rs, db, &singleKey); - if(res.e) return res; + if (res.e) + return res; } double end = getTime(); - return RES(WRITE_TRANSACTION_COUNT/(end - start), 0); + return RES(WRITE_TRANSACTION_COUNT / (end - start), 0); } -void runTests(struct ResultSet *rs) { - FDBDatabase *db = openDatabase(rs, &netThread); +void runTests(struct ResultSet* rs) { + FDBDatabase* db = openDatabase(rs, &netThread); printf("Loading database...\n"); insertData(rs, db); @@ -600,15 +638,15 @@ void runTests(struct ResultSet *rs) { fdb_stop_network(); } -int main(int argc, char **argv) { +int main(int argc, char** argv) { srand(time(NULL)); - struct ResultSet *rs = newResultSet(); + struct ResultSet* rs = newResultSet(); checkError(fdb_select_api_version(630), "select API version", rs); printf("Running performance test at client version: %s\n", fdb_get_client_version()); - valueStr = (uint8_t*)malloc((sizeof(uint8_t))*valueSize); + valueStr = (uint8_t*)malloc((sizeof(uint8_t)) * valueSize); int i; - for(i = 0; i < valueSize; i++) { + for (i = 0; i < valueSize; i++) { valueStr[i] = (uint8_t)'x'; } diff --git a/bindings/c/test/ryw_benchmark.c b/bindings/c/test/ryw_benchmark.c index cbb7fcf304..4b858ba39a 100644 --- a/bindings/c/test/ryw_benchmark.c +++ b/bindings/c/test/ryw_benchmark.c @@ -34,23 +34,26 @@ int numKeys = 10000; int keySize = 16; uint8_t** keys; -void insertData(FDBTransaction *tr) { +void insertData(FDBTransaction* tr) { fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1); - uint8_t *v = (uint8_t*)"foo"; + uint8_t* v = (uint8_t*)"foo"; uint32_t i; - for(i = 0; i <= numKeys; ++i) { + for (i = 0; i <= numKeys; ++i) { fdb_transaction_set(tr, keys[i], keySize, v, 3); } } -int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction *tr, struct ResultSet *rs, const char *kpiName) { +int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), + FDBTransaction* tr, + struct ResultSet* rs, + const char* kpiName) { int numRuns = 25; - int *results = malloc(sizeof(int)*numRuns); + int* results = malloc(sizeof(int) * numRuns); int i = 0; - for(; i < numRuns; ++i) { + for (; i < numRuns; ++i) { results[i] = testFxn(tr, rs); - if(results[i] < 0) { + if (results[i] < 0) { free(results); return -1; } @@ -64,17 +67,19 @@ int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction * return result; } -int getSingle(FDBTransaction *tr, struct ResultSet *rs) { +int getSingle(FDBTransaction* tr, struct ResultSet* rs) { int present; - uint8_t const *value; + uint8_t const* value; int length; int i; double start = getTime(); - for(i = 0; i < numKeys; ++i) { - FDBFuture *f = fdb_transaction_get(tr, keys[5001], keySize, 0); - if(getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs)) return -1; - if(getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs)) return -1; + for (i = 0; i < numKeys; ++i) { + FDBFuture* f = fdb_transaction_get(tr, keys[5001], keySize, 0); + if (getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs)) + return -1; + if (getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs)) + return -1; fdb_future_destroy(f); } double end = getTime(); @@ -82,17 +87,19 @@ int getSingle(FDBTransaction *tr, struct ResultSet *rs) { return numKeys / (end - start); } -int getManySequential(FDBTransaction *tr, struct ResultSet *rs) { +int getManySequential(FDBTransaction* tr, struct ResultSet* rs) { int present; - uint8_t const *value; + uint8_t const* value; int length; int i; double start = getTime(); - for(i = 0; i < numKeys; ++i) { - FDBFuture *f = fdb_transaction_get(tr, keys[i], keySize, 0); - if(getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs)) return -1; - if(getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs)) return -1; + for (i = 0; i < numKeys; ++i) { + FDBFuture* f = fdb_transaction_get(tr, keys[i], keySize, 0); + if (getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs)) + return -1; + if (getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs)) + return -1; fdb_future_destroy(f); } double end = getTime(); @@ -100,20 +107,30 @@ int getManySequential(FDBTransaction *tr, struct ResultSet *rs) { return numKeys / (end - start); } -int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) { +int getRangeBasic(FDBTransaction* tr, struct ResultSet* rs) { int count; - const FDBKeyValue *kvs; + const FDBKeyValue* kvs; int more; int i; double start = getTime(); - for(i = 0; i < 100; ++i) { - FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0); + for (i = 0; i < 100; ++i) { + FDBFuture* f = fdb_transaction_get_range(tr, + FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), + FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), + numKeys, + 0, + 0, + 1, + 0, + 0); - if(getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs)) return -1; - if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs)) return -1; + if (getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs)) + return -1; + if (getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs)) + return -1; - if(count != numKeys) { + if (count != numKeys) { fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys); addError(rs, "GetRangeBasic bad count"); return -1; @@ -124,26 +141,37 @@ int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) { return 100 * numKeys / (end - start); } -int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) { +int singleClearGetRange(FDBTransaction* tr, struct ResultSet* rs) { int count; - const FDBKeyValue *kvs; + const FDBKeyValue* kvs; int more; int i; - for(i = 0; i < numKeys; i+=2) { + for (i = 0; i < numKeys; i += 2) { fdb_transaction_clear(tr, keys[i], keySize); } double start = getTime(); - for(i = 0; i < 100; ++i) { - FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0); + for (i = 0; i < 100; ++i) { + FDBFuture* f = fdb_transaction_get_range(tr, + FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), + FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), + numKeys, + 0, + 0, + 1, + 0, + 0); - if(getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs)) return -1; - if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs)) return -1; + if (getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs)) + return -1; + if (getError( + fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs)) + return -1; fdb_future_destroy(f); - if(count != numKeys/2) { + if (count != numKeys / 2) { fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys); addError(rs, "SingleClearGetRange bad count"); return -1; @@ -155,27 +183,38 @@ int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) { return 100 * numKeys / 2 / (end - start); } -int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) { +int clearRangeGetRange(FDBTransaction* tr, struct ResultSet* rs) { int count; - const FDBKeyValue *kvs; + const FDBKeyValue* kvs; int more; int i; - for(i = 0; i < numKeys; i+=4) { - fdb_transaction_clear_range(tr, keys[i], keySize, keys[i+1], keySize); + for (i = 0; i < numKeys; i += 4) { + fdb_transaction_clear_range(tr, keys[i], keySize, keys[i + 1], keySize); } double start = getTime(); - for(i = 0; i < 100; ++i) { - FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0); + for (i = 0; i < 100; ++i) { + FDBFuture* f = fdb_transaction_get_range(tr, + FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), + FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), + numKeys, + 0, + 0, + 1, + 0, + 0); - if(getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs)) return -1; - if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs)) return -1; + if (getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs)) + return -1; + if (getError( + fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs)) + return -1; fdb_future_destroy(f); - if(count != numKeys*3/4) { - fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys*3/4); + if (count != numKeys * 3 / 4) { + fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys * 3 / 4); addError(rs, "ClearRangeGetRange bad count"); return -1; } @@ -186,13 +225,13 @@ int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) { return 100 * numKeys * 3 / 4 / (end - start); } -int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) { +int interleavedSetsGets(FDBTransaction* tr, struct ResultSet* rs) { int present; - uint8_t const *value; + uint8_t const* value; int length; int i; - uint8_t *k = (uint8_t*)"foo"; + uint8_t* k = (uint8_t*)"foo"; uint8_t v[10]; int num = 1; @@ -200,10 +239,12 @@ int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) { sprintf((char*)v, "%d", num); fdb_transaction_set(tr, k, 3, v, strlen((char*)v)); - for(i = 0; i < 10000; ++i) { - FDBFuture *f = fdb_transaction_get(tr, k, 3, 0); - if(getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs)) return -1; - if(getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs)) return -1; + for (i = 0; i < 10000; ++i) { + FDBFuture* f = fdb_transaction_get(tr, k, 3, 0); + if (getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs)) + return -1; + if (getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs)) + return -1; fdb_future_destroy(f); sprintf((char*)v, "%d", ++num); @@ -214,13 +255,13 @@ int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) { return 10000 / (end - start); } -void runTests(struct ResultSet *rs) { - FDBDatabase *db = openDatabase(rs, &netThread); +void runTests(struct ResultSet* rs) { + FDBDatabase* db = openDatabase(rs, &netThread); - FDBTransaction *tr; + FDBTransaction* tr; checkError(fdb_database_create_transaction(db, &tr), "create transaction", rs); - FDBFuture *f = fdb_transaction_get_read_version(tr); + FDBFuture* f = fdb_transaction_get_read_version(tr); checkError(fdb_future_block_until_ready(f), "block for read version", rs); int64_t version; @@ -241,9 +282,9 @@ void runTests(struct ResultSet *rs) { fdb_stop_network(); } -int main(int argc, char **argv) { +int main(int argc, char** argv) { srand(time(NULL)); - struct ResultSet *rs = newResultSet(); + struct ResultSet* rs = newResultSet(); checkError(fdb_select_api_version(630), "select API version", rs); printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version()); @@ -255,4 +296,3 @@ int main(int argc, char **argv) { return 0; } - diff --git a/bindings/c/test/test.h b/bindings/c/test/test.h index 5fb4268b78..e74283e506 100644 --- a/bindings/c/test/test.h +++ b/bindings/c/test/test.h @@ -38,27 +38,27 @@ double getTime() { static struct timeval tv; gettimeofday(&tv, NULL); - return tv.tv_usec/1000000.0 + tv.tv_sec; + return tv.tv_usec / 1000000.0 + tv.tv_sec; } -void writeKey(uint8_t **dest, int key, int keySize) { - *dest = (uint8_t*)malloc((sizeof(uint8_t))*keySize); +void writeKey(uint8_t** dest, int key, int keySize) { + *dest = (uint8_t*)malloc((sizeof(uint8_t)) * keySize); sprintf((char*)*dest, "%0*d", keySize, key); } -uint8_t **generateKeys(int numKeys, int keySize) { - uint8_t **keys = (uint8_t**)malloc(sizeof(uint8_t*)*(numKeys+1)); +uint8_t** generateKeys(int numKeys, int keySize) { + uint8_t** keys = (uint8_t**)malloc(sizeof(uint8_t*) * (numKeys + 1)); uint32_t i; - for(i = 0; i <= numKeys; ++i) { + for (i = 0; i <= numKeys; ++i) { writeKey(keys + i, i, keySize); } return keys; } -void freeKeys(uint8_t **keys, int numKeys) { +void freeKeys(uint8_t** keys, int numKeys) { uint32_t i; - for(i = 0; i < numKeys; i++) { + for (i = 0; i < numKeys; i++) { free(keys[i]); } free(keys); @@ -68,38 +68,39 @@ int cmpfunc(const void* a, const void* b) { return (*(int*)a - *(int*)b); } -int median(int *values, int length) { +int median(int* values, int length) { qsort(values, length, sizeof(int), cmpfunc); - return values[length/2]; + return values[length / 2]; } struct RunResult { int res; fdb_error_t e; }; -#define RES(x, y) (struct RunResult) { x, y } +#define RES(x, y) \ + (struct RunResult) { x, y } struct Kpi { - const char *name; + const char* name; int value; - const char *units; + const char* units; - struct Kpi *next; + struct Kpi* next; }; struct Error { - char *message; + char* message; - struct Error *next; + struct Error* next; }; struct ResultSet { - struct Kpi *kpis; - struct Error *errors; + struct Kpi* kpis; + struct Error* errors; }; struct ResultSet* newResultSet() { - struct ResultSet *rs = malloc(sizeof(struct ResultSet)); + struct ResultSet* rs = malloc(sizeof(struct ResultSet)); rs->kpis = NULL; rs->errors = NULL; @@ -107,8 +108,8 @@ struct ResultSet* newResultSet() { return rs; } -void addKpi(struct ResultSet *rs, const char *name, int value, const char *units) { - struct Kpi *k = malloc(sizeof(struct Kpi)); +void addKpi(struct ResultSet* rs, const char* name, int value, const char* units) { + struct Kpi* k = malloc(sizeof(struct Kpi)); k->name = name; k->value = value; k->units = units; @@ -116,20 +117,20 @@ void addKpi(struct ResultSet *rs, const char *name, int value, const char *units rs->kpis = k; } -void addError(struct ResultSet *rs, const char *message) { - struct Error *e = malloc(sizeof(struct Error)); - e->message = (char*)malloc(strlen(message)+1); +void addError(struct ResultSet* rs, const char* message) { + struct Error* e = malloc(sizeof(struct Error)); + e->message = (char*)malloc(strlen(message) + 1); strcpy(e->message, message); e->next = rs->errors; rs->errors = e; } -void writeResultSet(struct ResultSet *rs) { +void writeResultSet(struct ResultSet* rs) { uint64_t id = ((uint64_t)rand() << 32) + rand(); char name[100]; sprintf(name, "fdb-c_result-%" SCNu64 ".json", id); - FILE *fp = fopen(name, "w"); - if(!fp) { + FILE* fp = fopen(name, "w"); + if (!fp) { fprintf(stderr, "Could not open results file %s\n", name); exit(1); } @@ -137,10 +138,10 @@ void writeResultSet(struct ResultSet *rs) { fprintf(fp, "{\n"); fprintf(fp, "\t\"kpis\": {\n"); - struct Kpi *k = rs->kpis; - while(k != NULL) { + struct Kpi* k = rs->kpis; + while (k != NULL) { fprintf(fp, "\t\t\"%s\": { \"units\": \"%s\", \"value\": %d }", k->name, k->units, k->value); - if(k->next != NULL) { + if (k->next != NULL) { fprintf(fp, ","); } fprintf(fp, "\n"); @@ -150,10 +151,10 @@ void writeResultSet(struct ResultSet *rs) { fprintf(fp, "\t},\n"); fprintf(fp, "\t\"errors\": [\n"); - struct Error *e = rs->errors; - while(e != NULL) { + struct Error* e = rs->errors; + while (e != NULL) { fprintf(fp, "\t\t\"%s\"", e->message); - if(e->next != NULL) { + if (e->next != NULL) { fprintf(fp, ","); } fprintf(fp, "\n"); @@ -166,17 +167,17 @@ void writeResultSet(struct ResultSet *rs) { fclose(fp); } -void freeResultSet(struct ResultSet *rs) { - struct Kpi *k = rs->kpis; - while(k != NULL) { - struct Kpi *next = k->next; +void freeResultSet(struct ResultSet* rs) { + struct Kpi* k = rs->kpis; + while (k != NULL) { + struct Kpi* next = k->next; free(k); k = next; } - struct Error *e = rs->errors; - while(e != NULL) { - struct Error *next = e->next; + struct Error* e = rs->errors; + while (e != NULL) { + struct Error* next = e->next; free(e->message); free(e); e = next; @@ -185,12 +186,12 @@ void freeResultSet(struct ResultSet *rs) { free(rs); } -fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet *rs) { - if(err) { - char *msg = (char*)malloc(strlen(context) + 100); +fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet* rs) { + if (err) { + char* msg = (char*)malloc(strlen(context) + 100); sprintf(msg, "Error in %s: %s", context, fdb_get_error(err)); fprintf(stderr, "%s\n", msg); - if(rs != NULL) { + if (rs != NULL) { addError(rs, msg); } @@ -200,9 +201,9 @@ fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet *rs) return err; } -void checkError(fdb_error_t err, const char* context, struct ResultSet *rs) { - if(getError(err, context, rs)) { - if(rs != NULL) { +void checkError(fdb_error_t err, const char* context, struct ResultSet* rs) { + if (getError(err, context, rs)) { + if (rs != NULL) { writeResultSet(rs); freeResultSet(rs); } @@ -210,11 +211,11 @@ void checkError(fdb_error_t err, const char* context, struct ResultSet *rs) { } } -fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet *rs) { - char *msg = (char*)malloc(strlen(context) + 100); +fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet* rs) { + char* msg = (char*)malloc(strlen(context) + 100); sprintf(msg, "Error in %s: %s", context, fdb_get_error(err)); fprintf(stderr, "%s\n", msg); - if(rs != NULL) { + if (rs != NULL) { addError(rs, msg); } @@ -222,8 +223,8 @@ fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet *rs) return err; } -fdb_error_t maybeLogError(fdb_error_t err, const char* context, struct ResultSet *rs) { - if(err && !fdb_error_predicate( FDB_ERROR_PREDICATE_RETRYABLE, err ) ) { +fdb_error_t maybeLogError(fdb_error_t err, const char* context, struct ResultSet* rs) { + if (err && !fdb_error_predicate(FDB_ERROR_PREDICATE_RETRYABLE, err)) { return logError(err, context, rs); } return err; @@ -234,11 +235,11 @@ void* runNetwork() { return NULL; } -FDBDatabase* openDatabase(struct ResultSet *rs, pthread_t *netThread) { +FDBDatabase* openDatabase(struct ResultSet* rs, pthread_t* netThread) { checkError(fdb_setup_network(), "setup network", rs); pthread_create(netThread, NULL, (void*)(&runNetwork), NULL); - FDBDatabase *db; + FDBDatabase* db; checkError(fdb_create_database(NULL, &db), "create database", rs); return db; diff --git a/bindings/c/test/txn_size_test.c b/bindings/c/test/txn_size_test.c index 4f2744d199..3ba9158e55 100644 --- a/bindings/c/test/txn_size_test.c +++ b/bindings/c/test/txn_size_test.c @@ -31,14 +31,14 @@ pthread_t netThread; const int numKeys = 100; uint8_t** keys = NULL; -#define KEY_SIZE 16 +#define KEY_SIZE 16 #define VALUE_SIZE 100 uint8_t valueStr[VALUE_SIZE]; fdb_error_t getSize(struct ResultSet* rs, FDBTransaction* tr, int64_t* out_size) { fdb_error_t e; FDBFuture* future = fdb_transaction_get_approximate_size(tr); - + e = maybeLogError(fdb_future_block_until_ready(future), "waiting for get future", rs); if (e) { fdb_future_destroy(future); @@ -55,11 +55,11 @@ fdb_error_t getSize(struct ResultSet* rs, FDBTransaction* tr, int64_t* out_size) return 0; } -void runTests(struct ResultSet *rs) { +void runTests(struct ResultSet* rs) { int64_t sizes[numKeys]; int i = 0, j = 0; - FDBDatabase *db = openDatabase(rs, &netThread); - FDBTransaction *tr = NULL; + FDBDatabase* db = openDatabase(rs, &netThread); + FDBTransaction* tr = NULL; fdb_error_t e = fdb_database_create_transaction(db, &tr); checkError(e, "create transaction", rs); memset(sizes, 0, numKeys * sizeof(uint32_t)); @@ -82,7 +82,7 @@ void runTests(struct ResultSet *rs) { printf("size %d: %u\n", i, sizes[i]); i++; - fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i+1], KEY_SIZE); + fdb_transaction_clear_range(tr, keys[i], KEY_SIZE, keys[i + 1], KEY_SIZE); e = getSize(rs, tr, sizes + i); checkError(e, "transaction get size", rs); printf("size %d: %u\n", i, sizes[i]); @@ -94,9 +94,9 @@ void runTests(struct ResultSet *rs) { printf("Test passed!\n"); } -int main(int argc, char **argv) { +int main(int argc, char** argv) { srand(time(NULL)); - struct ResultSet *rs = newResultSet(); + struct ResultSet* rs = newResultSet(); checkError(fdb_select_api_version(630), "select API version", rs); printf("Running performance test at client version: %s\n", fdb_get_client_version()); diff --git a/bindings/c/test/workloads/SimpleWorkload.cpp b/bindings/c/test/workloads/SimpleWorkload.cpp index 35b18f71a3..62ee3fe044 100644 --- a/bindings/c/test/workloads/SimpleWorkload.cpp +++ b/bindings/c/test/workloads/SimpleWorkload.cpp @@ -104,7 +104,10 @@ struct SimpleWorkload : FDBWorkload { unsigned long from, to, lastTx = 0; std::unordered_map callbacks; - PopulateActor(const Callback& promise, SimpleWorkload& self, FDBDatabase* db, unsigned long from, + PopulateActor(const Callback& promise, + SimpleWorkload& self, + FDBDatabase* db, + unsigned long from, unsigned long to) : ActorBase(promise, self, db), from(from), to(to) { error = fdb_database_create_transaction(db, &tx); @@ -130,8 +133,11 @@ struct SimpleWorkload : FDBWorkload { for (; from < to && ops < self.insertsPerTx; ++ops, ++from) { std::string value = std::to_string(from); std::string key = KEY_PREFIX + value; - fdb_transaction_set(tx, reinterpret_cast(key.c_str()), key.size(), - reinterpret_cast(value.c_str()), value.size()); + fdb_transaction_set(tx, + reinterpret_cast(key.c_str()), + key.size(), + reinterpret_cast(value.c_str()), + value.size()); } lastTx = ops; auto commit_future = fdb_transaction_commit(tx); @@ -154,7 +160,8 @@ struct SimpleWorkload : FDBWorkload { run(); }, [this](fdb_error_t error) { - self.context->trace(FDBSeverity::Error, "AssertionFailure", + self.context->trace(FDBSeverity::Error, + "AssertionFailure", { { "Reason", "tx.onError failed" }, { "Error", std::string(fdb_get_error(error)) } }); self.success = false; @@ -230,7 +237,8 @@ struct SimpleWorkload : FDBWorkload { get(); }, [this](fdb_error_t) { - self.context->trace(FDBSeverity::Error, "AssertionFailure", + self.context->trace(FDBSeverity::Error, + "AssertionFailure", { { "Reason", "tx.onError failed" }, { "Error", std::string(fdb_get_error(error)) } }); self.success = false; @@ -260,8 +268,8 @@ struct SimpleWorkload : FDBWorkload { runFor = context->getOption("runFor", 10.0); auto err = fdb_select_api_version(630); if (err) { - context->trace(FDBSeverity::Info, "SelectAPIVersionFailed", - { { "Error", std::string(fdb_get_error(err)) } }); + context->trace( + FDBSeverity::Info, "SelectAPIVersionFailed", { { "Error", std::string(fdb_get_error(err)) } }); } return true; } diff --git a/bindings/c/test/workloads/workloads.cpp b/bindings/c/test/workloads/workloads.cpp index 498d58f2e8..1f01054e84 100644 --- a/bindings/c/test/workloads/workloads.cpp +++ b/bindings/c/test/workloads/workloads.cpp @@ -23,19 +23,19 @@ FDBWorkloadFactoryImpl::~FDBWorkloadFactoryImpl() {} std::map& FDBWorkloadFactoryImpl::factories() { - static std::map _factories; - return _factories; + static std::map _factories; + return _factories; } -std::shared_ptr FDBWorkloadFactoryImpl::create(const std::string &name) { - auto res = factories().find(name); - if (res == factories().end()) { - return nullptr; - } - return res->second->create(); +std::shared_ptr FDBWorkloadFactoryImpl::create(const std::string& name) { + auto res = factories().find(name); + if (res == factories().end()) { + return nullptr; + } + return res->second->create(); } FDBWorkloadFactory* workloadFactory(FDBLogger*) { - static FDBWorkloadFactoryImpl impl; - return &impl; + static FDBWorkloadFactoryImpl impl; + return &impl; } diff --git a/bindings/c/test/workloads/workloads.h b/bindings/c/test/workloads/workloads.h index d058922e68..b9666d80c2 100644 --- a/bindings/c/test/workloads/workloads.h +++ b/bindings/c/test/workloads/workloads.h @@ -33,15 +33,11 @@ struct FDBWorkloadFactoryImpl : FDBWorkloadFactory { std::shared_ptr create(const std::string& name) override; }; -template +template struct FDBWorkloadFactoryT : IFDBWorkloadFactory { - explicit FDBWorkloadFactoryT(const std::string& name) { - FDBWorkloadFactoryImpl::factories()[name] = this; - } + explicit FDBWorkloadFactoryT(const std::string& name) { FDBWorkloadFactoryImpl::factories()[name] = this; } - std::shared_ptr create() override { - return std::make_shared(); - } + std::shared_ptr create() override { return std::make_shared(); } }; extern "C" DLLEXPORT FDBWorkloadFactory* workloadFactory(FDBLogger*); diff --git a/bindings/flow/DirectoryLayer.actor.cpp b/bindings/flow/DirectoryLayer.actor.cpp index 1b80249773..3ef201456f 100644 --- a/bindings/flow/DirectoryLayer.actor.cpp +++ b/bindings/flow/DirectoryLayer.actor.cpp @@ -22,486 +22,541 @@ #include "DirectoryPartition.h" namespace FDB { - const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = {1,0,0,0,0,0,0,0}; - const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca"); - const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer"); - const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version"); - const int64_t DirectoryLayer::SUB_DIR_KEY = 0; +const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = { 1, 0, 0, 0, 0, 0, 0, 0 }; +const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca"); +const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer"); +const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version"); +const int64_t DirectoryLayer::SUB_DIR_KEY = 0; - const uint32_t DirectoryLayer::VERSION[3] = {1, 0, 0}; +const uint32_t DirectoryLayer::VERSION[3] = { 1, 0, 0 }; - const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = LiteralStringRef("\xfe"); - const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX); - const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace(); - const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition"); +const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = LiteralStringRef("\xfe"); +const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX); +const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace(); +const StringRef DirectoryLayer::PARTITION_LAYER = LiteralStringRef("partition"); - DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes) : - nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes), - rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY)) - { } +DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes) + : nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), allowManualPrefixes(allowManualPrefixes), + rootNode(nodeSubspace.get(nodeSubspace.key())), allocator(rootNode.get(HIGH_CONTENTION_KEY)) {} - Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const { - return nodeSubspace.get(prefix); - } - - template - Optional DirectoryLayer::nodeWithPrefix(Optional const& prefix) const { - if(!prefix.present()) { - return Optional(); - } - - return nodeWithPrefix(prefix.get()); - } - - ACTOR Future find(Reference dirLayer, Reference tr, IDirectory::Path path) { - state int pathIndex = 0; - state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path); - - for(; pathIndex != path.size(); ++pathIndex) { - ASSERT(node.subspace.present()); - Optional> val = wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key())); - - node.path.push_back(path[pathIndex]); - node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path); - - DirectoryLayer::Node _node = wait(node.loadMetadata(tr)); - node = _node; - - if(!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) { - return node; - } - } - - if(!node.loadedMetadata) { - DirectoryLayer::Node _node = wait(node.loadMetadata(tr)); - node = _node; - } - - return node; - } - - IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const { - Path path; - - path.reserve(this->path.size() + subpath.size()); - path.insert(path.end(), this->path.begin(), this->path.end()); - path.insert(path.end(), subpath.begin(), subpath.end()); - - return path; - } - - Reference DirectoryLayer::contentsOfNode(Subspace const& node, Path const& path, Standalone const& layer) { - Standalone prefix = nodeSubspace.unpack(node.key()).getString(0); - - if(layer == PARTITION_LAYER) { - return Reference(new DirectoryPartition(toAbsolutePath(path), prefix, Reference::addRef(this))); - } - else { - return Reference(new DirectorySubspace(toAbsolutePath(path), prefix, Reference::addRef(this), layer)); - } - } - - Reference DirectoryLayer::openInternal(Standalone const& layer, Node const& existingNode, bool allowOpen) { - if (!allowOpen) { - throw directory_already_exists(); - } - if(layer.size() > 0 && layer != existingNode.layer) { - throw mismatched_layer(); - } - - return existingNode.getContents(); - } - - Future> DirectoryLayer::open(Reference const& tr, Path const& path, Standalone const& layer) { - return createOrOpenInternal(tr, path, layer, Optional>(), false, true); - } - - void DirectoryLayer::initializeDirectory(Reference const& tr) const { - tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12)); - } - - ACTOR Future checkVersionInternal(const DirectoryLayer* dirLayer, Reference tr, bool writeAccess) { - Optional> versionBytes = wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY))); - - if(!versionBytes.present()) { - if(writeAccess) { - dirLayer->initializeDirectory(tr); - } - return Void(); - } - else { - if(versionBytes.get().size() != 12) { - throw invalid_directory_layer_metadata(); - } - if(((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) { - throw incompatible_directory_version(); - } - else if(((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) { - throw incompatible_directory_version(); - } - } - - return Void(); - } - - Future DirectoryLayer::checkVersion(Reference const& tr, bool writeAccess) const { - return checkVersionInternal(this, tr, writeAccess); - } - - ACTOR Future> getPrefix(Reference dirLayer, Reference tr, Optional> prefix) { - if(!prefix.present()) { - Standalone allocated = wait(dirLayer->allocator.allocate(tr)); - state Standalone finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key()); - - FDBStandalone result = wait(tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1)); - - if(result.size() > 0) { - throw directory_prefix_not_empty(); - } - - return finalPrefix; - } - - return prefix.get(); - } - - ACTOR Future> nodeContainingKey(Reference dirLayer, Reference tr, Standalone key, bool snapshot) { - if(key.startsWith(dirLayer->nodeSubspace.key())) { - return dirLayer->rootNode; - } - - KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key))); - FDBStandalone result = wait(tr->getRange(range, 1, snapshot, true)); - - if(result.size() > 0) { - Standalone prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0); - if(key.startsWith(prevPrefix)) { - return dirLayer->nodeWithPrefix(prevPrefix); - } - } +Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const { + return nodeSubspace.get(prefix); +} +template +Optional DirectoryLayer::nodeWithPrefix(Optional const& prefix) const { + if (!prefix.present()) { return Optional(); } - ACTOR Future isPrefixFree(Reference dirLayer, Reference tr, Standalone prefix, bool snapshot){ - if(!prefix.size()) { - return false; - } + return nodeWithPrefix(prefix.get()); +} - Optional node = wait(nodeContainingKey(dirLayer, tr, prefix, snapshot)); - if(node.present()) { - return false; - } +ACTOR Future find(Reference dirLayer, + Reference tr, + IDirectory::Path path) { + state int pathIndex = 0; + state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path); - FDBStandalone result = wait(tr->getRange(KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot)); - return !result.size(); + for (; pathIndex != path.size(); ++pathIndex) { + ASSERT(node.subspace.present()); + Optional> val = + wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key())); - } + node.path.push_back(path[pathIndex]); + node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path); - ACTOR Future getParentNode(Reference dirLayer, Reference tr, IDirectory::Path path) { - if(path.size() > 1) { - Reference parent = wait(dirLayer->createOrOpenInternal(tr, IDirectory::Path(path.begin(), path.end() - 1), StringRef(), Optional>(), true, true)); - return dirLayer->nodeWithPrefix(parent->key()); - } - else { - return dirLayer->rootNode; + DirectoryLayer::Node _node = wait(node.loadMetadata(tr)); + node = _node; + + if (!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) { + return node; } } - ACTOR Future> createInternal( - Reference dirLayer, Reference tr, IDirectory::Path path, - Standalone layer, Optional> prefix, bool allowCreate) - { - if(!allowCreate) { - throw directory_does_not_exist(); - } - - wait(dirLayer->checkVersion(tr, true)); - - state Standalone newPrefix = wait(getPrefix(dirLayer, tr, prefix)); - bool isFree = wait(isPrefixFree(dirLayer, tr, newPrefix, !prefix.present())); - - if(!isFree) { - throw directory_prefix_in_use(); - } - - Subspace parentNode = wait(getParentNode(dirLayer, tr, path)); - Subspace node = dirLayer->nodeWithPrefix(newPrefix); - - tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix); - tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer); - return dirLayer->contentsOfNode(node, path, layer); + if (!node.loadedMetadata) { + DirectoryLayer::Node _node = wait(node.loadMetadata(tr)); + node = _node; } - ACTOR Future> _createOrOpenInternal( - Reference dirLayer, Reference tr, IDirectory::Path path, - Standalone layer, Optional> prefix, bool allowCreate, bool allowOpen) - { - ASSERT(!prefix.present() || allowCreate); - wait(dirLayer->checkVersion(tr, false)); + return node; +} - if(prefix.present() && !dirLayer->allowManualPrefixes) { - if(!dirLayer->getPath().size()) { - throw manual_prefixes_not_enabled(); - } - else { - throw prefix_in_partition(); - } +IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const { + Path path; + + path.reserve(this->path.size() + subpath.size()); + path.insert(path.end(), this->path.begin(), this->path.end()); + path.insert(path.end(), subpath.begin(), subpath.end()); + + return path; +} + +Reference DirectoryLayer::contentsOfNode(Subspace const& node, + Path const& path, + Standalone const& layer) { + Standalone prefix = nodeSubspace.unpack(node.key()).getString(0); + + if (layer == PARTITION_LAYER) { + return Reference( + new DirectoryPartition(toAbsolutePath(path), prefix, Reference::addRef(this))); + } else { + return Reference( + new DirectorySubspace(toAbsolutePath(path), prefix, Reference::addRef(this), layer)); + } +} + +Reference DirectoryLayer::openInternal(Standalone const& layer, + Node const& existingNode, + bool allowOpen) { + if (!allowOpen) { + throw directory_already_exists(); + } + if (layer.size() > 0 && layer != existingNode.layer) { + throw mismatched_layer(); + } + + return existingNode.getContents(); +} + +Future> DirectoryLayer::open(Reference const& tr, + Path const& path, + Standalone const& layer) { + return createOrOpenInternal(tr, path, layer, Optional>(), false, true); +} + +void DirectoryLayer::initializeDirectory(Reference const& tr) const { + tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12)); +} + +ACTOR Future checkVersionInternal(const DirectoryLayer* dirLayer, Reference tr, bool writeAccess) { + Optional> versionBytes = + wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY))); + + if (!versionBytes.present()) { + if (writeAccess) { + dirLayer->initializeDirectory(tr); + } + return Void(); + } else { + if (versionBytes.get().size() != 12) { + throw invalid_directory_layer_metadata(); + } + if (((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) { + throw incompatible_directory_version(); + } else if (((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) { + throw incompatible_directory_version(); + } + } + + return Void(); +} + +Future DirectoryLayer::checkVersion(Reference const& tr, bool writeAccess) const { + return checkVersionInternal(this, tr, writeAccess); +} + +ACTOR Future> getPrefix(Reference dirLayer, + Reference tr, + Optional> prefix) { + if (!prefix.present()) { + Standalone allocated = wait(dirLayer->allocator.allocate(tr)); + state Standalone finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key()); + + FDBStandalone result = wait(tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1)); + + if (result.size() > 0) { + throw directory_prefix_not_empty(); } - if(!path.size()){ - throw cannot_open_root_directory(); - } + return finalPrefix; + } - state DirectoryLayer::Node existingNode = wait(find(dirLayer, tr, path)); - if(existingNode.exists()) { - if(existingNode.isInPartition()) { - IDirectory::Path subpath = existingNode.getPartitionSubpath(); - Reference dirSpace = wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal(tr, subpath, layer, prefix, allowCreate, allowOpen)); - return dirSpace; - } - return dirLayer->openInternal(layer, existingNode, allowOpen); + return prefix.get(); +} + +ACTOR Future> nodeContainingKey(Reference dirLayer, + Reference tr, + Standalone key, + bool snapshot) { + if (key.startsWith(dirLayer->nodeSubspace.key())) { + return dirLayer->rootNode; + } + + KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key))); + FDBStandalone result = wait(tr->getRange(range, 1, snapshot, true)); + + if (result.size() > 0) { + Standalone prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0); + if (key.startsWith(prevPrefix)) { + return dirLayer->nodeWithPrefix(prevPrefix); } - else { - Reference dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate)); + } + + return Optional(); +} + +ACTOR Future isPrefixFree(Reference dirLayer, + Reference tr, + Standalone prefix, + bool snapshot) { + if (!prefix.size()) { + return false; + } + + Optional node = wait(nodeContainingKey(dirLayer, tr, prefix, snapshot)); + if (node.present()) { + return false; + } + + FDBStandalone result = wait(tr->getRange( + KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot)); + return !result.size(); +} + +ACTOR Future getParentNode(Reference dirLayer, + Reference tr, + IDirectory::Path path) { + if (path.size() > 1) { + Reference parent = + wait(dirLayer->createOrOpenInternal(tr, + IDirectory::Path(path.begin(), path.end() - 1), + StringRef(), + Optional>(), + true, + true)); + return dirLayer->nodeWithPrefix(parent->key()); + } else { + return dirLayer->rootNode; + } +} + +ACTOR Future> createInternal(Reference dirLayer, + Reference tr, + IDirectory::Path path, + Standalone layer, + Optional> prefix, + bool allowCreate) { + if (!allowCreate) { + throw directory_does_not_exist(); + } + + wait(dirLayer->checkVersion(tr, true)); + + state Standalone newPrefix = wait(getPrefix(dirLayer, tr, prefix)); + bool isFree = wait(isPrefixFree(dirLayer, tr, newPrefix, !prefix.present())); + + if (!isFree) { + throw directory_prefix_in_use(); + } + + Subspace parentNode = wait(getParentNode(dirLayer, tr, path)); + Subspace node = dirLayer->nodeWithPrefix(newPrefix); + + tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix); + tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer); + return dirLayer->contentsOfNode(node, path, layer); +} + +ACTOR Future> _createOrOpenInternal(Reference dirLayer, + Reference tr, + IDirectory::Path path, + Standalone layer, + Optional> prefix, + bool allowCreate, + bool allowOpen) { + ASSERT(!prefix.present() || allowCreate); + wait(dirLayer->checkVersion(tr, false)); + + if (prefix.present() && !dirLayer->allowManualPrefixes) { + if (!dirLayer->getPath().size()) { + throw manual_prefixes_not_enabled(); + } else { + throw prefix_in_partition(); + } + } + + if (!path.size()) { + throw cannot_open_root_directory(); + } + + state DirectoryLayer::Node existingNode = wait(find(dirLayer, tr, path)); + if (existingNode.exists()) { + if (existingNode.isInPartition()) { + IDirectory::Path subpath = existingNode.getPartitionSubpath(); + Reference dirSpace = + wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal( + tr, subpath, layer, prefix, allowCreate, allowOpen)); return dirSpace; } + return dirLayer->openInternal(layer, existingNode, allowOpen); + } else { + Reference dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate)); + return dirSpace; + } +} + +Future> DirectoryLayer::createOrOpenInternal(Reference const& tr, + Path const& path, + Standalone const& layer, + Optional> const& prefix, + bool allowCreate, + bool allowOpen) { + return _createOrOpenInternal( + Reference::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen); +} + +Future> DirectoryLayer::create(Reference const& tr, + Path const& path, + Standalone const& layer, + Optional> const& prefix) { + return createOrOpenInternal(tr, path, layer, prefix, true, false); +} + +Future> DirectoryLayer::createOrOpen(Reference const& tr, + Path const& path, + Standalone const& layer) { + return createOrOpenInternal(tr, path, layer, Optional>(), true, true); +} + +ACTOR Future>> listInternal(Reference dirLayer, + Reference tr, + IDirectory::Path path) { + wait(dirLayer->checkVersion(tr, false)); + + state DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); + + if (!node.exists()) { + throw directory_does_not_exist(); + } + if (node.isInPartition(true)) { + Standalone> partitionList = + wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath())); + return partitionList; } - Future> DirectoryLayer::createOrOpenInternal( - Reference const& tr, Path const& path, Standalone const& layer, - Optional> const& prefix, bool allowCreate, bool allowOpen) - { - return _createOrOpenInternal(Reference::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen); - } + state Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY); + state Key begin = subdir.range().begin; + state Standalone> subdirectories; - Future> DirectoryLayer::create( - Reference const& tr, Path const& path, Standalone const& layer, - Optional> const& prefix) - { - return createOrOpenInternal(tr, path, layer, prefix, true, false); - } + loop { + FDBStandalone subdirRange = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end))); - Future> DirectoryLayer::createOrOpen(Reference const& tr, Path const& path, Standalone const& layer) { - return createOrOpenInternal(tr, path, layer, Optional>(), true, true); - } - - ACTOR Future>> listInternal(Reference dirLayer, Reference tr, IDirectory::Path path) { - wait(dirLayer->checkVersion(tr, false)); - - state DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); - - if(!node.exists()) { - throw directory_does_not_exist(); - } - if(node.isInPartition(true)) { - Standalone> partitionList = wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath())); - return partitionList; + for (int i = 0; i < subdirRange.size(); ++i) { + subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0)); } - state Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY); - state Key begin = subdir.range().begin; - state Standalone> subdirectories; - - loop { - FDBStandalone subdirRange = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end))); - - for(int i = 0; i < subdirRange.size(); ++i) { - subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0)); - } - - if(!subdirRange.more) { - return subdirectories; - } - - begin = keyAfter(subdirRange.back().key); + if (!subdirRange.more) { + return subdirectories; } - } - Future>> DirectoryLayer::list(Reference const& tr, Path const& path) { - return listInternal(Reference::addRef(this), tr, path); + begin = keyAfter(subdirRange.back().key); } +} - bool pathsEqual(IDirectory::Path const& path1, IDirectory::Path const& path2, size_t maxElementsToCheck = std::numeric_limits::max()) { - if(std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) { +Future>> DirectoryLayer::list(Reference const& tr, Path const& path) { + return listInternal(Reference::addRef(this), tr, path); +} + +bool pathsEqual(IDirectory::Path const& path1, + IDirectory::Path const& path2, + size_t maxElementsToCheck = std::numeric_limits::max()) { + if (std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) { + return false; + } + for (int i = 0; i < path1.size() && i < maxElementsToCheck; ++i) { + if (path1[i] != path2[i]) { return false; } - for(int i = 0; i < path1.size() && i < maxElementsToCheck; ++i) { - if(path1[i] != path2[i]) { - return false; - } - } - - return true; } - ACTOR Future removeFromParent(Reference dirLayer, Reference tr, IDirectory::Path path) { - ASSERT(path.size() >= 1); - DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1))); - if(parentNode.subspace.present()) { - tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key()); - } + return true; +} - return Void(); +ACTOR Future removeFromParent(Reference dirLayer, + Reference tr, + IDirectory::Path path) { + ASSERT(path.size() >= 1); + DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1))); + if (parentNode.subspace.present()) { + tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key()); } - ACTOR Future> moveInternal(Reference dirLayer, Reference tr, IDirectory::Path oldPath, IDirectory::Path newPath) { - wait(dirLayer->checkVersion(tr, true)); + return Void(); +} - if(oldPath.size() <= newPath.size()) { - if(pathsEqual(oldPath, newPath, oldPath.size())) { - throw invalid_destination_directory(); - } +ACTOR Future> moveInternal(Reference dirLayer, + Reference tr, + IDirectory::Path oldPath, + IDirectory::Path newPath) { + wait(dirLayer->checkVersion(tr, true)); + + if (oldPath.size() <= newPath.size()) { + if (pathsEqual(oldPath, newPath, oldPath.size())) { + throw invalid_destination_directory(); } - - std::vector> futures; - futures.push_back(find(dirLayer, tr, oldPath)); - futures.push_back(find(dirLayer, tr, newPath)); - - std::vector nodes = wait(getAll(futures)); - - state DirectoryLayer::Node oldNode = nodes[0]; - state DirectoryLayer::Node newNode = nodes[1]; - - if(!oldNode.exists()) { - throw directory_does_not_exist(); - } - - if(oldNode.isInPartition() || newNode.isInPartition()) { - if(!oldNode.isInPartition() || !newNode.isInPartition() || !pathsEqual(oldNode.path, newNode.path)) { - throw cannot_move_directory_between_partitions(); - } - - Reference partitionMove = wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath())); - return partitionMove; - } - - if(newNode.exists() || newPath.empty()) { - throw directory_already_exists(); - } - - DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1))); - if(!parentNode.exists()) { - throw parent_directory_does_not_exist(); - } - - tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(), dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0)); - wait(removeFromParent(dirLayer, tr, oldPath)); - - return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer); } - Future> DirectoryLayer::move(Reference const& tr, Path const& oldPath, Path const& newPath) { - return moveInternal(Reference::addRef(this), tr, oldPath, newPath); + std::vector> futures; + futures.push_back(find(dirLayer, tr, oldPath)); + futures.push_back(find(dirLayer, tr, newPath)); + + std::vector nodes = wait(getAll(futures)); + + state DirectoryLayer::Node oldNode = nodes[0]; + state DirectoryLayer::Node newNode = nodes[1]; + + if (!oldNode.exists()) { + throw directory_does_not_exist(); } - Future> DirectoryLayer::moveTo(Reference const& tr, Path const& newAbsolutePath) { + if (oldNode.isInPartition() || newNode.isInPartition()) { + if (!oldNode.isInPartition() || !newNode.isInPartition() || !pathsEqual(oldNode.path, newNode.path)) { + throw cannot_move_directory_between_partitions(); + } + + Reference partitionMove = + wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath())); + return partitionMove; + } + + if (newNode.exists() || newPath.empty()) { + throw directory_already_exists(); + } + + DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1))); + if (!parentNode.exists()) { + throw parent_directory_does_not_exist(); + } + + tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(), + dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0)); + wait(removeFromParent(dirLayer, tr, oldPath)); + + return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer); +} + +Future> DirectoryLayer::move(Reference const& tr, + Path const& oldPath, + Path const& newPath) { + return moveInternal(Reference::addRef(this), tr, oldPath, newPath); +} + +Future> DirectoryLayer::moveTo(Reference const& tr, + Path const& newAbsolutePath) { + throw cannot_modify_root_directory(); +} + +Future removeRecursive(Reference const&, Reference const&, Subspace const&); +ACTOR Future removeRecursive(Reference dirLayer, Reference tr, Subspace nodeSub) { + state Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY); + state Key begin = subdir.range().begin; + state std::vector> futures; + + loop { + FDBStandalone range = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end))); + for (int i = 0; i < range.size(); ++i) { + Subspace subNode = dirLayer->nodeWithPrefix(range[i].value); + futures.push_back(removeRecursive(dirLayer, tr, subNode)); + } + + if (!range.more) { + break; + } + + begin = keyAfter(range.back().key); + } + + // waits are done concurrently + wait(waitForAll(futures)); + + Standalone nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0); + + tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix))); + tr->clear(nodeSub.range()); + + return Void(); +} + +Future removeInternal(Reference const&, + Reference const&, + IDirectory::Path const&, + bool const&); +ACTOR Future removeInternal(Reference dirLayer, + Reference tr, + IDirectory::Path path, + bool failOnNonexistent) { + wait(dirLayer->checkVersion(tr, true)); + + if (path.empty()) { throw cannot_modify_root_directory(); } - Future removeRecursive(Reference const&, Reference const&, Subspace const&); - ACTOR Future removeRecursive(Reference dirLayer, Reference tr, Subspace nodeSub) { - state Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY); - state Key begin = subdir.range().begin; - state std::vector> futures; + state DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); - loop { - FDBStandalone range = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end))); - for (int i = 0; i < range.size(); ++i) { - Subspace subNode = dirLayer->nodeWithPrefix(range[i].value); - futures.push_back(removeRecursive(dirLayer, tr, subNode)); - } - - if(!range.more) { - break; - } - - begin = keyAfter(range.back().key); - } - - // waits are done concurrently - wait(waitForAll(futures)); - - Standalone nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0); - - tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix))); - tr->clear(nodeSub.range()); - - return Void(); - } - - Future removeInternal(Reference const&, Reference const&, IDirectory::Path const&, bool const&); - ACTOR Future removeInternal(Reference dirLayer, Reference tr, IDirectory::Path path, bool failOnNonexistent) { - wait(dirLayer->checkVersion(tr, true)); - - if(path.empty()) { - throw cannot_modify_root_directory(); - } - - state DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); - - if(!node.exists()) { - if(failOnNonexistent) { - throw directory_does_not_exist(); - } - else { - return false; - } - } - - if(node.isInPartition()) { - bool recurse = wait(removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent)); - return recurse; - } - - - state std::vector> futures; - futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get())); - futures.push_back(removeFromParent(dirLayer, tr, path)); - - wait(waitForAll(futures)); - - return true; - } - - Future DirectoryLayer::remove(Reference const& tr, Path const& path) { - return success(removeInternal(Reference::addRef(this), tr, path, true)); - } - - Future DirectoryLayer::removeIfExists(Reference const& tr, Path const& path) { - return removeInternal(Reference::addRef(this), tr, path, false); - } - - ACTOR Future existsInternal(Reference dirLayer, Reference tr, IDirectory::Path path) { - wait(dirLayer->checkVersion(tr, false)); - - DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); - - if(!node.exists()) { + if (!node.exists()) { + if (failOnNonexistent) { + throw directory_does_not_exist(); + } else { return false; } - - if(node.isInPartition()) { - bool exists = wait(node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath())); - return exists; - } - - return true; } - Future DirectoryLayer::exists(Reference const& tr, Path const& path) { - return existsInternal(Reference::addRef(this), tr, path); + if (node.isInPartition()) { + bool recurse = wait( + removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent)); + return recurse; } - Reference DirectoryLayer::getDirectoryLayer() { - return Reference::addRef(this); - } + state std::vector> futures; + futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get())); + futures.push_back(removeFromParent(dirLayer, tr, path)); - const Standalone DirectoryLayer::getLayer() const { - return StringRef(); - } + wait(waitForAll(futures)); - const IDirectory::Path DirectoryLayer::getPath() const { - return path; - } + return true; } + +Future DirectoryLayer::remove(Reference const& tr, Path const& path) { + return success(removeInternal(Reference::addRef(this), tr, path, true)); +} + +Future DirectoryLayer::removeIfExists(Reference const& tr, Path const& path) { + return removeInternal(Reference::addRef(this), tr, path, false); +} + +ACTOR Future existsInternal(Reference dirLayer, + Reference tr, + IDirectory::Path path) { + wait(dirLayer->checkVersion(tr, false)); + + DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); + + if (!node.exists()) { + return false; + } + + if (node.isInPartition()) { + bool exists = wait(node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath())); + return exists; + } + + return true; +} + +Future DirectoryLayer::exists(Reference const& tr, Path const& path) { + return existsInternal(Reference::addRef(this), tr, path); +} + +Reference DirectoryLayer::getDirectoryLayer() { + return Reference::addRef(this); +} + +const Standalone DirectoryLayer::getLayer() const { + return StringRef(); +} + +const IDirectory::Path DirectoryLayer::getPath() const { + return path; +} +} // namespace FDB diff --git a/bindings/flow/DirectoryLayer.h b/bindings/flow/DirectoryLayer.h old mode 100755 new mode 100644 index 36ef61c9fb..f5b87de263 --- a/bindings/flow/DirectoryLayer.h +++ b/bindings/flow/DirectoryLayer.h @@ -28,84 +28,108 @@ #include "HighContentionAllocator.h" namespace FDB { - class DirectoryLayer : public IDirectory { - public: - DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE, Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE, bool allowManualPrefixes = false); +class DirectoryLayer : public IDirectory { +public: + DirectoryLayer(Subspace nodeSubspace = DEFAULT_NODE_SUBSPACE, + Subspace contentSubspace = DEFAULT_CONTENT_SUBSPACE, + bool allowManualPrefixes = false); - Future> create(Reference const& tr, Path const& path, Standalone const& layer = Standalone(), Optional> const& prefix = Optional>()); - Future> open(Reference const& tr, Path const& path, Standalone const& layer = Standalone()); - Future> createOrOpen(Reference const& tr, Path const& path, Standalone const& layer = Standalone()); + Future> create( + Reference const& tr, + Path const& path, + Standalone const& layer = Standalone(), + Optional> const& prefix = Optional>()); + Future> open(Reference const& tr, + Path const& path, + Standalone const& layer = Standalone()); + Future> createOrOpen(Reference const& tr, + Path const& path, + Standalone const& layer = Standalone()); - Future exists(Reference const& tr, Path const& path = Path()); - Future>> list(Reference const& tr, Path const& path = Path()); + Future exists(Reference const& tr, Path const& path = Path()); + Future>> list(Reference const& tr, Path const& path = Path()); - Future> move(Reference const& tr, Path const& oldPath, Path const& newPath); - Future> moveTo(Reference const& tr, Path const& newAbsolutePath); + Future> move(Reference const& tr, + Path const& oldPath, + Path const& newPath); + Future> moveTo(Reference const& tr, Path const& newAbsolutePath); - Future remove(Reference const& tr, Path const& path = Path()); - Future removeIfExists(Reference const& tr, Path const& path = Path()); + Future remove(Reference const& tr, Path const& path = Path()); + Future removeIfExists(Reference const& tr, Path const& path = Path()); - Reference getDirectoryLayer(); - const Standalone getLayer() const; - const Path getPath() const; + Reference getDirectoryLayer(); + const Standalone getLayer() const; + const Path getPath() const; - static const Subspace DEFAULT_NODE_SUBSPACE; - static const Subspace DEFAULT_CONTENT_SUBSPACE; - static const StringRef PARTITION_LAYER; + static const Subspace DEFAULT_NODE_SUBSPACE; + static const Subspace DEFAULT_CONTENT_SUBSPACE; + static const StringRef PARTITION_LAYER; - //private: - static const uint8_t LITTLE_ENDIAN_LONG_ONE[8]; - static const StringRef HIGH_CONTENTION_KEY; - static const StringRef LAYER_KEY; - static const StringRef VERSION_KEY; - static const int64_t SUB_DIR_KEY; - static const uint32_t VERSION[3]; - static const StringRef DEFAULT_NODE_SUBSPACE_PREFIX; + // private: + static const uint8_t LITTLE_ENDIAN_LONG_ONE[8]; + static const StringRef HIGH_CONTENTION_KEY; + static const StringRef LAYER_KEY; + static const StringRef VERSION_KEY; + static const int64_t SUB_DIR_KEY; + static const uint32_t VERSION[3]; + static const StringRef DEFAULT_NODE_SUBSPACE_PREFIX; - struct Node { - Node() {} - Node(Reference const& directoryLayer, Optional const& subspace, Path const& path, Path const& targetPath); + struct Node { + Node() {} + Node(Reference const& directoryLayer, + Optional const& subspace, + Path const& path, + Path const& targetPath); - bool exists() const; + bool exists() const; - Future loadMetadata(Reference tr); - void ensureMetadataLoaded() const; + Future loadMetadata(Reference tr); + void ensureMetadataLoaded() const; - bool isInPartition(bool includeEmptySubpath = false) const; - Path getPartitionSubpath() const; - Reference getContents() const; - - Reference directoryLayer; - Optional subspace; - Path path; - Path targetPath; - Standalone layer; - - bool loadedMetadata; - }; - - Reference openInternal(Standalone const& layer, Node const& existingNode, bool allowOpen); - Future> createOrOpenInternal(Reference const& tr, Path const& path, Standalone const& layer, Optional> const& prefix, bool allowCreate, bool allowOpen); - - void initializeDirectory(Reference const& tr) const; - Future checkVersion(Reference const& tr, bool writeAccess) const; - - template - Optional nodeWithPrefix(Optional const& prefix) const; - Subspace nodeWithPrefix(StringRef const& prefix) const; - - Reference contentsOfNode(Subspace const& node, Path const& path, Standalone const& layer); - - Path toAbsolutePath(Path const& subpath) const; - - Subspace rootNode; - Subspace nodeSubspace; - Subspace contentSubspace; - HighContentionAllocator allocator; - bool allowManualPrefixes; + bool isInPartition(bool includeEmptySubpath = false) const; + Path getPartitionSubpath() const; + Reference getContents() const; + Reference directoryLayer; + Optional subspace; Path path; + Path targetPath; + Standalone layer; + + bool loadedMetadata; }; -} + + Reference openInternal(Standalone const& layer, + Node const& existingNode, + bool allowOpen); + Future> createOrOpenInternal(Reference const& tr, + Path const& path, + Standalone const& layer, + Optional> const& prefix, + bool allowCreate, + bool allowOpen); + + void initializeDirectory(Reference const& tr) const; + Future checkVersion(Reference const& tr, bool writeAccess) const; + + template + Optional nodeWithPrefix(Optional const& prefix) const; + Subspace nodeWithPrefix(StringRef const& prefix) const; + + Reference contentsOfNode(Subspace const& node, + Path const& path, + Standalone const& layer); + + Path toAbsolutePath(Path const& subpath) const; + + Subspace rootNode; + Subspace nodeSubspace; + Subspace contentSubspace; + HighContentionAllocator allocator; + bool allowManualPrefixes; + + Path path; +}; +} // namespace FDB #endif \ No newline at end of file diff --git a/bindings/flow/DirectoryPartition.h b/bindings/flow/DirectoryPartition.h index e154806373..8ec80e3288 100644 --- a/bindings/flow/DirectoryPartition.h +++ b/bindings/flow/DirectoryPartition.h @@ -28,34 +28,38 @@ #include "DirectoryLayer.h" namespace FDB { - class DirectoryPartition : public DirectorySubspace { +class DirectoryPartition : public DirectorySubspace { - public: - DirectoryPartition(Path const& path, StringRef const& prefix, Reference parentDirectoryLayer) - : DirectorySubspace(path, prefix, Reference(new DirectoryLayer(Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)), Subspace(prefix))), DirectoryLayer::PARTITION_LAYER), - parentDirectoryLayer(parentDirectoryLayer) - { - this->directoryLayer->path = path; - } - virtual ~DirectoryPartition() {} +public: + DirectoryPartition(Path const& path, StringRef const& prefix, Reference parentDirectoryLayer) + : DirectorySubspace(path, + prefix, + Reference(new DirectoryLayer( + Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)), + Subspace(prefix))), + DirectoryLayer::PARTITION_LAYER), + parentDirectoryLayer(parentDirectoryLayer) { + this->directoryLayer->path = path; + } + virtual ~DirectoryPartition() {} - virtual Key key() const { throw cannot_use_partition_as_subspace(); } - virtual bool contains(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); } + virtual Key key() const { throw cannot_use_partition_as_subspace(); } + virtual bool contains(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); } - virtual Key pack(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); } - virtual Tuple unpack(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); } - virtual KeyRange range(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); } + virtual Key pack(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); } + virtual Tuple unpack(KeyRef const& key) const { throw cannot_use_partition_as_subspace(); } + virtual KeyRange range(Tuple const& tuple = Tuple()) const { throw cannot_use_partition_as_subspace(); } - virtual Subspace subspace(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); } - virtual Subspace get(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); } + virtual Subspace subspace(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); } + virtual Subspace get(Tuple const& tuple) const { throw cannot_use_partition_as_subspace(); } - protected: - Reference parentDirectoryLayer; +protected: + Reference parentDirectoryLayer; - virtual Reference getDirectoryLayerForPath(Path const& path) const { - return path.empty() ? parentDirectoryLayer : directoryLayer; - } - }; -} + virtual Reference getDirectoryLayerForPath(Path const& path) const { + return path.empty() ? parentDirectoryLayer : directoryLayer; + } +}; +} // namespace FDB #endif \ No newline at end of file diff --git a/bindings/flow/DirectorySubspace.cpp b/bindings/flow/DirectorySubspace.cpp old mode 100755 new mode 100644 index 237d72285f..63b26b5765 --- a/bindings/flow/DirectorySubspace.cpp +++ b/bindings/flow/DirectorySubspace.cpp @@ -21,89 +21,100 @@ #include "DirectorySubspace.h" namespace FDB { - DirectorySubspace::DirectorySubspace(Path const& path, StringRef const& prefix, Reference directoryLayer, Standalone const& layer) - : Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) { } +DirectorySubspace::DirectorySubspace(Path const& path, + StringRef const& prefix, + Reference directoryLayer, + Standalone const& layer) + : Subspace(prefix), directoryLayer(directoryLayer), path(path), layer(layer) {} +Future> DirectorySubspace::create(Reference const& tr, + Path const& path, + Standalone const& layer, + Optional> const& prefix) { + return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix); +} - Future> DirectorySubspace::create(Reference const& tr, Path const& path, Standalone const& layer, - Optional> const& prefix) - { - return directoryLayer->create(tr, getPartitionSubpath(path), layer, prefix); +Future> DirectorySubspace::open(Reference const& tr, + Path const& path, + Standalone const& layer) { + return directoryLayer->open(tr, getPartitionSubpath(path), layer); +} + +Future> DirectorySubspace::createOrOpen(Reference const& tr, + Path const& path, + Standalone const& layer) { + return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer); +} + +Future DirectorySubspace::exists(Reference const& tr, Path const& path) { + Reference directoryLayer = getDirectoryLayerForPath(path); + return directoryLayer->exists(tr, getPartitionSubpath(path, directoryLayer)); +} + +Future>> DirectorySubspace::list(Reference const& tr, Path const& path) { + return directoryLayer->list(tr, getPartitionSubpath(path)); +} + +Future> DirectorySubspace::move(Reference const& tr, + Path const& oldPath, + Path const& newPath) { + return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath)); +} + +Future> DirectorySubspace::moveTo(Reference const& tr, + Path const& newAbsolutePath) { + Reference directoryLayer = getDirectoryLayerForPath(Path()); + Path directoryLayerPath = directoryLayer->getPath(); + + if (directoryLayerPath.size() > newAbsolutePath.size()) { + return cannot_move_directory_between_partitions(); } - Future> DirectorySubspace::open(Reference const& tr, Path const& path, Standalone const& layer) { - return directoryLayer->open(tr, getPartitionSubpath(path), layer); - } - - Future> DirectorySubspace::createOrOpen(Reference const& tr, Path const& path, Standalone const& layer) { - return directoryLayer->createOrOpen(tr, getPartitionSubpath(path), layer); - } - - Future DirectorySubspace::exists(Reference const& tr, Path const& path) { - Reference directoryLayer = getDirectoryLayerForPath(path); - return directoryLayer->exists(tr, getPartitionSubpath(path, directoryLayer)); - } - - Future>> DirectorySubspace::list(Reference const& tr, Path const& path) { - return directoryLayer->list(tr, getPartitionSubpath(path)); - } - - Future> DirectorySubspace::move(Reference const& tr, Path const& oldPath, Path const& newPath) { - return directoryLayer->move(tr, getPartitionSubpath(oldPath), getPartitionSubpath(newPath)); - } - - Future> DirectorySubspace::moveTo(Reference const& tr, Path const& newAbsolutePath) { - Reference directoryLayer = getDirectoryLayerForPath(Path()); - Path directoryLayerPath = directoryLayer->getPath(); - - if(directoryLayerPath.size() > newAbsolutePath.size()) { + for (int i = 0; i < directoryLayerPath.size(); ++i) { + if (directoryLayerPath[i] != newAbsolutePath[i]) { return cannot_move_directory_between_partitions(); } - - for(int i = 0; i < directoryLayerPath.size(); ++i) { - if(directoryLayerPath[i] != newAbsolutePath[i]) { - return cannot_move_directory_between_partitions(); - } - } - - Path newRelativePath(newAbsolutePath.begin() + directoryLayerPath.size(), newAbsolutePath.end()); - return directoryLayer->move(tr, getPartitionSubpath(Path(), directoryLayer), newRelativePath); } - Future DirectorySubspace::remove(Reference const& tr, Path const& path) { - Reference directoryLayer = getDirectoryLayerForPath(path); - return directoryLayer->remove(tr, getPartitionSubpath(path, directoryLayer)); - } - - Future DirectorySubspace::removeIfExists(Reference const& tr, Path const& path) { - Reference directoryLayer = getDirectoryLayerForPath(path); - return directoryLayer->removeIfExists(tr, getPartitionSubpath(path, directoryLayer)); - } - - Reference DirectorySubspace::getDirectoryLayer() { - return directoryLayer; - } - - const Standalone DirectorySubspace::getLayer() const { - return layer; - } - - const IDirectory::Path DirectorySubspace::getPath() const { - return path; - } - - IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path, Reference directoryLayer) const { - if(!directoryLayer) { - directoryLayer = this->directoryLayer; - } - - Path newPath(this->path.begin() + directoryLayer->getPath().size(), this->path.end()); - newPath.insert(newPath.end(), path.begin(), path.end()); - - return newPath; - } - - Reference DirectorySubspace::getDirectoryLayerForPath(Path const& path) const { - return directoryLayer; - } + Path newRelativePath(newAbsolutePath.begin() + directoryLayerPath.size(), newAbsolutePath.end()); + return directoryLayer->move(tr, getPartitionSubpath(Path(), directoryLayer), newRelativePath); } + +Future DirectorySubspace::remove(Reference const& tr, Path const& path) { + Reference directoryLayer = getDirectoryLayerForPath(path); + return directoryLayer->remove(tr, getPartitionSubpath(path, directoryLayer)); +} + +Future DirectorySubspace::removeIfExists(Reference const& tr, Path const& path) { + Reference directoryLayer = getDirectoryLayerForPath(path); + return directoryLayer->removeIfExists(tr, getPartitionSubpath(path, directoryLayer)); +} + +Reference DirectorySubspace::getDirectoryLayer() { + return directoryLayer; +} + +const Standalone DirectorySubspace::getLayer() const { + return layer; +} + +const IDirectory::Path DirectorySubspace::getPath() const { + return path; +} + +IDirectory::Path DirectorySubspace::getPartitionSubpath(Path const& path, + Reference directoryLayer) const { + if (!directoryLayer) { + directoryLayer = this->directoryLayer; + } + + Path newPath(this->path.begin() + directoryLayer->getPath().size(), this->path.end()); + newPath.insert(newPath.end(), path.begin(), path.end()); + + return newPath; +} + +Reference DirectorySubspace::getDirectoryLayerForPath(Path const& path) const { + return directoryLayer; +} +} // namespace FDB diff --git a/bindings/flow/DirectorySubspace.h b/bindings/flow/DirectorySubspace.h old mode 100755 new mode 100644 index 9a40cbf0b4..8c02fdd9e2 --- a/bindings/flow/DirectorySubspace.h +++ b/bindings/flow/DirectorySubspace.h @@ -28,39 +28,53 @@ #include "Subspace.h" namespace FDB { - class DirectorySubspace : public IDirectory, public Subspace { +class DirectorySubspace : public IDirectory, public Subspace { - public: - DirectorySubspace(Path const& path, StringRef const& prefix, Reference directorLayer, Standalone const& layer = Standalone()); - virtual ~DirectorySubspace() {} +public: + DirectorySubspace(Path const& path, + StringRef const& prefix, + Reference directorLayer, + Standalone const& layer = Standalone()); + virtual ~DirectorySubspace() {} - virtual Future> create(Reference const& tr, Path const& path, Standalone const& layer = Standalone(), - Optional> const& prefix = Optional>()); + virtual Future> create( + Reference const& tr, + Path const& path, + Standalone const& layer = Standalone(), + Optional> const& prefix = Optional>()); - virtual Future> open(Reference const& tr, Path const& path, Standalone const& layer = Standalone()); - virtual Future> createOrOpen(Reference const& tr, Path const& path, Standalone const& layer = Standalone()); + virtual Future> open(Reference const& tr, + Path const& path, + Standalone const& layer = Standalone()); + virtual Future> createOrOpen( + Reference const& tr, + Path const& path, + Standalone const& layer = Standalone()); - virtual Future exists(Reference const& tr, Path const& path = Path()); - virtual Future>> list(Reference const& tr, Path const& path = Path()); + virtual Future exists(Reference const& tr, Path const& path = Path()); + virtual Future>> list(Reference const& tr, Path const& path = Path()); - virtual Future> move(Reference const& tr, Path const& oldPath, Path const& newPath); - virtual Future> moveTo(Reference const& tr, Path const& newAbsolutePath); + virtual Future> move(Reference const& tr, + Path const& oldPath, + Path const& newPath); + virtual Future> moveTo(Reference const& tr, Path const& newAbsolutePath); - virtual Future remove(Reference const& tr, Path const& path = Path()); - virtual Future removeIfExists(Reference const& tr, Path const& path = Path()); + virtual Future remove(Reference const& tr, Path const& path = Path()); + virtual Future removeIfExists(Reference const& tr, Path const& path = Path()); - virtual Reference getDirectoryLayer(); - virtual const Standalone getLayer() const; - virtual const Path getPath() const; + virtual Reference getDirectoryLayer(); + virtual const Standalone getLayer() const; + virtual const Path getPath() const; - protected: - Reference directoryLayer; - Path path; - Standalone layer; +protected: + Reference directoryLayer; + Path path; + Standalone layer; - virtual Path getPartitionSubpath(Path const& path, Reference directoryLayer = Reference()) const; - virtual Reference getDirectoryLayerForPath(Path const& path) const; - }; -} + virtual Path getPartitionSubpath(Path const& path, + Reference directoryLayer = Reference()) const; + virtual Reference getDirectoryLayerForPath(Path const& path) const; +}; +} // namespace FDB #endif \ No newline at end of file diff --git a/bindings/flow/FDBLoanerTypes.h b/bindings/flow/FDBLoanerTypes.h old mode 100755 new mode 100644 index 3256b5339e..97e4394298 --- a/bindings/flow/FDBLoanerTypes.h +++ b/bindings/flow/FDBLoanerTypes.h @@ -22,292 +22,304 @@ #define FDB_FLOW_LOANER_TYPES_H namespace FDB { - typedef StringRef KeyRef; - typedef StringRef ValueRef; +typedef StringRef KeyRef; +typedef StringRef ValueRef; - typedef int64_t Version; +typedef int64_t Version; - typedef Standalone Key; - typedef Standalone Value; +typedef Standalone Key; +typedef Standalone Value; - inline Key keyAfter( const KeyRef& key ) { - if(key == LiteralStringRef("\xff\xff")) - return key; +inline Key keyAfter(const KeyRef& key) { + if (key == LiteralStringRef("\xff\xff")) + return key; - Standalone r; - uint8_t* s = new (r.arena()) uint8_t[ key.size() + 1 ]; - memcpy(s, key.begin(), key.size() ); - s[key.size()] = 0; - ((StringRef&) r) = StringRef( s, key.size() + 1 ); - return r; - } - - inline KeyRef keyAfter( const KeyRef& key, Arena& arena ) { - if(key == LiteralStringRef("\xff\xff")) - return key; - uint8_t* t = new ( arena ) uint8_t[ key.size()+1 ]; - memcpy(t, key.begin(), key.size() ); - t[key.size()] = 0; - return KeyRef(t,key.size()+1); - } - - struct KeySelectorRef { - KeyRef key; // Find the last item less than key - bool orEqual; // (or equal to key, if this is true) - int offset; // and then move forward this many items (or backward if negative) - KeySelectorRef() {} - KeySelectorRef( const KeyRef& key, bool orEqual, int offset ) : key(key), orEqual(orEqual), offset(offset) {} - - KeySelectorRef( Arena& arena, const KeySelectorRef& copyFrom ) : key(arena,copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {} - int expectedSize() const { return key.expectedSize(); } - - // std::string toString() const { - // if (offset > 0) { - // if (orEqual) return format("firstGreaterThan(%s)%+d", printable(key).c_str(), offset-1); - // else return format("firstGreaterOrEqual(%s)%+d", printable(key).c_str(), offset-1); - // } else { - // if (orEqual) return format("lastLessOrEqual(%s)%+d", printable(key).c_str(), offset); - // else return format("lastLessThan(%s)%+d", printable(key).c_str(), offset); - // } - // } - - bool isBackward() const { return !orEqual && offset<=0; } // True if the resolution of the KeySelector depends only on keys less than key - bool isFirstGreaterOrEqual() const { return !orEqual && offset==1; } - bool isFirstGreaterThan() const { return orEqual && offset==1; } - bool isLastLessOrEqual() const { return orEqual && offset==0; } - - // True iff, regardless of the contents of the database, lhs must resolve to a key > rhs - bool isDefinitelyGreater( KeyRef const& k ) { - return offset >= 1 && ( isFirstGreaterOrEqual() ? key > k : key >= k ); - } - // True iff, regardless of the contents of the database, lhs must resolve to a key < rhs - bool isDefinitelyLess( KeyRef const& k ) { - return offset <= 0 && ( isLastLessOrEqual() ? key < k : key <= k ); - } - - template - void serialize( Ar& ar ) { - serializer(ar, key, orEqual, offset); - } - }; - inline bool operator == (const KeySelectorRef& lhs, const KeySelectorRef& rhs) { return lhs.key == rhs.key && lhs.orEqual==rhs.orEqual && lhs.offset==rhs.offset; } - inline KeySelectorRef lastLessThan( const KeyRef& k ) { - return KeySelectorRef( k, false, 0 ); - } - inline KeySelectorRef lastLessOrEqual( const KeyRef& k ) { - return KeySelectorRef( k, true, 0 ); - } - inline KeySelectorRef firstGreaterThan( const KeyRef& k ) { - return KeySelectorRef( k, true, +1 ); - } - inline KeySelectorRef firstGreaterOrEqual( const KeyRef& k ) { - return KeySelectorRef( k, false, +1 ); - } - inline KeySelectorRef operator + (const KeySelectorRef& s, int off) { - return KeySelectorRef(s.key, s.orEqual, s.offset+off); - } - inline KeySelectorRef operator - (const KeySelectorRef& s, int off) { - return KeySelectorRef(s.key, s.orEqual, s.offset-off); - } - - typedef Standalone KeySelector; - - struct KeyValueRef { - KeyRef key; - ValueRef value; - KeyValueRef() {} - KeyValueRef( const KeyRef& key, const ValueRef& value ) : key(key), value(value) {} - KeyValueRef( Arena& a, const KeyValueRef& copyFrom ) : key(a, copyFrom.key), value(a, copyFrom.value) {} - bool operator == ( const KeyValueRef& r ) const { return key == r.key && value == r.value; } - - int expectedSize() const { return key.expectedSize() + value.expectedSize(); } - - template - force_inline void serialize(Ar& ar) { serializer(ar, key, value); } - - struct OrderByKey { - bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { - return a.key < b.key; - } - template - bool operator()(T const& a, KeyValueRef const& b) const { - return a < b.key; - } - template - bool operator()(KeyValueRef const& a, T const& b) const { - return a.key < b; - } - }; - - struct OrderByKeyBack { - bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { - return a.key > b.key; - } - template - bool operator()(T const& a, KeyValueRef const& b) const { - return a > b.key; - } - template - bool operator()(KeyValueRef const& a, T const& b) const { - return a.key > b; - } - }; - }; - - typedef Standalone KeyValue; - - struct RangeResultRef : VectorRef { - bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the limits requested) - // False implies that no such values remain - Optional readThrough; // Only present when 'more' is true. When present, this value represent the end (or beginning if reverse) of the range - // which was read to produce these results. This is guarenteed to be less than the requested range. - bool readToBegin; - bool readThroughEnd; - - RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {} - RangeResultRef( Arena& p, const RangeResultRef& toCopy ) : more( toCopy.more ), readToBegin( toCopy.readToBegin ), readThroughEnd( toCopy.readThroughEnd ), readThrough( toCopy.readThrough.present() ? KeyRef( p, toCopy.readThrough.get() ) : Optional() ), VectorRef( p, toCopy ) {} - RangeResultRef( const VectorRef& value, bool more, Optional readThrough = Optional() ) : VectorRef( value ), more( more ), readThrough( readThrough ), readToBegin( false ), readThroughEnd( false ) {} - RangeResultRef( bool readToBegin, bool readThroughEnd ) : more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) { } - - template - void serialize( Ar& ar ) { - serializer(ar, ((VectorRef&)*this), more, readThrough, readToBegin, readThroughEnd); - } - }; - - struct GetRangeLimits { - enum { ROW_LIMIT_UNLIMITED = -1, BYTE_LIMIT_UNLIMITED = -1 }; - - int rows; - int minRows; - int bytes; - - GetRangeLimits() : rows( ROW_LIMIT_UNLIMITED ), minRows(1), bytes( BYTE_LIMIT_UNLIMITED ) {} - explicit GetRangeLimits( int rowLimit ) : rows( rowLimit ), minRows(1), bytes( BYTE_LIMIT_UNLIMITED ) {} - GetRangeLimits( int rowLimit, int byteLimit ) : rows( rowLimit ), minRows(1), bytes( byteLimit ) {} - - void decrement( VectorRef const& data ); - void decrement( KeyValueRef const& data ); - - // True if either the row or byte limit has been reached - bool isReached(); - - // True if data would cause the row or byte limit to be reached - bool reachedBy( VectorRef const& data ); - - bool hasByteLimit(); - bool hasRowLimit(); - - bool hasSatisfiedMinRows(); - bool isValid() { return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED) - && (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED) - && minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED); } - }; - - struct KeyRangeRef { - const KeyRef begin, end; - KeyRangeRef() {} - KeyRangeRef( const KeyRef& begin, const KeyRef& end ) : begin(begin), end(end) { - if( begin > end ) { - throw inverted_range(); - } - } - KeyRangeRef( Arena& a, const KeyRangeRef& copyFrom ) : begin(a, copyFrom.begin), end(a, copyFrom.end) {} - bool operator == ( const KeyRangeRef& r ) const { return begin == r.begin && end == r.end; } - bool operator != ( const KeyRangeRef& r ) const { return begin != r.begin || end != r.end; } - bool contains( const KeyRef& key ) const { return begin <= key && key < end; } - bool contains( const KeyRangeRef& keys ) const { return begin <= keys.begin && keys.end <= end; } - bool intersects( const KeyRangeRef& keys ) const { return begin < keys.end && keys.begin < end; } - bool empty() const { return begin == end; } - - Standalone withPrefix( const StringRef& prefix ) const { - return KeyRangeRef( begin.withPrefix(prefix), end.withPrefix(prefix) ); - } - - const KeyRangeRef& operator = (const KeyRangeRef& rhs) { - const_cast(begin) = rhs.begin; - const_cast(end) = rhs.end; - return *this; - } - - int expectedSize() const { return begin.expectedSize() + end.expectedSize(); } - - template - force_inline void serialize(Ar& ar) { - serializer(ar, const_cast(begin), const_cast(end)); - if( begin > end ) { - throw inverted_range(); - }; - } - - struct ArbitraryOrder { - bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const { - if (a.begin < b.begin) return true; - if (a.begin > b.begin) return false; - return a.end < b.end; - } - }; - }; - - inline KeyRangeRef operator & (const KeyRangeRef& lhs, const KeyRangeRef& rhs) { - KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end); - if (e < b) - return KeyRangeRef(); - return KeyRangeRef(b,e); - } - - typedef Standalone KeyRange; - - template - static std::string describe(T const& item) { - return item.toString(); - } - template - static std::string describe(std::map const& items, int max_items = -1) { - if (!items.size()) - return "[no items]"; - - std::string s; - int count = 0; - for (auto it = items.begin(); it != items.end(); it++) { - if (++count > max_items && max_items >= 0) - break; - if (count > 1) s += ","; - s += describe(it->first) + "=>" + describe(it->second); - } - return s; - } - - template - static std::string describeList(T const& items, int max_items) { - if (!items.size()) - return "[no items]"; - - std::string s; - int count = 0; - for (auto const& item : items) { - if (++count > max_items && max_items >= 0) - break; - if (count > 1) s += ","; - s += describe(item); - } - return s; - } - - template - static std::string describe(std::vector const& items, int max_items = -1) { - return describeList(items, max_items); - } - - template - static std::string describe(std::set const& items, int max_items = -1) { - return describeList(items, max_items); - } - - template - static std::string describe(std::pair const& pair) { - return "first: " + describe(pair.first) + " second: " + describe(pair.second); - } + Standalone r; + uint8_t* s = new (r.arena()) uint8_t[key.size() + 1]; + memcpy(s, key.begin(), key.size()); + s[key.size()] = 0; + ((StringRef&)r) = StringRef(s, key.size() + 1); + return r; } +inline KeyRef keyAfter(const KeyRef& key, Arena& arena) { + if (key == LiteralStringRef("\xff\xff")) + return key; + uint8_t* t = new (arena) uint8_t[key.size() + 1]; + memcpy(t, key.begin(), key.size()); + t[key.size()] = 0; + return KeyRef(t, key.size() + 1); +} + +struct KeySelectorRef { + KeyRef key; // Find the last item less than key + bool orEqual; // (or equal to key, if this is true) + int offset; // and then move forward this many items (or backward if negative) + KeySelectorRef() {} + KeySelectorRef(const KeyRef& key, bool orEqual, int offset) : key(key), orEqual(orEqual), offset(offset) {} + + KeySelectorRef(Arena& arena, const KeySelectorRef& copyFrom) + : key(arena, copyFrom.key), orEqual(copyFrom.orEqual), offset(copyFrom.offset) {} + int expectedSize() const { return key.expectedSize(); } + + // std::string toString() const { + // if (offset > 0) { + // if (orEqual) return format("firstGreaterThan(%s)%+d", printable(key).c_str(), offset-1); + // else return format("firstGreaterOrEqual(%s)%+d", printable(key).c_str(), offset-1); + // } else { + // if (orEqual) return format("lastLessOrEqual(%s)%+d", printable(key).c_str(), offset); + // else return format("lastLessThan(%s)%+d", printable(key).c_str(), offset); + // } + // } + + bool isBackward() const { + return !orEqual && offset <= 0; + } // True if the resolution of the KeySelector depends only on keys less than key + bool isFirstGreaterOrEqual() const { return !orEqual && offset == 1; } + bool isFirstGreaterThan() const { return orEqual && offset == 1; } + bool isLastLessOrEqual() const { return orEqual && offset == 0; } + + // True iff, regardless of the contents of the database, lhs must resolve to a key > rhs + bool isDefinitelyGreater(KeyRef const& k) { return offset >= 1 && (isFirstGreaterOrEqual() ? key > k : key >= k); } + // True iff, regardless of the contents of the database, lhs must resolve to a key < rhs + bool isDefinitelyLess(KeyRef const& k) { return offset <= 0 && (isLastLessOrEqual() ? key < k : key <= k); } + + template + void serialize(Ar& ar) { + serializer(ar, key, orEqual, offset); + } +}; +inline bool operator==(const KeySelectorRef& lhs, const KeySelectorRef& rhs) { + return lhs.key == rhs.key && lhs.orEqual == rhs.orEqual && lhs.offset == rhs.offset; +} +inline KeySelectorRef lastLessThan(const KeyRef& k) { + return KeySelectorRef(k, false, 0); +} +inline KeySelectorRef lastLessOrEqual(const KeyRef& k) { + return KeySelectorRef(k, true, 0); +} +inline KeySelectorRef firstGreaterThan(const KeyRef& k) { + return KeySelectorRef(k, true, +1); +} +inline KeySelectorRef firstGreaterOrEqual(const KeyRef& k) { + return KeySelectorRef(k, false, +1); +} +inline KeySelectorRef operator+(const KeySelectorRef& s, int off) { + return KeySelectorRef(s.key, s.orEqual, s.offset + off); +} +inline KeySelectorRef operator-(const KeySelectorRef& s, int off) { + return KeySelectorRef(s.key, s.orEqual, s.offset - off); +} + +typedef Standalone KeySelector; + +struct KeyValueRef { + KeyRef key; + ValueRef value; + KeyValueRef() {} + KeyValueRef(const KeyRef& key, const ValueRef& value) : key(key), value(value) {} + KeyValueRef(Arena& a, const KeyValueRef& copyFrom) : key(a, copyFrom.key), value(a, copyFrom.value) {} + bool operator==(const KeyValueRef& r) const { return key == r.key && value == r.value; } + + int expectedSize() const { return key.expectedSize() + value.expectedSize(); } + + template + force_inline void serialize(Ar& ar) { + serializer(ar, key, value); + } + + struct OrderByKey { + bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key < b.key; } + template + bool operator()(T const& a, KeyValueRef const& b) const { + return a < b.key; + } + template + bool operator()(KeyValueRef const& a, T const& b) const { + return a.key < b; + } + }; + + struct OrderByKeyBack { + bool operator()(KeyValueRef const& a, KeyValueRef const& b) const { return a.key > b.key; } + template + bool operator()(T const& a, KeyValueRef const& b) const { + return a > b.key; + } + template + bool operator()(KeyValueRef const& a, T const& b) const { + return a.key > b; + } + }; +}; + +typedef Standalone KeyValue; + +struct RangeResultRef : VectorRef { + bool more; // True if (but not necessarily only if) values remain in the *key* range requested (possibly beyond the + // limits requested) + // False implies that no such values remain + Optional readThrough; // Only present when 'more' is true. When present, this value represent the end (or + // beginning if reverse) of the range + // which was read to produce these results. This is guarenteed to be less than the requested range. + bool readToBegin; + bool readThroughEnd; + + RangeResultRef() : more(false), readToBegin(false), readThroughEnd(false) {} + RangeResultRef(Arena& p, const RangeResultRef& toCopy) + : more(toCopy.more), readToBegin(toCopy.readToBegin), readThroughEnd(toCopy.readThroughEnd), + readThrough(toCopy.readThrough.present() ? KeyRef(p, toCopy.readThrough.get()) : Optional()), + VectorRef(p, toCopy) {} + RangeResultRef(const VectorRef& value, bool more, Optional readThrough = Optional()) + : VectorRef(value), more(more), readThrough(readThrough), readToBegin(false), readThroughEnd(false) { + } + RangeResultRef(bool readToBegin, bool readThroughEnd) + : more(false), readToBegin(readToBegin), readThroughEnd(readThroughEnd) {} + + template + void serialize(Ar& ar) { + serializer(ar, ((VectorRef&)*this), more, readThrough, readToBegin, readThroughEnd); + } +}; + +struct GetRangeLimits { + enum { ROW_LIMIT_UNLIMITED = -1, BYTE_LIMIT_UNLIMITED = -1 }; + + int rows; + int minRows; + int bytes; + + GetRangeLimits() : rows(ROW_LIMIT_UNLIMITED), minRows(1), bytes(BYTE_LIMIT_UNLIMITED) {} + explicit GetRangeLimits(int rowLimit) : rows(rowLimit), minRows(1), bytes(BYTE_LIMIT_UNLIMITED) {} + GetRangeLimits(int rowLimit, int byteLimit) : rows(rowLimit), minRows(1), bytes(byteLimit) {} + + void decrement(VectorRef const& data); + void decrement(KeyValueRef const& data); + + // True if either the row or byte limit has been reached + bool isReached(); + + // True if data would cause the row or byte limit to be reached + bool reachedBy(VectorRef const& data); + + bool hasByteLimit(); + bool hasRowLimit(); + + bool hasSatisfiedMinRows(); + bool isValid() { + return (rows >= 0 || rows == ROW_LIMIT_UNLIMITED) && (bytes >= 0 || bytes == BYTE_LIMIT_UNLIMITED) && + minRows >= 0 && (minRows <= rows || rows == ROW_LIMIT_UNLIMITED); + } +}; + +struct KeyRangeRef { + const KeyRef begin, end; + KeyRangeRef() {} + KeyRangeRef(const KeyRef& begin, const KeyRef& end) : begin(begin), end(end) { + if (begin > end) { + throw inverted_range(); + } + } + KeyRangeRef(Arena& a, const KeyRangeRef& copyFrom) : begin(a, copyFrom.begin), end(a, copyFrom.end) {} + bool operator==(const KeyRangeRef& r) const { return begin == r.begin && end == r.end; } + bool operator!=(const KeyRangeRef& r) const { return begin != r.begin || end != r.end; } + bool contains(const KeyRef& key) const { return begin <= key && key < end; } + bool contains(const KeyRangeRef& keys) const { return begin <= keys.begin && keys.end <= end; } + bool intersects(const KeyRangeRef& keys) const { return begin < keys.end && keys.begin < end; } + bool empty() const { return begin == end; } + + Standalone withPrefix(const StringRef& prefix) const { + return KeyRangeRef(begin.withPrefix(prefix), end.withPrefix(prefix)); + } + + const KeyRangeRef& operator=(const KeyRangeRef& rhs) { + const_cast(begin) = rhs.begin; + const_cast(end) = rhs.end; + return *this; + } + + int expectedSize() const { return begin.expectedSize() + end.expectedSize(); } + + template + force_inline void serialize(Ar& ar) { + serializer(ar, const_cast(begin), const_cast(end)); + if (begin > end) { + throw inverted_range(); + }; + } + + struct ArbitraryOrder { + bool operator()(KeyRangeRef const& a, KeyRangeRef const& b) const { + if (a.begin < b.begin) + return true; + if (a.begin > b.begin) + return false; + return a.end < b.end; + } + }; +}; + +inline KeyRangeRef operator&(const KeyRangeRef& lhs, const KeyRangeRef& rhs) { + KeyRef b = std::max(lhs.begin, rhs.begin), e = std::min(lhs.end, rhs.end); + if (e < b) + return KeyRangeRef(); + return KeyRangeRef(b, e); +} + +typedef Standalone KeyRange; + +template +static std::string describe(T const& item) { + return item.toString(); +} +template +static std::string describe(std::map const& items, int max_items = -1) { + if (!items.size()) + return "[no items]"; + + std::string s; + int count = 0; + for (auto it = items.begin(); it != items.end(); it++) { + if (++count > max_items && max_items >= 0) + break; + if (count > 1) + s += ","; + s += describe(it->first) + "=>" + describe(it->second); + } + return s; +} + +template +static std::string describeList(T const& items, int max_items) { + if (!items.size()) + return "[no items]"; + + std::string s; + int count = 0; + for (auto const& item : items) { + if (++count > max_items && max_items >= 0) + break; + if (count > 1) + s += ","; + s += describe(item); + } + return s; +} + +template +static std::string describe(std::vector const& items, int max_items = -1) { + return describeList(items, max_items); +} + +template +static std::string describe(std::set const& items, int max_items = -1) { + return describeList(items, max_items); +} + +template +static std::string describe(std::pair const& pair) { + return "first: " + describe(pair.first) + " second: " + describe(pair.second); +} +} // namespace FDB + #endif /* FDB_LOANER_TYPES_H */ diff --git a/bindings/flow/HighContentionAllocator.actor.cpp b/bindings/flow/HighContentionAllocator.actor.cpp index 79acedee8e..41d3481bcd 100644 --- a/bindings/flow/HighContentionAllocator.actor.cpp +++ b/bindings/flow/HighContentionAllocator.actor.cpp @@ -21,90 +21,90 @@ #include "HighContentionAllocator.h" namespace FDB { - ACTOR Future> _allocate(Reference tr, Subspace counters, Subspace recent){ - state int64_t start = 0; - state int64_t window = 0; +ACTOR Future> _allocate(Reference tr, Subspace counters, Subspace recent) { + state int64_t start = 0; + state int64_t window = 0; + + loop { + FDBStandalone range = wait(tr->getRange(counters.range(), 1, true, true)); + + if (range.size() > 0) { + start = counters.unpack(range[0].key).getInt(0); + } + + state bool windowAdvanced = false; + loop { + // if thread safety is needed, this should be locked { + if (windowAdvanced) { + tr->clear(KeyRangeRef(counters.key(), counters.get(start).key())); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); + tr->clear(KeyRangeRef(recent.key(), recent.get(start).key())); + } + + int64_t inc = 1; + tr->atomicOp(counters.get(start).key(), StringRef((uint8_t*)&inc, 8), FDB_MUTATION_TYPE_ADD); + Future>> countFuture = tr->get(counters.get(start).key(), true); + // } + + Optional> countValue = wait(countFuture); + + int64_t count = 0; + if (countValue.present()) { + if (countValue.get().size() != 8) { + throw invalid_directory_layer_metadata(); + } + count = *(int64_t*)countValue.get().begin(); + } + + window = HighContentionAllocator::windowSize(start); + if (count * 2 < window) { + break; + } + + start += window; + windowAdvanced = true; + } loop { - FDBStandalone range = wait(tr->getRange(counters.range(), 1, true, true)); + state int64_t candidate = deterministicRandom()->randomInt(start, start + window); - if(range.size() > 0) { - start = counters.unpack(range[0].key).getInt(0); + // if thread safety is needed, this should be locked { + state Future> latestCounter = tr->getRange(counters.range(), 1, true, true); + state Future>> candidateValue = tr->get(recent.get(candidate).key()); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); + tr->set(recent.get(candidate).key(), ValueRef()); + // } + + wait(success(latestCounter) && success(candidateValue)); + int64_t currentWindowStart = 0; + if (latestCounter.get().size() > 0) { + currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0); } - state bool windowAdvanced = false; - loop { - // if thread safety is needed, this should be locked { - if(windowAdvanced) { - tr->clear(KeyRangeRef(counters.key(), counters.get(start).key())); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); - tr->clear(KeyRangeRef(recent.key(), recent.get(start).key())); - } - - int64_t inc = 1; - tr->atomicOp(counters.get(start).key(), StringRef((uint8_t*)&inc, 8), FDB_MUTATION_TYPE_ADD); - Future>> countFuture = tr->get(counters.get(start).key(), true); - // } - - Optional> countValue = wait(countFuture); - - int64_t count = 0; - if(countValue.present()) { - if(countValue.get().size() != 8) { - throw invalid_directory_layer_metadata(); - } - count = *(int64_t*)countValue.get().begin(); - } - - window = HighContentionAllocator::windowSize(start); - if(count * 2 < window) { - break; - } - - start += window; - windowAdvanced = true; + if (currentWindowStart > start) { + break; } - loop { - state int64_t candidate = deterministicRandom()->randomInt(start, start + window); - - // if thread safety is needed, this should be locked { - state Future> latestCounter = tr->getRange(counters.range(), 1, true, true); - state Future>> candidateValue = tr->get(recent.get(candidate).key()); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); - tr->set(recent.get(candidate).key(), ValueRef()); - // } - - wait(success(latestCounter) && success(candidateValue)); - int64_t currentWindowStart = 0; - if(latestCounter.get().size() > 0) { - currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0); - } - - if(currentWindowStart > start) { - break; - } - - if(!candidateValue.get().present()) { - tr->addWriteConflictKey(recent.get(candidate).key()); - return Tuple().append(candidate).pack(); - } + if (!candidateValue.get().present()) { + tr->addWriteConflictKey(recent.get(candidate).key()); + return Tuple().append(candidate).pack(); } } } - - Future> HighContentionAllocator::allocate(Reference const& tr) const { - return _allocate(tr, counters, recent); - } - - int64_t HighContentionAllocator::windowSize(int64_t start) { - if (start < 255) { - return 64; - } - if (start < 65535) { - return 1024; - } - - return 8192; - } } + +Future> HighContentionAllocator::allocate(Reference const& tr) const { + return _allocate(tr, counters, recent); +} + +int64_t HighContentionAllocator::windowSize(int64_t start) { + if (start < 255) { + return 64; + } + if (start < 65535) { + return 1024; + } + + return 8192; +} +} // namespace FDB diff --git a/bindings/flow/HighContentionAllocator.h b/bindings/flow/HighContentionAllocator.h index 37e50a7897..bd4ed2cb97 100644 --- a/bindings/flow/HighContentionAllocator.h +++ b/bindings/flow/HighContentionAllocator.h @@ -26,16 +26,17 @@ #include "Subspace.h" namespace FDB { - class HighContentionAllocator { - public: - HighContentionAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {} - Future> allocate(Reference const& tr) const; +class HighContentionAllocator { +public: + HighContentionAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {} + Future> allocate(Reference const& tr) const; - static int64_t windowSize(int64_t start); - private: - Subspace counters; - Subspace recent; - }; -} + static int64_t windowSize(int64_t start); + +private: + Subspace counters; + Subspace recent; +}; +} // namespace FDB #endif diff --git a/bindings/flow/IDirectory.h b/bindings/flow/IDirectory.h old mode 100755 new mode 100644 index 39256b1661..e629b7cf82 --- a/bindings/flow/IDirectory.h +++ b/bindings/flow/IDirectory.h @@ -27,34 +27,46 @@ #include "bindings/flow/fdb_flow.h" namespace FDB { - class DirectoryLayer; - class DirectorySubspace; +class DirectoryLayer; +class DirectorySubspace; - class IDirectory : public ReferenceCounted { - public: - typedef std::vector> Path; +class IDirectory : public ReferenceCounted { +public: + typedef std::vector> Path; - virtual Future> create(Reference const& tr, Path const& path, Standalone const& layer = Standalone(), - Optional> const& prefix = Optional>()) = 0; + virtual Future> create( + Reference const& tr, + Path const& path, + Standalone const& layer = Standalone(), + Optional> const& prefix = Optional>()) = 0; - virtual Future> open(Reference const& tr, Path const& path, Standalone const& layer = Standalone()) = 0; - virtual Future> createOrOpen(Reference const& tr, Path const& path, Standalone const& layer = Standalone()) = 0; + virtual Future> open(Reference const& tr, + Path const& path, + Standalone const& layer = Standalone()) = 0; + virtual Future> createOrOpen( + Reference const& tr, + Path const& path, + Standalone const& layer = Standalone()) = 0; - virtual Future exists(Reference const& tr, Path const& path = Path()) = 0; - virtual Future>> list(Reference const& tr, Path const& path = Path()) = 0; + virtual Future exists(Reference const& tr, Path const& path = Path()) = 0; + virtual Future>> list(Reference const& tr, + Path const& path = Path()) = 0; - virtual Future> move(Reference const& tr, Path const& oldPath, Path const& newPath) = 0; - virtual Future> moveTo(Reference const& tr, Path const& newAbsolutePath) = 0; + virtual Future> move(Reference const& tr, + Path const& oldPath, + Path const& newPath) = 0; + virtual Future> moveTo(Reference const& tr, + Path const& newAbsolutePath) = 0; - virtual Future remove(Reference const& tr, Path const& path = Path()) = 0; - virtual Future removeIfExists(Reference const& tr, Path const& path = Path()) = 0; + virtual Future remove(Reference const& tr, Path const& path = Path()) = 0; + virtual Future removeIfExists(Reference const& tr, Path const& path = Path()) = 0; - virtual Reference getDirectoryLayer() = 0; - virtual const Standalone getLayer() const = 0; - virtual const Path getPath() const = 0; + virtual Reference getDirectoryLayer() = 0; + virtual const Standalone getLayer() const = 0; + virtual const Path getPath() const = 0; - virtual ~IDirectory() {}; - }; -} + virtual ~IDirectory(){}; +}; +} // namespace FDB #endif \ No newline at end of file diff --git a/bindings/flow/Node.actor.cpp b/bindings/flow/Node.actor.cpp old mode 100755 new mode 100644 index 27521b9524..901376aa68 --- a/bindings/flow/Node.actor.cpp +++ b/bindings/flow/Node.actor.cpp @@ -21,50 +21,49 @@ #include "DirectoryLayer.h" namespace FDB { - DirectoryLayer::Node::Node(Reference const& directoryLayer, Optional const& subspace, IDirectory::Path const& path, IDirectory::Path const& targetPath) - : directoryLayer(directoryLayer), - subspace(subspace), - path(path), - targetPath(targetPath), - loadedMetadata(false) - { } +DirectoryLayer::Node::Node(Reference const& directoryLayer, + Optional const& subspace, + IDirectory::Path const& path, + IDirectory::Path const& targetPath) + : directoryLayer(directoryLayer), subspace(subspace), path(path), targetPath(targetPath), loadedMetadata(false) {} - bool DirectoryLayer::Node::exists() const { - return subspace.present(); - } +bool DirectoryLayer::Node::exists() const { + return subspace.present(); +} - ACTOR Future loadMetadata(DirectoryLayer::Node *n, Reference tr) { - if(!n->exists()){ - n->loadedMetadata = true; - return *n; - } - - Optional> layer = wait(tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY))); - - n->layer = layer.present() ? layer.get() : Standalone(); +ACTOR Future loadMetadata(DirectoryLayer::Node* n, Reference tr) { + if (!n->exists()) { n->loadedMetadata = true; - return *n; } - //Calls to loadMetadata must keep the Node alive while the future is outstanding - Future DirectoryLayer::Node::loadMetadata(Reference tr) { - return FDB::loadMetadata(this, tr); - } + Optional> layer = wait(tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY))); - bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const { - ASSERT(loadedMetadata); - return exists() && layer == DirectoryLayer::PARTITION_LAYER && (includeEmptySubpath || targetPath.size() > path.size()); - } + n->layer = layer.present() ? layer.get() : Standalone(); + n->loadedMetadata = true; - IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const { - return Path(targetPath.begin() + path.size(), targetPath.end()); - } - - Reference DirectoryLayer::Node::getContents() const { - ASSERT(exists()); - ASSERT(loadedMetadata); - - return directoryLayer->contentsOfNode(subspace.get(), path, layer); + return *n; } + +// Calls to loadMetadata must keep the Node alive while the future is outstanding +Future DirectoryLayer::Node::loadMetadata(Reference tr) { + return FDB::loadMetadata(this, tr); } + +bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const { + ASSERT(loadedMetadata); + return exists() && layer == DirectoryLayer::PARTITION_LAYER && + (includeEmptySubpath || targetPath.size() > path.size()); +} + +IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const { + return Path(targetPath.begin() + path.size(), targetPath.end()); +} + +Reference DirectoryLayer::Node::getContents() const { + ASSERT(exists()); + ASSERT(loadedMetadata); + + return directoryLayer->contentsOfNode(subspace.get(), path, layer); +} +} // namespace FDB diff --git a/bindings/flow/Subspace.cpp b/bindings/flow/Subspace.cpp old mode 100755 new mode 100644 index a86ffb25e3..896e67f551 --- a/bindings/flow/Subspace.cpp +++ b/bindings/flow/Subspace.cpp @@ -21,71 +21,72 @@ #include "Subspace.h" namespace FDB { - Subspace::Subspace(Tuple const& tuple, StringRef const& rawPrefix){ - StringRef packed = tuple.pack(); +Subspace::Subspace(Tuple const& tuple, StringRef const& rawPrefix) { + StringRef packed = tuple.pack(); - this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + packed.size()); - this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); - this->rawPrefix.append(this->rawPrefix.arena(), packed.begin(), packed.size()); + this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + packed.size()); + this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); + this->rawPrefix.append(this->rawPrefix.arena(), packed.begin(), packed.size()); +} + +Subspace::Subspace(Tuple const& tuple, Standalone> const& rawPrefix) { + this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + tuple.pack().size()); + this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); + this->rawPrefix.append(this->rawPrefix.arena(), tuple.pack().begin(), tuple.pack().size()); +} + +Subspace::Subspace(StringRef const& rawPrefix) { + this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); +} + +Subspace::~Subspace() {} + +Key Subspace::key() const { + return StringRef(rawPrefix.begin(), rawPrefix.size()); +} + +Key Subspace::pack(const Tuple& tuple) const { + return tuple.pack().withPrefix(StringRef(rawPrefix.begin(), rawPrefix.size())); +} + +Tuple Subspace::unpack(StringRef const& key) const { + if (!contains(key)) { + throw key_not_in_subspace(); } + return Tuple::unpack(key.substr(rawPrefix.size())); +} - Subspace::Subspace(Tuple const& tuple, Standalone> const& rawPrefix) { - this->rawPrefix.reserve(this->rawPrefix.arena(), rawPrefix.size() + tuple.pack().size()); - this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); - this->rawPrefix.append(this->rawPrefix.arena(), tuple.pack().begin(), tuple.pack().size()); - } +KeyRange Subspace::range(Tuple const& tuple) const { + VectorRef begin; + VectorRef end; - Subspace::Subspace(StringRef const& rawPrefix){ - this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); - } + KeyRange keyRange; - Subspace::~Subspace() { } + begin.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1); + begin.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size()); + begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); + begin.push_back(keyRange.arena(), uint8_t('\x00')); - Key Subspace::key() const { - return StringRef(rawPrefix.begin(), rawPrefix.size()); - } + end.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1); + end.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size()); + end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); + end.push_back(keyRange.arena(), uint8_t('\xff')); - Key Subspace::pack(const Tuple& tuple) const { - return tuple.pack().withPrefix(StringRef(rawPrefix.begin(), rawPrefix.size())); - } + // FIXME: test that this uses the keyRange arena and doesn't create another one + keyRange.KeyRangeRef::operator=( + KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size()))); + return keyRange; +} - Tuple Subspace::unpack(StringRef const& key) const { - if (!contains(key)) { - throw key_not_in_subspace(); - } - return Tuple::unpack(key.substr(rawPrefix.size())); - } +bool Subspace::contains(KeyRef const& key) const { + return key.startsWith(StringRef(rawPrefix.begin(), rawPrefix.size())); +} - KeyRange Subspace::range(Tuple const& tuple) const { - VectorRef begin; - VectorRef end; +Subspace Subspace::subspace(Tuple const& tuple) const { + return Subspace(tuple, rawPrefix); +} - KeyRange keyRange; - - begin.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1); - begin.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size()); - begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); - begin.push_back(keyRange.arena(), uint8_t('\x00')); - - end.reserve(keyRange.arena(), rawPrefix.size() + tuple.pack().size() + 1); - end.append(keyRange.arena(), rawPrefix.begin(), rawPrefix.size()); - end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); - end.push_back(keyRange.arena(), uint8_t('\xff')); - - // FIXME: test that this uses the keyRange arena and doesn't create another one - keyRange.KeyRangeRef::operator=(KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size()))); - return keyRange; - } - - bool Subspace::contains(KeyRef const& key) const { - return key.startsWith(StringRef(rawPrefix.begin(), rawPrefix.size())); - } - - Subspace Subspace::subspace(Tuple const& tuple) const { - return Subspace(tuple, rawPrefix); - } - - Subspace Subspace::get(Tuple const& tuple) const { - return subspace(tuple); - } -} \ No newline at end of file +Subspace Subspace::get(Tuple const& tuple) const { + return subspace(tuple); +} +} // namespace FDB \ No newline at end of file diff --git a/bindings/flow/Subspace.h b/bindings/flow/Subspace.h old mode 100755 new mode 100644 index ffd22ac02c..bdbf0e4d04 --- a/bindings/flow/Subspace.h +++ b/bindings/flow/Subspace.h @@ -28,65 +28,65 @@ #include "Tuple.h" namespace FDB { - class Subspace { - public: - Subspace(Tuple const& tuple = Tuple(), StringRef const& rawPrefix = StringRef()); - Subspace(StringRef const& rawPrefix); +class Subspace { +public: + Subspace(Tuple const& tuple = Tuple(), StringRef const& rawPrefix = StringRef()); + Subspace(StringRef const& rawPrefix); - virtual ~Subspace(); + virtual ~Subspace(); - virtual Key key() const; - virtual bool contains(KeyRef const& key) const; + virtual Key key() const; + virtual bool contains(KeyRef const& key) const; - virtual Key pack(Tuple const& tuple = Tuple()) const; - virtual Tuple unpack(KeyRef const& key) const; - virtual KeyRange range(Tuple const& tuple = Tuple()) const; + virtual Key pack(Tuple const& tuple = Tuple()) const; + virtual Tuple unpack(KeyRef const& key) const; + virtual KeyRange range(Tuple const& tuple = Tuple()) const; - template - Key pack(T const& item) const { - Tuple t; - t.append(item); - return pack(t); - } + template + Key pack(T const& item) const { + Tuple t; + t.append(item); + return pack(t); + } - Key packNested(Tuple const& item) const { - Tuple t; - t.appendNested(item); - return pack(t); - } + Key packNested(Tuple const& item) const { + Tuple t; + t.appendNested(item); + return pack(t); + } - Key pack(StringRef const& item, bool utf8=false) const { - Tuple t; - t.append(item, utf8); - return pack(t); - } + Key pack(StringRef const& item, bool utf8 = false) const { + Tuple t; + t.append(item, utf8); + return pack(t); + } - virtual Subspace subspace(Tuple const& tuple) const; - virtual Subspace get(Tuple const& tuple) const; + virtual Subspace subspace(Tuple const& tuple) const; + virtual Subspace get(Tuple const& tuple) const; - template - Subspace get(T const& item) const { - Tuple t; - t.append(item); - return get(t); - } + template + Subspace get(T const& item) const { + Tuple t; + t.append(item); + return get(t); + } - Subspace getNested(Tuple const& item) const { - Tuple t; - t.appendNested(item); - return get(t); - } + Subspace getNested(Tuple const& item) const { + Tuple t; + t.appendNested(item); + return get(t); + } - Subspace get(StringRef const& item, bool utf8=false) const { - Tuple t; - t.append(item, utf8); - return get(t); - } + Subspace get(StringRef const& item, bool utf8 = false) const { + Tuple t; + t.append(item, utf8); + return get(t); + } - private: - Subspace(Tuple const& tuple, Standalone> const& rawPrefix); - Standalone> rawPrefix; - }; -} +private: + Subspace(Tuple const& tuple, Standalone> const& rawPrefix); + Standalone> rawPrefix; +}; +} // namespace FDB #endif \ No newline at end of file diff --git a/bindings/flow/Tuple.cpp b/bindings/flow/Tuple.cpp old mode 100755 new mode 100644 index e81f0529b6..bef7775cca --- a/bindings/flow/Tuple.cpp +++ b/bindings/flow/Tuple.cpp @@ -22,623 +22,602 @@ #include namespace FDB { - // The floating point operations depend on this using the IEEE 754 standard. - BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); - BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); +// The floating point operations depend on this using the IEEE 754 standard. +BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); +BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); - const size_t Uuid::SIZE = 16; +const size_t Uuid::SIZE = 16; - const uint8_t Tuple::NULL_CODE = 0x00; - const uint8_t Tuple::BYTES_CODE = 0x01; - const uint8_t Tuple::STRING_CODE = 0x02; - const uint8_t Tuple::NESTED_CODE = 0x05; - const uint8_t Tuple::INT_ZERO_CODE = 0x14; - const uint8_t Tuple::POS_INT_END = 0x1c; - const uint8_t Tuple::NEG_INT_START = 0x0c; - const uint8_t Tuple::FLOAT_CODE = 0x20; - const uint8_t Tuple::DOUBLE_CODE = 0x21; - const uint8_t Tuple::FALSE_CODE = 0x26; - const uint8_t Tuple::TRUE_CODE = 0x27; - const uint8_t Tuple::UUID_CODE = 0x30; +const uint8_t Tuple::NULL_CODE = 0x00; +const uint8_t Tuple::BYTES_CODE = 0x01; +const uint8_t Tuple::STRING_CODE = 0x02; +const uint8_t Tuple::NESTED_CODE = 0x05; +const uint8_t Tuple::INT_ZERO_CODE = 0x14; +const uint8_t Tuple::POS_INT_END = 0x1c; +const uint8_t Tuple::NEG_INT_START = 0x0c; +const uint8_t Tuple::FLOAT_CODE = 0x20; +const uint8_t Tuple::DOUBLE_CODE = 0x21; +const uint8_t Tuple::FALSE_CODE = 0x26; +const uint8_t Tuple::TRUE_CODE = 0x27; +const uint8_t Tuple::UUID_CODE = 0x30; - static float bigEndianFloat(float orig) { - int32_t big = *(int32_t*)&orig; - big = bigEndian32(big); - return *(float*)&big; +static float bigEndianFloat(float orig) { + int32_t big = *(int32_t*)&orig; + big = bigEndian32(big); + return *(float*)&big; +} + +static double bigEndianDouble(double orig) { + int64_t big = *(int64_t*)&orig; + big = bigEndian64(big); + return *(double*)&big; +} + +static size_t find_string_terminator(const StringRef data, size_t offset) { + size_t i = offset; + while (i < data.size() - 1 && !(data[i] == (uint8_t)'\x00' && data[i + 1] != (uint8_t)'\xff')) { + i += (data[i] == '\x00' ? 2 : 1); } - static double bigEndianDouble(double orig) { - int64_t big = *(int64_t*)&orig; - big = bigEndian64(big); - return *(double*)&big; + return i; +} + +static size_t find_string_terminator(const Standalone> data, size_t offset) { + size_t i = offset; + while (i < data.size() - 1 && !(data[i] == '\x00' && data[i + 1] != (uint8_t)'\xff')) { + i += (data[i] == '\x00' ? 2 : 1); } - static size_t find_string_terminator(const StringRef data, size_t offset) { - size_t i = offset; - while (i < data.size() - 1 && !(data[i] == (uint8_t)'\x00' && data[i+1] != (uint8_t)'\xff')) { - i += (data[i] == '\x00' ? 2 : 1); + return i; +} + +// If encoding and the sign bit is 1 (the number is negative), flip all the bits. +// If decoding and the sign bit is 0 (the number is negative), flip all the bits. +// Otherwise, the number is positive, so flip the sign bit. +static void adjust_floating_point(uint8_t* bytes, size_t size, bool encode) { + if ((encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x00)) || + (!encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x80))) { + for (size_t i = 0; i < size; i++) { + bytes[i] ^= (uint8_t)0xff; } - - return i; + } else { + bytes[0] ^= (uint8_t)0x80; } +} - static size_t find_string_terminator(const Standalone > data, size_t offset ) { - size_t i = offset; - while (i < data.size() - 1 && !(data[i] == '\x00' && data[i+1] != (uint8_t)'\xff')) { - i += (data[i] == '\x00' ? 2 : 1); - } +Tuple::Tuple(StringRef const& str) { + data.append(data.arena(), str.begin(), str.size()); - return i; - } + size_t i = 0; + int depth = 0; + while (i < data.size()) { + if (depth == 0) + offsets.push_back(i); - // If encoding and the sign bit is 1 (the number is negative), flip all the bits. - // If decoding and the sign bit is 0 (the number is negative), flip all the bits. - // Otherwise, the number is positive, so flip the sign bit. - static void adjust_floating_point(uint8_t *bytes, size_t size, bool encode) { - if((encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x00)) || (!encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x80))) { - for(size_t i = 0; i < size; i++) { - bytes[i] ^= (uint8_t)0xff; + if (depth > 0 && data[i] == NULL_CODE) { + if (i + 1 < data.size() && data[i + 1] == 0xff) { + // NULL value. + i += 2; + } else { + // Nested terminator. + i += 1; + depth -= 1; } + } else if (data[i] == BYTES_CODE || data[i] == STRING_CODE) { + i = find_string_terminator(str, i + 1) + 1; + } else if (data[i] >= NEG_INT_START && data[i] <= POS_INT_END) { + i += abs(data[i] - INT_ZERO_CODE) + 1; + } else if (data[i] == NULL_CODE || data[i] == TRUE_CODE || data[i] == FALSE_CODE) { + i += 1; + } else if (data[i] == UUID_CODE) { + i += Uuid::SIZE + 1; + } else if (data[i] == FLOAT_CODE) { + i += sizeof(float) + 1; + } else if (data[i] == DOUBLE_CODE) { + i += sizeof(double) + 1; + } else if (data[i] == NESTED_CODE) { + i += 1; + depth += 1; } else { - bytes[0] ^= (uint8_t)0x80; + throw invalid_tuple_data_type(); } } - Tuple::Tuple(StringRef const& str) { - data.append(data.arena(), str.begin(), str.size()); + if (depth != 0) { + throw invalid_tuple_data_type(); + } +} - size_t i = 0; - int depth = 0; - while(i < data.size()) { - if(depth == 0) offsets.push_back(i); +// Note: this is destructive of the original offsets, so should only +// be used once we are done. +Tuple::Tuple(Standalone> data, std::vector offsets) { + this->data = data; + this->offsets = std::move(offsets); +} - if(depth > 0 && data[i] == NULL_CODE) { - if(i + 1 < data.size() && data[i+1] == 0xff) { - // NULL value. +Tuple Tuple::unpack(StringRef const& str) { + return Tuple(str); +} + +Tuple& Tuple::append(Tuple const& tuple) { + for (size_t offset : tuple.offsets) { + offsets.push_back(offset + data.size()); + } + + data.append(data.arena(), tuple.data.begin(), tuple.data.size()); + + return *this; +} + +Tuple& Tuple::append(StringRef const& str, bool utf8) { + offsets.push_back(data.size()); + + const uint8_t utfChar = utf8 ? STRING_CODE : BYTES_CODE; + data.append(data.arena(), &utfChar, 1); + + size_t lastPos = 0; + for (size_t pos = 0; pos < str.size(); ++pos) { + if (str[pos] == '\x00') { + data.append(data.arena(), str.begin() + lastPos, pos - lastPos); + data.push_back(data.arena(), (uint8_t)'\x00'); + data.push_back(data.arena(), (uint8_t)'\xff'); + lastPos = pos + 1; + } + } + + data.append(data.arena(), str.begin() + lastPos, str.size() - lastPos); + data.push_back(data.arena(), (uint8_t)'\x00'); + + return *this; +} + +Tuple& Tuple::append(int32_t value) { + return append((int64_t)value); +} + +Tuple& Tuple::append(int64_t value) { + uint64_t swap = value; + bool neg = false; + + offsets.push_back(data.size()); + + if (value < 0) { + value = ~(-value); + neg = true; + } + + swap = bigEndian64(value); + + for (int i = 0; i < 8; i++) { + if (((uint8_t*)&swap)[i] != (neg ? 255 : 0)) { + data.push_back(data.arena(), (uint8_t)(INT_ZERO_CODE + (8 - i) * (neg ? -1 : 1))); + data.append(data.arena(), ((const uint8_t*)&swap) + i, 8 - i); + return *this; + } + } + + data.push_back(data.arena(), INT_ZERO_CODE); + return *this; +} + +Tuple& Tuple::append(bool value) { + offsets.push_back(data.size()); + if (value) { + data.push_back(data.arena(), TRUE_CODE); + } else { + data.push_back(data.arena(), FALSE_CODE); + } + return *this; +} + +Tuple& Tuple::append(float value) { + offsets.push_back(data.size()); + float swap = bigEndianFloat(value); + uint8_t* bytes = (uint8_t*)&swap; + adjust_floating_point(bytes, sizeof(float), true); + + data.push_back(data.arena(), FLOAT_CODE); + data.append(data.arena(), bytes, sizeof(float)); + return *this; +} + +Tuple& Tuple::append(double value) { + offsets.push_back(data.size()); + double swap = value; + swap = bigEndianDouble(swap); + uint8_t* bytes = (uint8_t*)&swap; + adjust_floating_point(bytes, sizeof(double), true); + + data.push_back(data.arena(), DOUBLE_CODE); + data.append(data.arena(), bytes, sizeof(double)); + return *this; +} + +Tuple& Tuple::append(Uuid value) { + offsets.push_back(data.size()); + data.push_back(data.arena(), UUID_CODE); + data.append(data.arena(), value.getData().begin(), Uuid::SIZE); + return *this; +} + +Tuple& Tuple::appendNested(Tuple const& value) { + offsets.push_back(data.size()); + data.push_back(data.arena(), NESTED_CODE); + + for (size_t i = 0; i < value.size(); i++) { + size_t offset = value.offsets[i]; + size_t next_offset = (i + 1 < value.offsets.size() ? value.offsets[i + 1] : value.data.size()); + ASSERT_LT(offset, value.data.size()); + ASSERT_LE(next_offset, value.data.size()); + uint8_t code = value.data[offset]; + if (code == NULL_CODE) { + data.push_back(data.arena(), NULL_CODE); + data.push_back(data.arena(), 0xff); + } else { + data.append(data.arena(), value.data.begin() + offset, next_offset - offset); + } + } + + data.push_back(data.arena(), (uint8_t)'\x00'); + + return *this; +} + +Tuple& Tuple::appendNull() { + offsets.push_back(data.size()); + data.push_back(data.arena(), NULL_CODE); + return *this; +} + +Tuple::ElementType Tuple::getType(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + + uint8_t code = data[offsets[index]]; + + if (code == NULL_CODE) { + return ElementType::NULL_TYPE; + } else if (code == BYTES_CODE) { + return ElementType::BYTES; + } else if (code == STRING_CODE) { + return ElementType::UTF8; + } else if (code == NESTED_CODE) { + return ElementType::NESTED; + } else if (code >= NEG_INT_START && code <= POS_INT_END) { + return ElementType::INT; + } else if (code == FLOAT_CODE) { + return ElementType::FLOAT; + } else if (code == DOUBLE_CODE) { + return ElementType::DOUBLE; + } else if (code == FALSE_CODE || code == TRUE_CODE) { + return ElementType::BOOL; + } else if (code == UUID_CODE) { + return ElementType::UUID; + } else { + throw invalid_tuple_data_type(); + } +} + +Standalone Tuple::getString(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + + uint8_t code = data[offsets[index]]; + if (code != BYTES_CODE && code != STRING_CODE) { + throw invalid_tuple_data_type(); + } + + size_t b = offsets[index] + 1; + size_t e; + if (offsets.size() > index + 1) { + e = offsets[index + 1]; + } else { + e = data.size(); + } + + Standalone result; + VectorRef staging; + + for (size_t i = b; i < e; ++i) { + if (data[i] == '\x00') { + staging.append(result.arena(), data.begin() + b, i - b); + ++i; + b = i + 1; + + if (i < e) { + staging.push_back(result.arena(), '\x00'); + } + } + } + + if (b < e) { + staging.append(result.arena(), data.begin() + b, e - b); + } + + result.StringRef::operator=(StringRef(staging.begin(), staging.size())); + return result; +} + +int64_t Tuple::getInt(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + + int64_t swap; + bool neg = false; + + ASSERT_LT(offsets[index], data.size()); + uint8_t code = data[offsets[index]]; + if (code < NEG_INT_START || code > POS_INT_END) { + throw invalid_tuple_data_type(); + } + + int8_t len = code - INT_ZERO_CODE; + + if (len < 0) { + len = -len; + neg = true; + } + + memset(&swap, neg ? '\xff' : 0, 8 - len); + memcpy(((uint8_t*)&swap) + 8 - len, data.begin() + offsets[index] + 1, len); + + swap = bigEndian64(swap); + + if (neg) { + swap = -(~swap); + } + + return swap; +} + +bool Tuple::getBool(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + ASSERT_LT(offsets[index], data.size()); + uint8_t code = data[offsets[index]]; + if (code == FALSE_CODE) { + return false; + } else if (code == TRUE_CODE) { + return true; + } else { + throw invalid_tuple_data_type(); + } +} + +float Tuple::getFloat(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + ASSERT_LT(offsets[index], data.size()); + uint8_t code = data[offsets[index]]; + if (code != FLOAT_CODE) { + throw invalid_tuple_data_type(); + } + + float swap; + uint8_t* bytes = (uint8_t*)&swap; + ASSERT_LE(offsets[index] + 1 + sizeof(float), data.size()); + swap = *(float*)(data.begin() + offsets[index] + 1); + adjust_floating_point(bytes, sizeof(float), false); + + return bigEndianFloat(swap); +} + +double Tuple::getDouble(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + ASSERT_LT(offsets[index], data.size()); + uint8_t code = data[offsets[index]]; + if (code != DOUBLE_CODE) { + throw invalid_tuple_data_type(); + } + + double swap; + uint8_t* bytes = (uint8_t*)&swap; + ASSERT_LE(offsets[index] + 1 + sizeof(double), data.size()); + swap = *(double*)(data.begin() + offsets[index] + 1); + adjust_floating_point(bytes, sizeof(double), false); + + return bigEndianDouble(swap); +} + +Uuid Tuple::getUuid(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + size_t offset = offsets[index]; + ASSERT_LT(offset, data.size()); + uint8_t code = data[offset]; + if (code != UUID_CODE) { + throw invalid_tuple_data_type(); + } + ASSERT_LE(offset + Uuid::SIZE + 1, data.size()); + StringRef uuidData(data.begin() + offset + 1, Uuid::SIZE); + return Uuid(uuidData); +} + +Tuple Tuple::getNested(size_t index) const { + if (index >= offsets.size()) { + throw invalid_tuple_index(); + } + size_t offset = offsets[index]; + ASSERT_LT(offset, data.size()); + uint8_t code = data[offset]; + if (code != NESTED_CODE) { + throw invalid_tuple_data_type(); + } + + size_t next_offset = (index + 1 < offsets.size() ? offsets[index + 1] : data.size()); + ASSERT_LE(next_offset, data.size()); + ASSERT_EQ(data[next_offset - 1], (uint8_t)0x00); + Standalone> dest; + dest.reserve(dest.arena(), next_offset - offset); + std::vector dest_offsets; + + size_t i = offset + 1; + int depth = 0; + while (i < next_offset - 1) { + if (depth == 0) + dest_offsets.push_back(dest.size()); + uint8_t code = data[i]; + dest.push_back(dest.arena(), code); // Copy over the type code. + if (code == NULL_CODE) { + if (depth > 0) { + if (i + 1 < next_offset - 1 && data[i + 1] == 0xff) { + // Null with a tuple nested in the nested tuple. + dest.push_back(dest.arena(), 0xff); i += 2; } else { // Nested terminator. - i += 1; depth -= 1; + i += 1; } - } - else if(data[i] == BYTES_CODE || data[i] == STRING_CODE) { - i = find_string_terminator(str, i+1) + 1; - } - else if(data[i] >= NEG_INT_START && data[i] <= POS_INT_END) { - i += abs(data[i] - INT_ZERO_CODE) + 1; - } - else if(data[i] == NULL_CODE || data[i] == TRUE_CODE || data[i] == FALSE_CODE) { - i += 1; - } - else if(data[i] == UUID_CODE) { - i += Uuid::SIZE + 1; - } - else if(data[i] == FLOAT_CODE) { - i += sizeof(float) + 1; - } - else if(data[i] == DOUBLE_CODE) { - i += sizeof(double) + 1; - } - else if(data[i] == NESTED_CODE) { - i += 1; - depth += 1; - } - else { - throw invalid_tuple_data_type(); - } - } - - if(depth != 0) { - throw invalid_tuple_data_type(); - } - } - - // Note: this is destructive of the original offsets, so should only - // be used once we are done. - Tuple::Tuple(Standalone> data, std::vector offsets) { - this->data = data; - this->offsets = std::move(offsets); - } - - Tuple Tuple::unpack(StringRef const& str) { - return Tuple(str); - } - - Tuple& Tuple::append(Tuple const& tuple) { - for(size_t offset : tuple.offsets) { - offsets.push_back(offset + data.size()); - } - - data.append(data.arena(), tuple.data.begin(), tuple.data.size()); - - return *this; - } - - Tuple& Tuple::append(StringRef const& str, bool utf8) { - offsets.push_back(data.size()); - - const uint8_t utfChar = utf8 ? STRING_CODE : BYTES_CODE; - data.append(data.arena(), &utfChar, 1); - - size_t lastPos = 0; - for(size_t pos = 0; pos < str.size(); ++pos) { - if(str[pos] == '\x00') { - data.append(data.arena(), str.begin() + lastPos, pos - lastPos); - data.push_back(data.arena(), (uint8_t)'\x00'); - data.push_back(data.arena(), (uint8_t)'\xff'); - lastPos = pos + 1; - } - } - - data.append(data.arena(), str.begin() + lastPos, str.size() - lastPos); - data.push_back(data.arena(), (uint8_t)'\x00'); - - return *this; - } - - Tuple& Tuple::append( int32_t value ) { - return append((int64_t)value); - } - - Tuple& Tuple::append( int64_t value ) { - uint64_t swap = value; - bool neg = false; - - offsets.push_back( data.size() ); - - if ( value < 0 ) { - value = ~(-value); - neg = true; - } - - swap = bigEndian64(value); - - for ( int i = 0; i < 8; i++ ) { - if ( ((uint8_t*)&swap)[i] != (neg ? 255 : 0) ) { - data.push_back( data.arena(), (uint8_t)(INT_ZERO_CODE + (8-i) * (neg ? -1 : 1)) ); - data.append( data.arena(), ((const uint8_t *)&swap) + i, 8 - i ); - return *this; - } - } - - data.push_back( data.arena(), INT_ZERO_CODE ); - return *this; - } - - Tuple& Tuple::append( bool value ) { - offsets.push_back( data.size() ); - if(value) { - data.push_back( data.arena(), TRUE_CODE ); - } else { - data.push_back( data.arena(), FALSE_CODE ); - } - return *this; - } - - Tuple& Tuple::append( float value ) { - offsets.push_back( data.size() ); - float swap = bigEndianFloat(value); - uint8_t *bytes = (uint8_t*)&swap; - adjust_floating_point(bytes, sizeof(float), true); - - data.push_back( data.arena(), FLOAT_CODE ); - data.append( data.arena(), bytes, sizeof(float) ); - return *this; - } - - Tuple& Tuple::append( double value ) { - offsets.push_back( data.size() ); - double swap = value; - swap = bigEndianDouble(swap); - uint8_t *bytes = (uint8_t*)&swap; - adjust_floating_point(bytes, sizeof(double), true); - - data.push_back( data.arena(), DOUBLE_CODE ); - data.append( data.arena(), bytes, sizeof(double) ); - return *this; - } - - Tuple& Tuple::append( Uuid value ) { - offsets.push_back( data.size() ); - data.push_back( data.arena(), UUID_CODE ); - data.append( data.arena(), value.getData().begin(), Uuid::SIZE ); - return *this; - } - - Tuple& Tuple::appendNested( Tuple const& value ) { - offsets.push_back( data.size() ); - data.push_back( data.arena(), NESTED_CODE ); - - for(size_t i = 0; i < value.size(); i++) { - size_t offset = value.offsets[i]; - size_t next_offset = (i+1 < value.offsets.size() ? value.offsets[i+1] : value.data.size()); - ASSERT_LT(offset, value.data.size()); - ASSERT_LE(next_offset, value.data.size()); - uint8_t code = value.data[offset]; - if(code == NULL_CODE) { - data.push_back( data.arena(), NULL_CODE ); - data.push_back( data.arena(), 0xff ); } else { - data.append( data.arena(), value.data.begin() + offset, next_offset - offset); - } - } - - data.push_back( data.arena(), (uint8_t)'\x00'); - - return *this; - } - - Tuple& Tuple::appendNull() { - offsets.push_back(data.size()); - data.push_back(data.arena(), NULL_CODE); - return *this; - } - - Tuple::ElementType Tuple::getType(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - - uint8_t code = data[offsets[index]]; - - if(code == NULL_CODE) { - return ElementType::NULL_TYPE; - } - else if(code == BYTES_CODE) { - return ElementType::BYTES; - } - else if(code == STRING_CODE) { - return ElementType::UTF8; - } - else if(code == NESTED_CODE) { - return ElementType::NESTED; - } - else if(code >= NEG_INT_START && code <= POS_INT_END) { - return ElementType::INT; - } - else if(code == FLOAT_CODE) { - return ElementType::FLOAT; - } - else if(code == DOUBLE_CODE) { - return ElementType::DOUBLE; - } - else if(code == FALSE_CODE || code == TRUE_CODE) { - return ElementType::BOOL; - } - else if(code == UUID_CODE) { - return ElementType::UUID; - } - else { - throw invalid_tuple_data_type(); - } - } - - Standalone Tuple::getString(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - - uint8_t code = data[offsets[index]]; - if(code != BYTES_CODE && code != STRING_CODE) { - throw invalid_tuple_data_type(); - } - - size_t b = offsets[index] + 1; - size_t e; - if (offsets.size() > index + 1) { - e = offsets[index+1]; - } else { - e = data.size(); - } - - Standalone result; - VectorRef staging; - - for (size_t i = b; i < e; ++i) { - if(data[i] == '\x00') { - staging.append(result.arena(), data.begin() + b, i - b); - ++i; - b = i + 1; - - if(i < e) { - staging.push_back(result.arena(), '\x00'); - } - } - } - - if(b < e) { - staging.append(result.arena(), data.begin() + b, e - b); - } - - result.StringRef::operator=(StringRef(staging.begin(), staging.size())); - return result; - } - - int64_t Tuple::getInt(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - - int64_t swap; - bool neg = false; - - ASSERT_LT(offsets[index], data.size()); - uint8_t code = data[offsets[index]]; - if(code < NEG_INT_START || code > POS_INT_END) { - throw invalid_tuple_data_type(); - } - - int8_t len = code - INT_ZERO_CODE; - - if ( len < 0 ) { - len = -len; - neg = true; - } - - memset( &swap, neg ? '\xff' : 0, 8 - len ); - memcpy( ((uint8_t*)&swap) + 8 - len, data.begin() + offsets[index] + 1, len ); - - swap = bigEndian64( swap ); - - if ( neg ) { - swap = -(~swap); - } - - return swap; - } - - bool Tuple::getBool(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - ASSERT_LT(offsets[index], data.size()); - uint8_t code = data[offsets[index]]; - if(code == FALSE_CODE) { - return false; - } else if(code == TRUE_CODE) { - return true; - } else { - throw invalid_tuple_data_type(); - } - } - - float Tuple::getFloat(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - ASSERT_LT(offsets[index], data.size()); - uint8_t code = data[offsets[index]]; - if(code != FLOAT_CODE) { - throw invalid_tuple_data_type(); - } - - float swap; - uint8_t* bytes = (uint8_t*)&swap; - ASSERT_LE(offsets[index] + 1 + sizeof(float), data.size()); - swap = *(float*)(data.begin() + offsets[index] + 1); - adjust_floating_point( bytes, sizeof(float), false ); - - return bigEndianFloat(swap); - } - - double Tuple::getDouble(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - ASSERT_LT(offsets[index], data.size()); - uint8_t code = data[offsets[index]]; - if(code != DOUBLE_CODE) { - throw invalid_tuple_data_type(); - } - - double swap; - uint8_t* bytes = (uint8_t*)&swap; - ASSERT_LE(offsets[index] + 1 + sizeof(double), data.size()); - swap = *(double*)(data.begin() + offsets[index] + 1); - adjust_floating_point( bytes, sizeof(double), false ); - - return bigEndianDouble(swap); - } - - Uuid Tuple::getUuid(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - size_t offset = offsets[index]; - ASSERT_LT(offset, data.size()); - uint8_t code = data[offset]; - if(code != UUID_CODE) { - throw invalid_tuple_data_type(); - } - ASSERT_LE(offset + Uuid::SIZE + 1, data.size()); - StringRef uuidData(data.begin() + offset + 1, Uuid::SIZE); - return Uuid(uuidData); - } - - Tuple Tuple::getNested(size_t index) const { - if(index >= offsets.size()) { - throw invalid_tuple_index(); - } - size_t offset = offsets[index]; - ASSERT_LT(offset, data.size()); - uint8_t code = data[offset]; - if(code != NESTED_CODE) { - throw invalid_tuple_data_type(); - } - - size_t next_offset = (index + 1 < offsets.size() ? offsets[index+1] : data.size()); - ASSERT_LE(next_offset, data.size()); - ASSERT_EQ(data[next_offset - 1], (uint8_t)0x00); - Standalone> dest; - dest.reserve(dest.arena(), next_offset - offset); - std::vector dest_offsets; - - size_t i = offset + 1; - int depth = 0; - while(i < next_offset - 1) { - if (depth == 0) dest_offsets.push_back(dest.size()); - uint8_t code = data[i]; - dest.push_back(dest.arena(), code); // Copy over the type code. - if(code == NULL_CODE) { - if(depth > 0) { - if(i + 1 < next_offset - 1 && data[i+1] == 0xff) { - // Null with a tuple nested in the nested tuple. - dest.push_back(dest.arena(), 0xff); - i += 2; - } else { - // Nested terminator. - depth -= 1; - i += 1; - } - } else { - // A null object within the nested tuple. - ASSERT_LT(i + 1, next_offset - 1); - ASSERT_EQ(data[i+1], 0xff); - i += 2; - } - } - else if(code == BYTES_CODE || code == STRING_CODE) { - size_t next_i = find_string_terminator(data, i+1) + 1; - ASSERT_LE(next_i, next_offset - 1); - size_t length = next_i - i - 1; - dest.append(dest.arena(), data.begin() + i + 1, length); - i = next_i; - } - else if(code >= NEG_INT_START && code <= POS_INT_END) { - size_t int_size = abs(code - INT_ZERO_CODE); - ASSERT_LE(i + int_size, next_offset - 1); - dest.append(dest.arena(), data.begin() + i + 1, int_size); - i += int_size + 1; - } - else if(code == TRUE_CODE || code == FALSE_CODE) { - i += 1; - } - else if(code == UUID_CODE) { - ASSERT_LE(i + 1 + Uuid::SIZE, next_offset - 1); - dest.append(dest.arena(), data.begin() + i + 1, Uuid::SIZE); - i += Uuid::SIZE + 1; - } - else if(code == FLOAT_CODE) { - ASSERT_LE(i + 1 + sizeof(float), next_offset - 1); - dest.append(dest.arena(), data.begin() + i + 1, sizeof(float)); - i += sizeof(float) + 1; - } - else if(code == DOUBLE_CODE) { - ASSERT_LE(i + 1 + sizeof(double), next_offset - 1); - dest.append(dest.arena(), data.begin() + i + 1, sizeof(double)); - i += sizeof(double) + 1; - } - else if(code == NESTED_CODE) { - i += 1; - depth += 1; - } - else { - throw invalid_tuple_data_type(); - } - } - - // The item may shrink because of escaped nulls that are unespaced. - return Tuple(dest, dest_offsets); - } - - KeyRange Tuple::range(Tuple const& tuple) const { - VectorRef begin; - VectorRef end; - - KeyRange keyRange; - - begin.reserve(keyRange.arena(), data.size() + tuple.pack().size() + 1); - begin.append(keyRange.arena(), data.begin(), data.size()); - begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); - begin.push_back(keyRange.arena(), uint8_t('\x00')); - - end.reserve(keyRange.arena(), data.size() + tuple.pack().size() + 1); - end.append(keyRange.arena(), data.begin(), data.size()); - end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); - end.push_back(keyRange.arena(), uint8_t('\xff')); - - keyRange.KeyRangeRef::operator=(KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size()))); - return keyRange; - } - - Tuple Tuple::subTuple(size_t start, size_t end) const { - if(start >= offsets.size() || end <= start) { - return Tuple(); - } - - size_t endPos = end < offsets.size() ? offsets[end] : data.size(); - return Tuple(StringRef(data.begin() + offsets[start], endPos - offsets[start])); - } - - // Comparisons - int compare(Standalone > const& v1, Standalone > const& v2) { - size_t i = 0; - while(i < v1.size() && i < v2.size()) { - if(v1[i] < v2[i]) { - return -1; - } else if(v1[i] > v2[i]) { - return 1; + // A null object within the nested tuple. + ASSERT_LT(i + 1, next_offset - 1); + ASSERT_EQ(data[i + 1], 0xff); + i += 2; } + } else if (code == BYTES_CODE || code == STRING_CODE) { + size_t next_i = find_string_terminator(data, i + 1) + 1; + ASSERT_LE(next_i, next_offset - 1); + size_t length = next_i - i - 1; + dest.append(dest.arena(), data.begin() + i + 1, length); + i = next_i; + } else if (code >= NEG_INT_START && code <= POS_INT_END) { + size_t int_size = abs(code - INT_ZERO_CODE); + ASSERT_LE(i + int_size, next_offset - 1); + dest.append(dest.arena(), data.begin() + i + 1, int_size); + i += int_size + 1; + } else if (code == TRUE_CODE || code == FALSE_CODE) { i += 1; + } else if (code == UUID_CODE) { + ASSERT_LE(i + 1 + Uuid::SIZE, next_offset - 1); + dest.append(dest.arena(), data.begin() + i + 1, Uuid::SIZE); + i += Uuid::SIZE + 1; + } else if (code == FLOAT_CODE) { + ASSERT_LE(i + 1 + sizeof(float), next_offset - 1); + dest.append(dest.arena(), data.begin() + i + 1, sizeof(float)); + i += sizeof(float) + 1; + } else if (code == DOUBLE_CODE) { + ASSERT_LE(i + 1 + sizeof(double), next_offset - 1); + dest.append(dest.arena(), data.begin() + i + 1, sizeof(double)); + i += sizeof(double) + 1; + } else if (code == NESTED_CODE) { + i += 1; + depth += 1; + } else { + throw invalid_tuple_data_type(); } + } - if(i < v1.size()) { + // The item may shrink because of escaped nulls that are unespaced. + return Tuple(dest, dest_offsets); +} + +KeyRange Tuple::range(Tuple const& tuple) const { + VectorRef begin; + VectorRef end; + + KeyRange keyRange; + + begin.reserve(keyRange.arena(), data.size() + tuple.pack().size() + 1); + begin.append(keyRange.arena(), data.begin(), data.size()); + begin.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); + begin.push_back(keyRange.arena(), uint8_t('\x00')); + + end.reserve(keyRange.arena(), data.size() + tuple.pack().size() + 1); + end.append(keyRange.arena(), data.begin(), data.size()); + end.append(keyRange.arena(), tuple.pack().begin(), tuple.pack().size()); + end.push_back(keyRange.arena(), uint8_t('\xff')); + + keyRange.KeyRangeRef::operator=( + KeyRangeRef(StringRef(begin.begin(), begin.size()), StringRef(end.begin(), end.size()))); + return keyRange; +} + +Tuple Tuple::subTuple(size_t start, size_t end) const { + if (start >= offsets.size() || end <= start) { + return Tuple(); + } + + size_t endPos = end < offsets.size() ? offsets[end] : data.size(); + return Tuple(StringRef(data.begin() + offsets[start], endPos - offsets[start])); +} + +// Comparisons +int compare(Standalone> const& v1, Standalone> const& v2) { + size_t i = 0; + while (i < v1.size() && i < v2.size()) { + if (v1[i] < v2[i]) { + return -1; + } else if (v1[i] > v2[i]) { return 1; } - if(i < v2.size()) { - return -1; - } - return 0; + i += 1; } - bool Tuple::operator==(Tuple const& other) const { - return compare(data, other.data) == 0; + if (i < v1.size()) { + return 1; } - bool Tuple::operator!=(Tuple const& other) const { - return compare(data, other.data) != 0; - } - bool Tuple::operator<(Tuple const& other) const { - return compare(data, other.data) < 0; - } - bool Tuple::operator<=(Tuple const& other) const { - return compare(data, other.data) <= 0; - } - bool Tuple::operator>(Tuple const& other) const { - return compare(data, other.data) > 0; - } - bool Tuple::operator>=(Tuple const& other) const { - return compare(data, other.data) >= 0; - } - - // UUID implementation - Uuid::Uuid(const StringRef& data) { - if (data.size() != Uuid::SIZE) { - throw invalid_uuid_size(); - } - this->data = data; - } - - StringRef Uuid::getData() const { - return data; - } - - bool Uuid::operator==(Uuid const& other) const { - return data == other.data; - } - bool Uuid::operator!=(Uuid const& other) const { - return data != other.data; - } - bool Uuid::operator<(Uuid const& other) const { - return data < other.data; - } - bool Uuid::operator<=(Uuid const& other) const { - return data <= other.data; - } - bool Uuid::operator>(Uuid const& other) const { - return data > other.data; - } - bool Uuid::operator>=(Uuid const& other) const { - return data >= other.data; + if (i < v2.size()) { + return -1; } + return 0; } + +bool Tuple::operator==(Tuple const& other) const { + return compare(data, other.data) == 0; +} +bool Tuple::operator!=(Tuple const& other) const { + return compare(data, other.data) != 0; +} +bool Tuple::operator<(Tuple const& other) const { + return compare(data, other.data) < 0; +} +bool Tuple::operator<=(Tuple const& other) const { + return compare(data, other.data) <= 0; +} +bool Tuple::operator>(Tuple const& other) const { + return compare(data, other.data) > 0; +} +bool Tuple::operator>=(Tuple const& other) const { + return compare(data, other.data) >= 0; +} + +// UUID implementation +Uuid::Uuid(const StringRef& data) { + if (data.size() != Uuid::SIZE) { + throw invalid_uuid_size(); + } + this->data = data; +} + +StringRef Uuid::getData() const { + return data; +} + +bool Uuid::operator==(Uuid const& other) const { + return data == other.data; +} +bool Uuid::operator!=(Uuid const& other) const { + return data != other.data; +} +bool Uuid::operator<(Uuid const& other) const { + return data < other.data; +} +bool Uuid::operator<=(Uuid const& other) const { + return data <= other.data; +} +bool Uuid::operator>(Uuid const& other) const { + return data > other.data; +} +bool Uuid::operator>=(Uuid const& other) const { + return data >= other.data; +} +} // namespace FDB diff --git a/bindings/flow/Tuple.h b/bindings/flow/Tuple.h old mode 100755 new mode 100644 index 25867ef71f..e26b489c61 --- a/bindings/flow/Tuple.h +++ b/bindings/flow/Tuple.h @@ -26,92 +26,93 @@ #include "bindings/flow/fdb_flow.h" namespace FDB { - struct Uuid { - const static size_t SIZE; +struct Uuid { + const static size_t SIZE; - Uuid(StringRef const& data); + Uuid(StringRef const& data); - StringRef getData() const; + StringRef getData() const; - // Comparisons - bool operator==(Uuid const& other) const; - bool operator!=(Uuid const& other) const; - bool operator<(Uuid const& other) const; - bool operator<=(Uuid const& other) const; - bool operator>(Uuid const& other) const; - bool operator>=(Uuid const& other) const; - private: - Standalone data; - }; + // Comparisons + bool operator==(Uuid const& other) const; + bool operator!=(Uuid const& other) const; + bool operator<(Uuid const& other) const; + bool operator<=(Uuid const& other) const; + bool operator>(Uuid const& other) const; + bool operator>=(Uuid const& other) const; - struct Tuple { - Tuple() {} +private: + Standalone data; +}; - static Tuple unpack(StringRef const& str); +struct Tuple { + Tuple() {} - Tuple& append(Tuple const& tuple); - Tuple& append(StringRef const& str, bool utf8=false); - Tuple& append(int32_t); - Tuple& append(int64_t); - Tuple& append(bool); - Tuple& append(float); - Tuple& append(double); - Tuple& append(Uuid); - Tuple& appendNested(Tuple const&); - Tuple& appendNull(); + static Tuple unpack(StringRef const& str); - StringRef pack() const { return StringRef(data.begin(), data.size()); } + Tuple& append(Tuple const& tuple); + Tuple& append(StringRef const& str, bool utf8 = false); + Tuple& append(int32_t); + Tuple& append(int64_t); + Tuple& append(bool); + Tuple& append(float); + Tuple& append(double); + Tuple& append(Uuid); + Tuple& appendNested(Tuple const&); + Tuple& appendNull(); - template - Tuple& operator<<(T const& t) { - return append(t); - } + StringRef pack() const { return StringRef(data.begin(), data.size()); } - enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED }; + template + Tuple& operator<<(T const& t) { + return append(t); + } - // this is number of elements, not length of data - size_t size() const { return offsets.size(); } + enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED }; - ElementType getType(size_t index) const; - Standalone getString(size_t index) const; - int64_t getInt(size_t index) const; - bool getBool(size_t index) const; - float getFloat(size_t index) const; - double getDouble(size_t index) const; - Uuid getUuid(size_t index) const; - Tuple getNested(size_t index) const; + // this is number of elements, not length of data + size_t size() const { return offsets.size(); } - KeyRange range(Tuple const& tuple = Tuple()) const; + ElementType getType(size_t index) const; + Standalone getString(size_t index) const; + int64_t getInt(size_t index) const; + bool getBool(size_t index) const; + float getFloat(size_t index) const; + double getDouble(size_t index) const; + Uuid getUuid(size_t index) const; + Tuple getNested(size_t index) const; - Tuple subTuple(size_t beginIndex, size_t endIndex = std::numeric_limits::max()) const; + KeyRange range(Tuple const& tuple = Tuple()) const; - // Comparisons - bool operator==(Tuple const& other) const; - bool operator!=(Tuple const& other) const; - bool operator<(Tuple const& other) const; - bool operator<=(Tuple const& other) const; - bool operator>(Tuple const& other) const; - bool operator>=(Tuple const& other) const; + Tuple subTuple(size_t beginIndex, size_t endIndex = std::numeric_limits::max()) const; - private: - static const uint8_t NULL_CODE; - static const uint8_t BYTES_CODE; - static const uint8_t STRING_CODE; - static const uint8_t NESTED_CODE; - static const uint8_t INT_ZERO_CODE; - static const uint8_t POS_INT_END; - static const uint8_t NEG_INT_START; - static const uint8_t FLOAT_CODE; - static const uint8_t DOUBLE_CODE; - static const uint8_t FALSE_CODE; - static const uint8_t TRUE_CODE; - static const uint8_t UUID_CODE; + // Comparisons + bool operator==(Tuple const& other) const; + bool operator!=(Tuple const& other) const; + bool operator<(Tuple const& other) const; + bool operator<=(Tuple const& other) const; + bool operator>(Tuple const& other) const; + bool operator>=(Tuple const& other) const; - Tuple(const StringRef& data); - Tuple(Standalone> data, std::vector offsets); - Standalone> data; - std::vector offsets; - }; -} +private: + static const uint8_t NULL_CODE; + static const uint8_t BYTES_CODE; + static const uint8_t STRING_CODE; + static const uint8_t NESTED_CODE; + static const uint8_t INT_ZERO_CODE; + static const uint8_t POS_INT_END; + static const uint8_t NEG_INT_START; + static const uint8_t FLOAT_CODE; + static const uint8_t DOUBLE_CODE; + static const uint8_t FALSE_CODE; + static const uint8_t TRUE_CODE; + static const uint8_t UUID_CODE; + + Tuple(const StringRef& data); + Tuple(Standalone> data, std::vector offsets); + Standalone> data; + std::vector offsets; +}; +} // namespace FDB #endif /* _FDB_TUPLE_H_ */ diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.actor.cpp index 13e371cc01..659eb5b37a 100644 --- a/bindings/flow/fdb_flow.actor.cpp +++ b/bindings/flow/fdb_flow.actor.cpp @@ -36,41 +36,42 @@ THREAD_FUNC networkThread(void* fdb) { } ACTOR Future _test() { - API *fdb = FDB::API::selectAPIVersion(630); + API* fdb = FDB::API::selectAPIVersion(630); auto db = fdb->createDatabase(); state Reference tr = db->createTransaction(); // tr->setVersion(1); - Version ver = wait( tr->getReadVersion() ); + Version ver = wait(tr->getReadVersion()); printf("%" PRId64 "\n", ver); - state std::vector< Future > versions; + state std::vector> versions; state double starttime = timer_monotonic(); state int i; // for (i = 0; i < 100000; i++) { // Version v = wait( tr->getReadVersion() ); // } - for ( i = 0; i < 100000; i++ ) { - versions.push_back( tr->getReadVersion() ); + for (i = 0; i < 100000; i++) { + versions.push_back(tr->getReadVersion()); } - for ( i = 0; i < 100000; i++ ) { - Version v = wait( versions[i] ); + for (i = 0; i < 100000; i++) { + Version v = wait(versions[i]); } // wait( waitForAllReady( versions ) ); - printf("Elapsed: %lf\n", timer_monotonic() - starttime ); + printf("Elapsed: %lf\n", timer_monotonic() - starttime); - tr->set( LiteralStringRef("foo"), LiteralStringRef("bar") ); + tr->set(LiteralStringRef("foo"), LiteralStringRef("bar")); - Optional< FDBStandalone > v = wait( tr->get( LiteralStringRef("foo") ) ); - if ( v.present() ) { - printf("%s\n", v.get().toString().c_str() ); + Optional> v = wait(tr->get(LiteralStringRef("foo"))); + if (v.present()) { + printf("%s\n", v.get().toString().c_str()); } - FDBStandalone r = wait( tr->getRange( KeyRangeRef( LiteralStringRef("a"), LiteralStringRef("z") ), 100 ) ); + FDBStandalone r = + wait(tr->getRange(KeyRangeRef(LiteralStringRef("a"), LiteralStringRef("z")), 100)); - for ( auto kv : r ) { + for (auto kv : r) { printf("%s is %s\n", kv.key.toString().c_str(), kv.value.toString().c_str()); } @@ -79,7 +80,7 @@ ACTOR Future _test() { } void fdb_flow_test() { - API *fdb = FDB::API::selectAPIVersion(630); + API* fdb = FDB::API::selectAPIVersion(630); fdb->setupNetwork(); startThread(networkThread, fdb); @@ -96,353 +97,384 @@ void fdb_flow_test() { // FDB object used by bindings namespace FDB { - class DatabaseImpl : public Database, NonCopyable { - public: - virtual ~DatabaseImpl() { fdb_database_destroy(db); } +class DatabaseImpl : public Database, NonCopyable { +public: + virtual ~DatabaseImpl() { fdb_database_destroy(db); } - Reference createTransaction() override; - void setDatabaseOption(FDBDatabaseOption option, Optional value = Optional()) override; + Reference createTransaction() override; + void setDatabaseOption(FDBDatabaseOption option, Optional value = Optional()) override; - private: - FDBDatabase* db; - explicit DatabaseImpl(FDBDatabase* db) : db(db) {} +private: + FDBDatabase* db; + explicit DatabaseImpl(FDBDatabase* db) : db(db) {} - friend class API; - }; + friend class API; +}; - class TransactionImpl : public Transaction, private NonCopyable, public FastAllocated { - friend class DatabaseImpl; +class TransactionImpl : public Transaction, private NonCopyable, public FastAllocated { + friend class DatabaseImpl; - public: - virtual ~TransactionImpl() { - if (tr) { - fdb_transaction_destroy(tr); - } +public: + virtual ~TransactionImpl() { + if (tr) { + fdb_transaction_destroy(tr); } - - void setReadVersion(Version v) override; - Future getReadVersion() override; - - Future>> get(const Key& key, bool snapshot = false) override; - Future> getKey(const KeySelector& key, bool snapshot = false) override; - - Future watch(const Key& key) override; - - using Transaction::getRange; - Future> getRange(const KeySelector& begin, const KeySelector& end, - GetRangeLimits limits = GetRangeLimits(), bool snapshot = false, - bool reverse = false, - FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override; - - Future getEstimatedRangeSizeBytes(const KeyRange& keys) override; - - void addReadConflictRange(KeyRangeRef const& keys) override; - void addReadConflictKey(KeyRef const& key) override; - void addWriteConflictRange(KeyRangeRef const& keys) override; - void addWriteConflictKey(KeyRef const& key) override; - - void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) override; - void set(const KeyRef& key, const ValueRef& value) override; - void clear(const KeyRangeRef& range) override; - void clear(const KeyRef& key) override; - - Future commit() override; - Version getCommittedVersion() override; - Future> getVersionstamp() override; - - void setOption(FDBTransactionOption option, Optional value = Optional()) override; - - Future getApproximateSize() override; - Future onError(Error const& e) override; - - void cancel() override; - void reset() override; - - TransactionImpl() : tr(NULL) {} - TransactionImpl(TransactionImpl&& r) BOOST_NOEXCEPT { - tr = r.tr; - r.tr = NULL; - } - TransactionImpl& operator=(TransactionImpl&& r) BOOST_NOEXCEPT { - tr = r.tr; - r.tr = NULL; - return *this; - } - - private: - FDBTransaction* tr; - - explicit TransactionImpl(FDBDatabase* db); - }; - - static inline void throw_on_error( fdb_error_t e ) { - if (e) - throw Error(e); } - void CFuture::blockUntilReady() { - throw_on_error( fdb_future_block_until_ready( f ) ); + void setReadVersion(Version v) override; + Future getReadVersion() override; + + Future>> get(const Key& key, bool snapshot = false) override; + Future> getKey(const KeySelector& key, bool snapshot = false) override; + + Future watch(const Key& key) override; + + using Transaction::getRange; + Future> getRange(const KeySelector& begin, + const KeySelector& end, + GetRangeLimits limits = GetRangeLimits(), + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override; + + Future getEstimatedRangeSizeBytes(const KeyRange& keys) override; + + void addReadConflictRange(KeyRangeRef const& keys) override; + void addReadConflictKey(KeyRef const& key) override; + void addWriteConflictRange(KeyRangeRef const& keys) override; + void addWriteConflictKey(KeyRef const& key) override; + + void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) override; + void set(const KeyRef& key, const ValueRef& value) override; + void clear(const KeyRangeRef& range) override; + void clear(const KeyRef& key) override; + + Future commit() override; + Version getCommittedVersion() override; + Future> getVersionstamp() override; + + void setOption(FDBTransactionOption option, Optional value = Optional()) override; + + Future getApproximateSize() override; + Future onError(Error const& e) override; + + void cancel() override; + void reset() override; + + TransactionImpl() : tr(NULL) {} + TransactionImpl(TransactionImpl&& r) BOOST_NOEXCEPT { + tr = r.tr; + r.tr = NULL; + } + TransactionImpl& operator=(TransactionImpl&& r) BOOST_NOEXCEPT { + tr = r.tr; + r.tr = NULL; + return *this; } - void backToFutureCallback( FDBFuture* f, void* data ) { - g_network->onMainThread( Promise((SAV*)data), TaskPriority::DefaultOnMainThread ); // SOMEDAY: think about this priority - } +private: + FDBTransaction* tr; - // backToFuture( FDBFuture*, (FDBFuture* -> Type) ) -> Future - // Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread) - // and converts it into a Future (with callbacks working on this thread, cancellation etc). - // You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type - ACTOR template static Future backToFuture( FDBFuture* _f, Function convertValue ) { - state Reference f( new CFuture(_f) ); + explicit TransactionImpl(FDBDatabase* db); +}; - Promise ready; - Future onReady = ready.getFuture(); +static inline void throw_on_error(fdb_error_t e) { + if (e) + throw Error(e); +} - throw_on_error( fdb_future_set_callback( f->f, backToFutureCallback, ready.extractRawPointer() ) ); - wait( onReady ); +void CFuture::blockUntilReady() { + throw_on_error(fdb_future_block_until_ready(f)); +} - return convertValue( f ); - } +void backToFutureCallback(FDBFuture* f, void* data) { + g_network->onMainThread(Promise((SAV*)data), + TaskPriority::DefaultOnMainThread); // SOMEDAY: think about this priority +} - void API::setNetworkOption( FDBNetworkOption option, Optional value ) { - if ( value.present() ) - throw_on_error( fdb_network_set_option( option, value.get().begin(), value.get().size() ) ); - else - throw_on_error( fdb_network_set_option( option, NULL, 0 ) ); - } +// backToFuture( FDBFuture*, (FDBFuture* -> Type) ) -> Future +// Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread) +// and converts it into a Future (with callbacks working on this thread, cancellation etc). +// You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type +ACTOR template +static Future backToFuture(FDBFuture* _f, Function convertValue) { + state Reference f(new CFuture(_f)); - API* API::instance = NULL; - API::API(int version) : version(version) {} + Promise ready; + Future onReady = ready.getFuture(); - API* API::selectAPIVersion(int apiVersion) { - if(API::instance) { - if(apiVersion != API::instance->version) { - throw api_version_already_set(); - } - else { - return API::instance; - } - } + throw_on_error(fdb_future_set_callback(f->f, backToFutureCallback, ready.extractRawPointer())); + wait(onReady); - if(apiVersion < 500 || apiVersion > FDB_API_VERSION) { - throw api_version_not_supported(); - } + return convertValue(f); +} - throw_on_error( fdb_select_api_version_impl(apiVersion, FDB_API_VERSION) ); +void API::setNetworkOption(FDBNetworkOption option, Optional value) { + if (value.present()) + throw_on_error(fdb_network_set_option(option, value.get().begin(), value.get().size())); + else + throw_on_error(fdb_network_set_option(option, NULL, 0)); +} - API::instance = new API(apiVersion); - return API::instance; - } +API* API::instance = NULL; +API::API(int version) : version(version) {} - bool API::isAPIVersionSelected() { - return API::instance != NULL; - } - - API* API::getInstance() { - if(API::instance == NULL) { - throw api_version_unset(); - } - else { +API* API::selectAPIVersion(int apiVersion) { + if (API::instance) { + if (apiVersion != API::instance->version) { + throw api_version_already_set(); + } else { return API::instance; } } - void API::setupNetwork() { - throw_on_error( fdb_setup_network() ); + if (apiVersion < 500 || apiVersion > FDB_API_VERSION) { + throw api_version_not_supported(); } - void API::runNetwork() { - throw_on_error( fdb_run_network() ); + throw_on_error(fdb_select_api_version_impl(apiVersion, FDB_API_VERSION)); + + API::instance = new API(apiVersion); + return API::instance; +} + +bool API::isAPIVersionSelected() { + return API::instance != NULL; +} + +API* API::getInstance() { + if (API::instance == NULL) { + throw api_version_unset(); + } else { + return API::instance; } +} - void API::stopNetwork() { - throw_on_error( fdb_stop_network() ); +void API::setupNetwork() { + throw_on_error(fdb_setup_network()); +} + +void API::runNetwork() { + throw_on_error(fdb_run_network()); +} + +void API::stopNetwork() { + throw_on_error(fdb_stop_network()); +} + +bool API::evaluatePredicate(FDBErrorPredicate pred, Error const& e) { + return fdb_error_predicate(pred, e.code()); +} + +Reference API::createDatabase(std::string const& connFilename) { + FDBDatabase* db; + throw_on_error(fdb_create_database(connFilename.c_str(), &db)); + return Reference(new DatabaseImpl(db)); +} + +int API::getAPIVersion() const { + return version; +} + +Reference DatabaseImpl::createTransaction() { + return Reference(new TransactionImpl(db)); +} + +void DatabaseImpl::setDatabaseOption(FDBDatabaseOption option, Optional value) { + if (value.present()) + throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size())); + else + throw_on_error(fdb_database_set_option(db, option, NULL, 0)); +} + +TransactionImpl::TransactionImpl(FDBDatabase* db) { + throw_on_error(fdb_database_create_transaction(db, &tr)); +} + +void TransactionImpl::setReadVersion(Version v) { + fdb_transaction_set_read_version(tr, v); +} + +Future TransactionImpl::getReadVersion() { + return backToFuture(fdb_transaction_get_read_version(tr), [](Reference f) { + Version value; + + throw_on_error(fdb_future_get_int64(f->f, &value)); + + return value; + }); +} + +Future>> TransactionImpl::get(const Key& key, bool snapshot) { + return backToFuture>>( + fdb_transaction_get(tr, key.begin(), key.size(), snapshot), [](Reference f) { + fdb_bool_t present; + uint8_t const* value; + int value_length; + + throw_on_error(fdb_future_get_value(f->f, &present, &value, &value_length)); + + if (present) { + return Optional>(FDBStandalone(f, ValueRef(value, value_length))); + } else { + return Optional>(); + } + }); +} + +Future TransactionImpl::watch(const Key& key) { + return backToFuture(fdb_transaction_watch(tr, key.begin(), key.size()), [](Reference f) { + throw_on_error(fdb_future_get_error(f->f)); + return Void(); + }); +} + +Future> TransactionImpl::getKey(const KeySelector& key, bool snapshot) { + return backToFuture>( + fdb_transaction_get_key(tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot), + [](Reference f) { + uint8_t const* key; + int key_length; + + throw_on_error(fdb_future_get_key(f->f, &key, &key_length)); + + return FDBStandalone(f, KeyRef(key, key_length)); + }); +} + +Future> TransactionImpl::getRange(const KeySelector& begin, + const KeySelector& end, + GetRangeLimits limits, + bool snapshot, + bool reverse, + FDBStreamingMode streamingMode) { + // FIXME: iteration + return backToFuture>( + fdb_transaction_get_range(tr, + begin.key.begin(), + begin.key.size(), + begin.orEqual, + begin.offset, + end.key.begin(), + end.key.size(), + end.orEqual, + end.offset, + limits.rows, + limits.bytes, + streamingMode, + 1, + snapshot, + reverse), + [](Reference f) { + FDBKeyValue const* kv; + int count; + fdb_bool_t more; + + throw_on_error(fdb_future_get_keyvalue_array(f->f, &kv, &count, &more)); + + return FDBStandalone(f, + RangeResultRef(VectorRef((KeyValueRef*)kv, count), more)); + }); +} + +Future TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) { + return backToFuture(fdb_transaction_get_estimated_range_size_bytes( + tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()), + [](Reference f) { + int64_t bytes; + throw_on_error(fdb_future_get_int64(f->f, &bytes)); + return bytes; + }); +} + +void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) { + throw_on_error(fdb_transaction_add_conflict_range( + tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ)); +} + +void TransactionImpl::addReadConflictKey(KeyRef const& key) { + return addReadConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key)))); +} + +void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) { + throw_on_error(fdb_transaction_add_conflict_range( + tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE)); +} + +void TransactionImpl::addWriteConflictKey(KeyRef const& key) { + return addWriteConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key)))); +} + +void TransactionImpl::atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) { + fdb_transaction_atomic_op(tr, key.begin(), key.size(), operand.begin(), operand.size(), operationType); +} + +void TransactionImpl::set(const KeyRef& key, const ValueRef& value) { + fdb_transaction_set(tr, key.begin(), key.size(), value.begin(), value.size()); +} + +void TransactionImpl::clear(const KeyRangeRef& range) { + fdb_transaction_clear_range(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size()); +} + +void TransactionImpl::clear(const KeyRef& key) { + fdb_transaction_clear(tr, key.begin(), key.size()); +} + +Future TransactionImpl::commit() { + return backToFuture(fdb_transaction_commit(tr), [](Reference f) { + throw_on_error(fdb_future_get_error(f->f)); + return Void(); + }); +} + +Version TransactionImpl::getCommittedVersion() { + Version v; + + throw_on_error(fdb_transaction_get_committed_version(tr, &v)); + return v; +} + +Future> TransactionImpl::getVersionstamp() { + return backToFuture>(fdb_transaction_get_versionstamp(tr), [](Reference f) { + uint8_t const* key; + int key_length; + + throw_on_error(fdb_future_get_key(f->f, &key, &key_length)); + + return FDBStandalone(f, StringRef(key, key_length)); + }); +} + +void TransactionImpl::setOption(FDBTransactionOption option, Optional value) { + if (value.present()) { + throw_on_error(fdb_transaction_set_option(tr, option, value.get().begin(), value.get().size())); + } else { + throw_on_error(fdb_transaction_set_option(tr, option, NULL, 0)); } +} - bool API::evaluatePredicate(FDBErrorPredicate pred, Error const& e) { - return fdb_error_predicate( pred, e.code() ); - } +Future TransactionImpl::getApproximateSize() { + return backToFuture(fdb_transaction_get_approximate_size(tr), [](Reference f) { + int64_t size = 0; + throw_on_error(fdb_future_get_int64(f->f, &size)); + return size; + }); +} - Reference API::createDatabase(std::string const& connFilename) { - FDBDatabase *db; - throw_on_error(fdb_create_database(connFilename.c_str(), &db)); - return Reference(new DatabaseImpl(db)); - } +Future TransactionImpl::onError(Error const& e) { + return backToFuture(fdb_transaction_on_error(tr, e.code()), [](Reference f) { + throw_on_error(fdb_future_get_error(f->f)); + return Void(); + }); +} - int API::getAPIVersion() const { - return version; - } +void TransactionImpl::cancel() { + fdb_transaction_cancel(tr); +} - Reference DatabaseImpl::createTransaction() { - return Reference(new TransactionImpl(db)); - } +void TransactionImpl::reset() { + fdb_transaction_reset(tr); +} - void DatabaseImpl::setDatabaseOption(FDBDatabaseOption option, Optional value) { - if (value.present()) - throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size())); - else - throw_on_error(fdb_database_set_option(db, option, NULL, 0)); - } - - TransactionImpl::TransactionImpl(FDBDatabase* db) { - throw_on_error(fdb_database_create_transaction(db, &tr)); - } - - void TransactionImpl::setReadVersion(Version v) { - fdb_transaction_set_read_version( tr, v ); - } - - Future TransactionImpl::getReadVersion() { - return backToFuture( fdb_transaction_get_read_version( tr ), [](Reference f){ - Version value; - - throw_on_error( fdb_future_get_int64( f->f, &value ) ); - - return value; - } ); - } - - Future>> TransactionImpl::get(const Key& key, bool snapshot) { - return backToFuture< Optional> >( fdb_transaction_get( tr, key.begin(), key.size(), snapshot ), [](Reference f) { - fdb_bool_t present; - uint8_t const* value; - int value_length; - - throw_on_error( fdb_future_get_value( f->f, &present, &value, &value_length ) ); - - if ( present ) { - return Optional>( FDBStandalone( f, ValueRef( value, value_length ) ) ); - } else { - return Optional>(); - } - } ); - } - - Future TransactionImpl::watch(const Key& key) { - return backToFuture< Void >( fdb_transaction_watch( tr, key.begin(), key.size() ), [](Reference f) { - throw_on_error( fdb_future_get_error( f->f ) ); - return Void(); - } ); - } - - Future> TransactionImpl::getKey(const KeySelector& key, bool snapshot) { - return backToFuture< FDBStandalone >( fdb_transaction_get_key( tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot ), [](Reference f) { - uint8_t const* key; - int key_length; - - throw_on_error( fdb_future_get_key( f->f, &key, &key_length ) ); - - return FDBStandalone( f, KeyRef( key, key_length ) ); - } ); - } - - Future> TransactionImpl::getRange(const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, bool snapshot, bool reverse, FDBStreamingMode streamingMode) { - // FIXME: iteration - return backToFuture< FDBStandalone >( fdb_transaction_get_range( tr, begin.key.begin(), begin.key.size(), begin.orEqual, begin.offset, end.key.begin(), end.key.size(), end.orEqual, end.offset, limits.rows, limits.bytes, streamingMode, 1, snapshot, reverse ), [](Reference f) { - FDBKeyValue const* kv; - int count; - fdb_bool_t more; - - throw_on_error( fdb_future_get_keyvalue_array( f->f, &kv, &count, &more ) ); - - return FDBStandalone( f, RangeResultRef( VectorRef( (KeyValueRef*)kv, count ), more ) ); - } ); - } - - Future TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) { - return backToFuture(fdb_transaction_get_estimated_range_size_bytes(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()), [](Reference f) { - int64_t bytes; - throw_on_error(fdb_future_get_int64(f->f, &bytes)); - return bytes; - }); - } - - void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) { - throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ ) ); - } - - void TransactionImpl::addReadConflictKey(KeyRef const& key) { - return addReadConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key)))); - } - - void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) { - throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE ) ); - } - - void TransactionImpl::addWriteConflictKey(KeyRef const& key) { - return addWriteConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key)))); - } - - void TransactionImpl::atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) { - fdb_transaction_atomic_op( tr, key.begin(), key.size(), operand.begin(), operand.size(), operationType ); - } - - void TransactionImpl::set(const KeyRef& key, const ValueRef& value) { - fdb_transaction_set( tr, key.begin(), key.size(), value.begin(), value.size() ); - } - - void TransactionImpl::clear(const KeyRangeRef& range) { - fdb_transaction_clear_range( tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size() ); - } - - void TransactionImpl::clear(const KeyRef& key) { - fdb_transaction_clear( tr, key.begin(), key.size() ); - } - - Future TransactionImpl::commit() { - return backToFuture< Void >( fdb_transaction_commit( tr ), [](Reference f) { - throw_on_error( fdb_future_get_error( f->f ) ); - return Void(); - } ); - } - - Version TransactionImpl::getCommittedVersion() { - Version v; - - throw_on_error( fdb_transaction_get_committed_version( tr, &v ) ); - return v; - } - - Future> TransactionImpl::getVersionstamp() { - return backToFuture>(fdb_transaction_get_versionstamp(tr), [](Reference f) { - uint8_t const* key; - int key_length; - - throw_on_error( fdb_future_get_key( f->f, &key, &key_length ) ); - - return FDBStandalone( f, StringRef( key, key_length ) ); - }); - } - - void TransactionImpl::setOption(FDBTransactionOption option, Optional value) { - if ( value.present() ) { - throw_on_error( fdb_transaction_set_option( tr, option, value.get().begin(), value.get().size() ) ); - } else { - throw_on_error( fdb_transaction_set_option( tr, option, NULL, 0 ) ); - } - } - - Future TransactionImpl::getApproximateSize() { - return backToFuture(fdb_transaction_get_approximate_size(tr), [](Reference f) { - int64_t size = 0; - throw_on_error(fdb_future_get_int64(f->f, &size)); - return size; - }); - } - - Future TransactionImpl::onError(Error const& e) { - return backToFuture< Void >( fdb_transaction_on_error( tr, e.code() ), [](Reference f) { - throw_on_error( fdb_future_get_error( f->f ) ); - return Void(); - } ); - } - - void TransactionImpl::cancel() { - fdb_transaction_cancel( tr ); - } - - void TransactionImpl::reset() { - fdb_transaction_reset( tr ); - } - -} // namespace FDB +} // namespace FDB diff --git a/bindings/flow/fdb_flow.h b/bindings/flow/fdb_flow.h index e261052fae..fdd5b63d8f 100644 --- a/bindings/flow/fdb_flow.h +++ b/bindings/flow/fdb_flow.h @@ -30,123 +30,139 @@ #include "FDBLoanerTypes.h" namespace FDB { - struct CFuture : NonCopyable, ReferenceCounted, FastAllocated { - CFuture() : f(NULL) {} - explicit CFuture(FDBFuture* f) : f(f) {} - ~CFuture() { - if (f) { - fdb_future_destroy(f); - } +struct CFuture : NonCopyable, ReferenceCounted, FastAllocated { + CFuture() : f(NULL) {} + explicit CFuture(FDBFuture* f) : f(f) {} + ~CFuture() { + if (f) { + fdb_future_destroy(f); } + } - void blockUntilReady(); + void blockUntilReady(); - FDBFuture* f; - }; + FDBFuture* f; +}; - template - class FDBStandalone : public T { - public: - FDBStandalone() {} - FDBStandalone(Reference f, T const& t) : T(t), f(f) {} - FDBStandalone(FDBStandalone const& o) : T((T const&)o), f(o.f) {} +template +class FDBStandalone : public T { +public: + FDBStandalone() {} + FDBStandalone(Reference f, T const& t) : T(t), f(f) {} + FDBStandalone(FDBStandalone const& o) : T((T const&)o), f(o.f) {} - private: - Reference f; - }; +private: + Reference f; +}; - class ReadTransaction : public ReferenceCounted { - public: - virtual ~ReadTransaction(){}; - virtual void setReadVersion(Version v) = 0; - virtual Future getReadVersion() = 0; +class ReadTransaction : public ReferenceCounted { +public: + virtual ~ReadTransaction(){}; + virtual void setReadVersion(Version v) = 0; + virtual Future getReadVersion() = 0; - virtual Future>> get(const Key& key, bool snapshot = false) = 0; - virtual Future> getKey(const KeySelector& key, bool snapshot = false) = 0; - virtual Future watch(const Key& key) = 0; + virtual Future>> get(const Key& key, bool snapshot = false) = 0; + virtual Future> getKey(const KeySelector& key, bool snapshot = false) = 0; + virtual Future watch(const Key& key) = 0; - virtual Future> getRange( - const KeySelector& begin, const KeySelector& end, GetRangeLimits limits = GetRangeLimits(), - bool snapshot = false, bool reverse = false, - FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0; - virtual Future> getRange( - const KeySelector& begin, const KeySelector& end, int limit, bool snapshot = false, bool reverse = false, - FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { - return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode); - } - virtual Future> getRange( - const KeyRange& keys, int limit, bool snapshot = false, bool reverse = false, - FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { - return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), - KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limit, snapshot, reverse, - streamingMode); - } - virtual Future> getRange( - const KeyRange& keys, GetRangeLimits limits = GetRangeLimits(), bool snapshot = false, bool reverse = false, - FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { - return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), - KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limits, snapshot, reverse, - streamingMode); - } + virtual Future> getRange( + const KeySelector& begin, + const KeySelector& end, + GetRangeLimits limits = GetRangeLimits(), + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) = 0; + virtual Future> getRange(const KeySelector& begin, + const KeySelector& end, + int limit, + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { + return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse, streamingMode); + } + virtual Future> getRange(const KeyRange& keys, + int limit, + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { + return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), + KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), + limit, + snapshot, + reverse, + streamingMode); + } + virtual Future> getRange(const KeyRange& keys, + GetRangeLimits limits = GetRangeLimits(), + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { + return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), + KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), + limits, + snapshot, + reverse, + streamingMode); + } - virtual Future getEstimatedRangeSizeBytes(const KeyRange& keys) = 0; + virtual Future getEstimatedRangeSizeBytes(const KeyRange& keys) = 0; - virtual void addReadConflictRange(KeyRangeRef const& keys) = 0; - virtual void addReadConflictKey(KeyRef const& key) = 0; + virtual void addReadConflictRange(KeyRangeRef const& keys) = 0; + virtual void addReadConflictKey(KeyRef const& key) = 0; - virtual void setOption(FDBTransactionOption option, Optional value = Optional()) = 0; + virtual void setOption(FDBTransactionOption option, Optional value = Optional()) = 0; - virtual Future onError(Error const& e) = 0; + virtual Future onError(Error const& e) = 0; - virtual void cancel() = 0; - virtual void reset() = 0; - }; + virtual void cancel() = 0; + virtual void reset() = 0; +}; - class Transaction : public ReadTransaction { - public: - virtual void addWriteConflictRange(KeyRangeRef const& keys) = 0; - virtual void addWriteConflictKey(KeyRef const& key) = 0; +class Transaction : public ReadTransaction { +public: + virtual void addWriteConflictRange(KeyRangeRef const& keys) = 0; + virtual void addWriteConflictKey(KeyRef const& key) = 0; - virtual void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) = 0; - virtual void set(const KeyRef& key, const ValueRef& value) = 0; - virtual void clear(const KeyRangeRef& range) = 0; - virtual void clear(const KeyRef& key) = 0; + virtual void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) = 0; + virtual void set(const KeyRef& key, const ValueRef& value) = 0; + virtual void clear(const KeyRangeRef& range) = 0; + virtual void clear(const KeyRef& key) = 0; - virtual Future commit() = 0; - virtual Version getCommittedVersion() = 0; - virtual Future getApproximateSize() = 0; - virtual Future> getVersionstamp() = 0; - }; + virtual Future commit() = 0; + virtual Version getCommittedVersion() = 0; + virtual Future getApproximateSize() = 0; + virtual Future> getVersionstamp() = 0; +}; - class Database : public ReferenceCounted { - public: - virtual ~Database(){}; - virtual Reference createTransaction() = 0; - virtual void setDatabaseOption(FDBDatabaseOption option, Optional value = Optional()) = 0; - }; +class Database : public ReferenceCounted { +public: + virtual ~Database(){}; + virtual Reference createTransaction() = 0; + virtual void setDatabaseOption(FDBDatabaseOption option, Optional value = Optional()) = 0; +}; - class API { - public: - static API* selectAPIVersion(int apiVersion); - static API* getInstance(); - static bool isAPIVersionSelected(); +class API { +public: + static API* selectAPIVersion(int apiVersion); + static API* getInstance(); + static bool isAPIVersionSelected(); - void setNetworkOption(FDBNetworkOption option, Optional value = Optional()); + void setNetworkOption(FDBNetworkOption option, Optional value = Optional()); - void setupNetwork(); - void runNetwork(); - void stopNetwork(); + void setupNetwork(); + void runNetwork(); + void stopNetwork(); - Reference createDatabase(std::string const& connFilename = ""); + Reference createDatabase(std::string const& connFilename = ""); - bool evaluatePredicate(FDBErrorPredicate pred, Error const& e); - int getAPIVersion() const; + bool evaluatePredicate(FDBErrorPredicate pred, Error const& e); + int getAPIVersion() const; - private: - static API* instance; +private: + static API* instance; - API(int version); - int version; - }; - } // namespace FDB + API(int version); + int version; +}; +} // namespace FDB #endif // FDB_FLOW_FDB_FLOW_H diff --git a/bindings/flow/tester/DirectoryTester.actor.cpp b/bindings/flow/tester/DirectoryTester.actor.cpp index a04317792b..35b95dabd6 100644 --- a/bindings/flow/tester/DirectoryTester.actor.cpp +++ b/bindings/flow/tester/DirectoryTester.actor.cpp @@ -19,14 +19,14 @@ */ #include "Tester.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. +#include "flow/actorcompiler.h" // This must be the last #include. using namespace FDB; ACTOR Future> popTuples(Reference data, int count = 1) { state std::vector tuples; - while(tuples.size() < count) { + while (tuples.size() < count) { Standalone sizeStr = wait(data->stack.pop()[0].value); int size = Tuple::unpack(sizeStr).getInt(0); @@ -34,7 +34,7 @@ ACTOR Future> popTuples(Reference data, int c state Tuple tuple; state int index; - for(index = 0; index < tupleItems.size(); ++index) { + for (index = 0; index < tupleItems.size(); ++index) { Standalone itemStr = wait(tupleItems[index].value); tuple.append(Tuple::unpack(itemStr)); } @@ -54,9 +54,9 @@ ACTOR Future> popPaths(Reference d std::vector tuples = wait(popTuples(data, count)); std::vector paths; - for(auto &tuple : tuples) { + for (auto& tuple : tuples) { IDirectory::Path path; - for(int i = 0; i < tuple.size(); ++i) { + for (int i = 0; i < tuple.size(); ++i) { path.push_back(tuple.getString(i)); } @@ -74,9 +74,9 @@ ACTOR Future popPath(Reference data) { std::string pathToString(IDirectory::Path const& path) { std::string str; str += "["; - for(int i = 0; i < path.size(); ++i) { + for (int i = 0; i < path.size(); ++i) { str += path[i].toString(); - if(i < path.size() - 1) { + if (i < path.size() - 1) { str += ", "; } } @@ -86,21 +86,21 @@ std::string pathToString(IDirectory::Path const& path) { IDirectory::Path combinePaths(IDirectory::Path const& path1, IDirectory::Path const& path2) { IDirectory::Path outPath(path1.begin(), path1.end()); - for(auto p : path2) { + for (auto p : path2) { outPath.push_back(p); } return outPath; } -void logOp(std::string message, bool force=false) { - if(LOG_OPS || force) { +void logOp(std::string message, bool force = false) { + if (LOG_OPS || force) { printf("%s\n", message.c_str()); fflush(stdout); } } -//DIRECTORY_CREATE_SUBSPACE +// DIRECTORY_CREATE_SUBSPACE struct DirectoryCreateSubspaceFunc : InstructionFunc { static const char* name; @@ -108,7 +108,8 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc { state Tuple path = wait(popTuple(data)); Tuple rawPrefix = wait(data->stack.waitAndPop()); - logOp(format("Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str())); + logOp(format( + "Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str())); data->directoryData.push(new Subspace(path, rawPrefix.getString(0))); return Void(); } @@ -116,7 +117,7 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc { const char* DirectoryCreateSubspaceFunc::name = "DIRECTORY_CREATE_SUBSPACE"; REGISTER_INSTRUCTION_FUNC(DirectoryCreateSubspaceFunc); -//DIRECTORY_CREATE_LAYER +// DIRECTORY_CREATE_LAYER struct DirectoryCreateLayerFunc : InstructionFunc { static const char* name; @@ -127,15 +128,21 @@ struct DirectoryCreateLayerFunc : InstructionFunc { int index2 = args[1].getInt(0); bool allowManualPrefixes = args[2].getInt(0) != 0; - if(!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) { + if (!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) { logOp("Create directory layer: None"); data->directoryData.push(); - } - else { + } else { Subspace* nodeSubspace = data->directoryData.directoryList[index1].subspace.get(); Subspace* contentSubspace = data->directoryData.directoryList[index2].subspace.get(); - logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, allow_manual_prefixes = %d", index1, nodeSubspace->key().printable().c_str(), index2, nodeSubspace->key().printable().c_str(), allowManualPrefixes)); - data->directoryData.push(Reference(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes))); + logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, " + "allow_manual_prefixes = %d", + index1, + nodeSubspace->key().printable().c_str(), + index2, + nodeSubspace->key().printable().c_str(), + allowManualPrefixes)); + data->directoryData.push( + Reference(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes))); } return Void(); @@ -144,7 +151,7 @@ struct DirectoryCreateLayerFunc : InstructionFunc { const char* DirectoryCreateLayerFunc::name = "DIRECTORY_CREATE_LAYER"; REGISTER_INSTRUCTION_FUNC(DirectoryCreateLayerFunc); -//DIRECTORY_CHANGE +// DIRECTORY_CHANGE struct DirectoryChangeFunc : InstructionFunc { static const char* name; @@ -153,13 +160,17 @@ struct DirectoryChangeFunc : InstructionFunc { data->directoryData.directoryListIndex = index.getInt(0); ASSERT(data->directoryData.directoryListIndex < data->directoryData.directoryList.size()); - if(!data->directoryData.directoryList[data->directoryData.directoryListIndex].valid()) { + if (!data->directoryData.directoryList[data->directoryData.directoryListIndex].valid()) { data->directoryData.directoryListIndex = data->directoryData.directoryErrorIndex; } - if(LOG_DIRS) { + if (LOG_DIRS) { DirectoryOrSubspace d = data->directoryData.directoryList[data->directoryData.directoryListIndex]; - printf("Changed directory to %d (%s @\'%s\')\n", data->directoryData.directoryListIndex, d.typeString().c_str(), d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str() : d.subspace.get()->key().printable().c_str()); + printf("Changed directory to %d (%s @\'%s\')\n", + data->directoryData.directoryListIndex, + d.typeString().c_str(), + d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str() + : d.subspace.get()->key().printable().c_str()); fflush(stdout); } @@ -169,7 +180,7 @@ struct DirectoryChangeFunc : InstructionFunc { const char* DirectoryChangeFunc::name = "DIRECTORY_CHANGE"; REGISTER_INSTRUCTION_FUNC(DirectoryChangeFunc); -//DIRECTORY_SET_ERROR_INDEX +// DIRECTORY_SET_ERROR_INDEX struct DirectorySetErrorIndexFunc : InstructionFunc { static const char* name; @@ -183,7 +194,7 @@ struct DirectorySetErrorIndexFunc : InstructionFunc { const char* DirectorySetErrorIndexFunc::name = "DIRECTORY_SET_ERROR_INDEX"; REGISTER_INSTRUCTION_FUNC(DirectorySetErrorIndexFunc); -//DIRECTORY_CREATE_OR_OPEN +// DIRECTORY_CREATE_OR_OPEN struct DirectoryCreateOrOpenFunc : InstructionFunc { static const char* name; @@ -193,11 +204,12 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc { Standalone layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0); Reference directory = data->directoryData.directory(); - logOp(format("create_or_open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str())); + logOp(format("create_or_open %s: layer=%s", + pathToString(combinePaths(directory->getPath(), path)).c_str(), + layer.printable().c_str())); - Reference dirSubspace = wait(executeMutation(instruction, [this, directory, layer] () { - return directory->createOrOpen(instruction->tr, path, layer); - })); + Reference dirSubspace = wait(executeMutation( + instruction, [this, directory, layer]() { return directory->createOrOpen(instruction->tr, path, layer); })); data->directoryData.push(dirSubspace); @@ -207,7 +219,7 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc { const char* DirectoryCreateOrOpenFunc::name = "DIRECTORY_CREATE_OR_OPEN"; REGISTER_INSTRUCTION_FUNC(DirectoryCreateOrOpenFunc); -//DIRECTORY_CREATE +// DIRECTORY_CREATE struct DirectoryCreateFunc : InstructionFunc { static const char* name; @@ -215,14 +227,19 @@ struct DirectoryCreateFunc : InstructionFunc { state IDirectory::Path path = wait(popPath(data)); std::vector args = wait(data->stack.waitAndPop(2)); Standalone layer = args[0].getType(0) == Tuple::NULL_TYPE ? StringRef() : args[0].getString(0); - Optional> prefix = args[1].getType(0) == Tuple::NULL_TYPE ? Optional>() : args[1].getString(0); + Optional> prefix = + args[1].getType(0) == Tuple::NULL_TYPE ? Optional>() : args[1].getString(0); Reference directory = data->directoryData.directory(); - logOp(format("create %s: layer=%s, prefix=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str(), prefix.present() ? prefix.get().printable().c_str() : "")); + logOp(format("create %s: layer=%s, prefix=%s", + pathToString(combinePaths(directory->getPath(), path)).c_str(), + layer.printable().c_str(), + prefix.present() ? prefix.get().printable().c_str() : "")); - Reference dirSubspace = wait(executeMutation(instruction, [this, directory, layer, prefix] () { - return directory->create(instruction->tr, path, layer, prefix); - })); + Reference dirSubspace = + wait(executeMutation(instruction, [this, directory, layer, prefix]() { + return directory->create(instruction->tr, path, layer, prefix); + })); data->directoryData.push(dirSubspace); @@ -232,7 +249,7 @@ struct DirectoryCreateFunc : InstructionFunc { const char* DirectoryCreateFunc::name = "DIRECTORY_CREATE"; REGISTER_INSTRUCTION_FUNC(DirectoryCreateFunc); -//DIRECTORY_OPEN +// DIRECTORY_OPEN struct DirectoryOpenFunc : InstructionFunc { static const char* name; @@ -242,7 +259,9 @@ struct DirectoryOpenFunc : InstructionFunc { Standalone layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0); Reference directory = data->directoryData.directory(); - logOp(format("open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str())); + logOp(format("open %s: layer=%s", + pathToString(combinePaths(directory->getPath(), path)).c_str(), + layer.printable().c_str())); Reference dirSubspace = wait(directory->open(instruction->tr, path, layer)); data->directoryData.push(dirSubspace); @@ -252,7 +271,7 @@ struct DirectoryOpenFunc : InstructionFunc { const char* DirectoryOpenFunc::name = "DIRECTORY_OPEN"; REGISTER_INSTRUCTION_FUNC(DirectoryOpenFunc); -//DIRECTORY_MOVE +// DIRECTORY_MOVE struct DirectoryMoveFunc : InstructionFunc { static const char* name; @@ -260,11 +279,12 @@ struct DirectoryMoveFunc : InstructionFunc { std::vector paths = wait(popPaths(data, 2)); Reference directory = data->directoryData.directory(); - logOp(format("move %s to %s", pathToString(combinePaths(directory->getPath(), paths[0])).c_str(), pathToString(combinePaths(directory->getPath(), paths[1])).c_str())); + logOp(format("move %s to %s", + pathToString(combinePaths(directory->getPath(), paths[0])).c_str(), + pathToString(combinePaths(directory->getPath(), paths[1])).c_str())); - Reference dirSubspace = wait(executeMutation(instruction, [this, directory, paths] () { - return directory->move(instruction->tr, paths[0], paths[1]); - })); + Reference dirSubspace = wait(executeMutation( + instruction, [this, directory, paths]() { return directory->move(instruction->tr, paths[0], paths[1]); })); data->directoryData.push(dirSubspace); @@ -274,7 +294,7 @@ struct DirectoryMoveFunc : InstructionFunc { const char* DirectoryMoveFunc::name = "DIRECTORY_MOVE"; REGISTER_INSTRUCTION_FUNC(DirectoryMoveFunc); -//DIRECTORY_MOVE_TO +// DIRECTORY_MOVE_TO struct DirectoryMoveToFunc : InstructionFunc { static const char* name; @@ -284,9 +304,8 @@ struct DirectoryMoveToFunc : InstructionFunc { Reference directory = data->directoryData.directory(); logOp(format("move %s to %s", pathToString(directory->getPath()).c_str(), pathToString(path).c_str())); - Reference dirSubspace = wait(executeMutation(instruction, [this, directory, path] () { - return directory->moveTo(instruction->tr, path); - })); + Reference dirSubspace = wait(executeMutation( + instruction, [this, directory, path]() { return directory->moveTo(instruction->tr, path); })); data->directoryData.push(dirSubspace); @@ -296,27 +315,22 @@ struct DirectoryMoveToFunc : InstructionFunc { const char* DirectoryMoveToFunc::name = "DIRECTORY_MOVE_TO"; REGISTER_INSTRUCTION_FUNC(DirectoryMoveToFunc); -//DIRECTORY_REMOVE +// DIRECTORY_REMOVE struct DirectoryRemoveFunc : InstructionFunc { static const char* name; ACTOR static Future call(Reference data, Reference instruction) { Tuple count = wait(data->stack.waitAndPop()); state Reference directory = data->directoryData.directory(); - if(count.getInt(0) == 0) { + if (count.getInt(0) == 0) { logOp(format("remove %s", pathToString(directory->getPath()).c_str())); - wait(executeMutation(instruction, [this] () { - return directory->remove(instruction->tr); - })); - } - else { + wait(executeMutation(instruction, [this]() { return directory->remove(instruction->tr); })); + } else { IDirectory::Path path = wait(popPath(data)); logOp(format("remove %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); - wait(executeMutation(instruction, [this, path] () { - return directory->remove(instruction->tr, path); - })); + wait(executeMutation(instruction, [this, path]() { return directory->remove(instruction->tr, path); })); } return Void(); @@ -325,27 +339,24 @@ struct DirectoryRemoveFunc : InstructionFunc { const char* DirectoryRemoveFunc::name = "DIRECTORY_REMOVE"; REGISTER_INSTRUCTION_FUNC(DirectoryRemoveFunc); -//DIRECTORY_REMOVE_IF_EXISTS +// DIRECTORY_REMOVE_IF_EXISTS struct DirectoryRemoveIfExistsFunc : InstructionFunc { static const char* name; ACTOR static Future call(Reference data, Reference instruction) { Tuple count = wait(data->stack.waitAndPop()); state Reference directory = data->directoryData.directory(); - if(count.getInt(0) == 0) { + if (count.getInt(0) == 0) { logOp(format("remove_if_exists %s", pathToString(directory->getPath()).c_str())); - wait(success(executeMutation(instruction, [this] () { - return directory->removeIfExists(instruction->tr); - }))); - } - else { + wait( + success(executeMutation(instruction, [this]() { return directory->removeIfExists(instruction->tr); }))); + } else { IDirectory::Path path = wait(popPath(data)); logOp(format("remove_if_exists %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); - wait(success(executeMutation(instruction, [this, path] () { - return directory->removeIfExists(instruction->tr, path); - }))); + wait(success(executeMutation(instruction, + [this, path]() { return directory->removeIfExists(instruction->tr, path); }))); } return Void(); @@ -354,7 +365,7 @@ struct DirectoryRemoveIfExistsFunc : InstructionFunc { const char* DirectoryRemoveIfExistsFunc::name = "DIRECTORY_REMOVE_IF_EXISTS"; REGISTER_INSTRUCTION_FUNC(DirectoryRemoveIfExistsFunc); -//DIRECTORY_LIST +// DIRECTORY_LIST struct DirectoryListFunc : InstructionFunc { static const char* name; @@ -362,12 +373,11 @@ struct DirectoryListFunc : InstructionFunc { Tuple count = wait(data->stack.waitAndPop()); state Reference directory = data->directoryData.directory(); state Standalone> subdirs; - if(count.getInt(0) == 0) { + if (count.getInt(0) == 0) { logOp(format("list %s", pathToString(directory->getPath()).c_str())); Standalone> _subdirs = wait(directory->list(instruction->tr)); subdirs = _subdirs; - } - else { + } else { IDirectory::Path path = wait(popPath(data)); logOp(format("list %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); Standalone> _subdirs = wait(directory->list(instruction->tr, path)); @@ -375,7 +385,7 @@ struct DirectoryListFunc : InstructionFunc { } Tuple subdirTuple; - for(auto &sd : subdirs) { + for (auto& sd : subdirs) { subdirTuple.append(sd, true); } @@ -386,7 +396,7 @@ struct DirectoryListFunc : InstructionFunc { const char* DirectoryListFunc::name = "DIRECTORY_LIST"; REGISTER_INSTRUCTION_FUNC(DirectoryListFunc); -//DIRECTORY_EXISTS +// DIRECTORY_EXISTS struct DirectoryExistsFunc : InstructionFunc { static const char* name; @@ -394,12 +404,11 @@ struct DirectoryExistsFunc : InstructionFunc { Tuple count = wait(data->stack.waitAndPop()); state Reference directory = data->directoryData.directory(); state bool result; - if(count.getInt(0) == 0) { + if (count.getInt(0) == 0) { bool _result = wait(directory->exists(instruction->tr)); result = _result; logOp(format("exists %s: %d", pathToString(directory->getPath()).c_str(), result)); - } - else { + } else { state IDirectory::Path path = wait(popPath(data)); bool _result = wait(directory->exists(instruction->tr, path)); result = _result; @@ -413,7 +422,7 @@ struct DirectoryExistsFunc : InstructionFunc { const char* DirectoryExistsFunc::name = "DIRECTORY_EXISTS"; REGISTER_INSTRUCTION_FUNC(DirectoryExistsFunc); -//DIRECTORY_PACK_KEY +// DIRECTORY_PACK_KEY struct DirectoryPackKeyFunc : InstructionFunc { static const char* name; @@ -427,17 +436,19 @@ struct DirectoryPackKeyFunc : InstructionFunc { const char* DirectoryPackKeyFunc::name = "DIRECTORY_PACK_KEY"; REGISTER_INSTRUCTION_FUNC(DirectoryPackKeyFunc); -//DIRECTORY_UNPACK_KEY +// DIRECTORY_UNPACK_KEY struct DirectoryUnpackKeyFunc : InstructionFunc { static const char* name; ACTOR static Future call(Reference data, Reference instruction) { Tuple key = wait(data->stack.waitAndPop()); - Subspace *subspace = data->directoryData.subspace(); - logOp(format("Unpack %s in subspace with prefix %s", key.getString(0).printable().c_str(), subspace->key().printable().c_str())); + Subspace* subspace = data->directoryData.subspace(); + logOp(format("Unpack %s in subspace with prefix %s", + key.getString(0).printable().c_str(), + subspace->key().printable().c_str())); Tuple tuple = subspace->unpack(key.getString(0)); - for(int i = 0; i < tuple.size(); ++i) { - data->stack.push(tuple.subTuple(i, i+1).pack()); + for (int i = 0; i < tuple.size(); ++i) { + data->stack.push(tuple.subTuple(i, i + 1).pack()); } return Void(); @@ -446,7 +457,7 @@ struct DirectoryUnpackKeyFunc : InstructionFunc { const char* DirectoryUnpackKeyFunc::name = "DIRECTORY_UNPACK_KEY"; REGISTER_INSTRUCTION_FUNC(DirectoryUnpackKeyFunc); -//DIRECTORY_RANGE +// DIRECTORY_RANGE struct DirectoryRangeFunc : InstructionFunc { static const char* name; @@ -462,7 +473,7 @@ struct DirectoryRangeFunc : InstructionFunc { const char* DirectoryRangeFunc::name = "DIRECTORY_RANGE"; REGISTER_INSTRUCTION_FUNC(DirectoryRangeFunc); -//DIRECTORY_CONTAINS +// DIRECTORY_CONTAINS struct DirectoryContainsFunc : InstructionFunc { static const char* name; @@ -477,15 +488,15 @@ struct DirectoryContainsFunc : InstructionFunc { const char* DirectoryContainsFunc::name = "DIRECTORY_CONTAINS"; REGISTER_INSTRUCTION_FUNC(DirectoryContainsFunc); -//DIRECTORY_OPEN_SUBSPACE +// DIRECTORY_OPEN_SUBSPACE struct DirectoryOpenSubspaceFunc : InstructionFunc { static const char* name; ACTOR static Future call(Reference data, Reference instruction) { Tuple tuple = wait(popTuple(data)); - Subspace *subspace = data->directoryData.subspace(); + Subspace* subspace = data->directoryData.subspace(); logOp(format("open_subspace %s (at %s)", tupleToString(tuple).c_str(), subspace->key().printable().c_str())); - Subspace *child = new Subspace(subspace->subspace(tuple)); + Subspace* child = new Subspace(subspace->subspace(tuple)); data->directoryData.push(child); return Void(); @@ -494,7 +505,7 @@ struct DirectoryOpenSubspaceFunc : InstructionFunc { const char* DirectoryOpenSubspaceFunc::name = "DIRECTORY_OPEN_SUBSPACE"; REGISTER_INSTRUCTION_FUNC(DirectoryOpenSubspaceFunc); -//DIRECTORY_LOG_SUBSPACE +// DIRECTORY_LOG_SUBSPACE struct DirectoryLogSubspaceFunc : InstructionFunc { static const char* name; @@ -510,7 +521,7 @@ struct DirectoryLogSubspaceFunc : InstructionFunc { const char* DirectoryLogSubspaceFunc::name = "DIRECTORY_LOG_SUBSPACE"; REGISTER_INSTRUCTION_FUNC(DirectoryLogSubspaceFunc); -//DIRECTORY_LOG_DIRECTORY +// DIRECTORY_LOG_DIRECTORY struct DirectoryLogDirectoryFunc : InstructionFunc { static const char* name; @@ -520,22 +531,23 @@ struct DirectoryLogDirectoryFunc : InstructionFunc { state bool exists = wait(directory->exists(instruction->tr)); state Tuple childrenTuple; - if(exists) { + if (exists) { Standalone> children = wait(directory->list(instruction->tr)); - for(auto &c : children) { + for (auto& c : children) { childrenTuple.append(c, true); } - } + } Subspace logSubspace(Tuple().append(data->directoryData.directoryListIndex), prefix.getString(0)); Tuple pathTuple; - for(auto &p : directory->getPath()) { + for (auto& p : directory->getPath()) { pathTuple.append(p, true); } instruction->tr->set(logSubspace.pack(LiteralStringRef("path"), true), pathTuple.pack()); - instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true), Tuple().append(directory->getLayer()).pack()); + instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true), + Tuple().append(directory->getLayer()).pack()); instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append(exists ? 1 : 0).pack()); instruction->tr->set(logSubspace.pack(LiteralStringRef("children"), true), childrenTuple.pack()); @@ -545,13 +557,13 @@ struct DirectoryLogDirectoryFunc : InstructionFunc { const char* DirectoryLogDirectoryFunc::name = "DIRECTORY_LOG_DIRECTORY"; REGISTER_INSTRUCTION_FUNC(DirectoryLogDirectoryFunc); -//DIRECTORY_STRIP_PREFIX +// DIRECTORY_STRIP_PREFIX struct DirectoryStripPrefixFunc : InstructionFunc { static const char* name; ACTOR static Future call(Reference data, Reference instruction) { Tuple str = wait(data->stack.waitAndPop()); - Subspace *subspace = data->directoryData.subspace(); + Subspace* subspace = data->directoryData.subspace(); ASSERT(str.getString(0).startsWith(subspace->key())); data->stack.pushTuple(str.getString(0).substr(subspace->key().size())); return Void(); @@ -559,4 +571,3 @@ struct DirectoryStripPrefixFunc : InstructionFunc { }; const char* DirectoryStripPrefixFunc::name = "DIRECTORY_STRIP_PREFIX"; REGISTER_INSTRUCTION_FUNC(DirectoryStripPrefixFunc); - diff --git a/bindings/flow/tester/Tester.actor.cpp b/bindings/flow/tester/Tester.actor.cpp index c98f9c8f4b..9b13fa975a 100644 --- a/bindings/flow/tester/Tester.actor.cpp +++ b/bindings/flow/tester/Tester.actor.cpp @@ -20,7 +20,7 @@ #include "Tester.actor.h" #include -#ifdef __linux__ +#ifdef __linux__ #include #endif @@ -42,33 +42,32 @@ std::map, Reference> trMap; // NOTE: This was taken from within fdb_c.cpp (where it is defined as a static within the get_range function). // If that changes, this will also have to be changed. const int ITERATION_PROGRESSION[] = { 256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000 }; -const int MAX_ITERATION = sizeof(ITERATION_PROGRESSION)/sizeof(int); +const int MAX_ITERATION = sizeof(ITERATION_PROGRESSION) / sizeof(int); -static Future runTest(Reference const& data, Reference const& db, +static Future runTest(Reference const& data, + Reference const& db, StringRef const& prefix); -THREAD_FUNC networkThread( void* api ) { +THREAD_FUNC networkThread(void* api) { // This is the fdb_flow network we're running on a thread ((API*)api)->runNetwork(); THREAD_RETURN; } -bool hasEnding(std::string const &fullString, std::string const &ending) -{ +bool hasEnding(std::string const& fullString, std::string const& ending) { if (fullString.length() >= ending.length()) { return (0 == fullString.compare(fullString.length() - ending.length(), ending.length(), ending)); - } - else { + } else { return false; } } -ACTOR Future> waitAndPop(FlowTesterStack *self, int count) { +ACTOR Future> waitAndPop(FlowTesterStack* self, int count) { state std::vector tuples; state std::vector items = self->pop(count); state int index; - for(index = 0; index < items.size(); ++index) { + for (index = 0; index < items.size(); ++index) { Standalone itemStr = wait(items[index].value); tuples.push_back(Tuple::unpack(itemStr)); } @@ -80,7 +79,7 @@ Future> FlowTesterStack::waitAndPop(int count) { return ::waitAndPop(this, count); } -ACTOR Future waitAndPop(FlowTesterStack *self) { +ACTOR Future waitAndPop(FlowTesterStack* self) { std::vector tuples = wait(waitAndPop(self, 1)); return tuples[0]; } @@ -91,41 +90,33 @@ Future FlowTesterStack::waitAndPop() { std::string tupleToString(Tuple const& tuple) { std::string str = "("; - for(int i = 0; i < tuple.size(); ++i) { + for (int i = 0; i < tuple.size(); ++i) { Tuple::ElementType type = tuple.getType(i); - if(type == Tuple::NULL_TYPE) { + if (type == Tuple::NULL_TYPE) { str += "NULL"; - } - else if(type == Tuple::BYTES || type == Tuple::UTF8) { - if(type == Tuple::UTF8) { + } else if (type == Tuple::BYTES || type == Tuple::UTF8) { + if (type == Tuple::UTF8) { str += "u"; } str += "\'" + tuple.getString(i).printable() + "\'"; - } - else if(type == Tuple::INT) { + } else if (type == Tuple::INT) { str += format("%ld", tuple.getInt(i)); - } - else if(type == Tuple::FLOAT) { + } else if (type == Tuple::FLOAT) { str += format("%f", tuple.getFloat(i)); - } - else if(type == Tuple::DOUBLE) { + } else if (type == Tuple::DOUBLE) { str += format("%f", tuple.getDouble(i)); - } - else if(type == Tuple::BOOL) { + } else if (type == Tuple::BOOL) { str += tuple.getBool(i) ? "true" : "false"; - } - else if(type == Tuple::UUID) { + } else if (type == Tuple::UUID) { Uuid u = tuple.getUuid(i); str += format("%016llx%016llx", *(uint64_t*)u.getData().begin(), *(uint64_t*)(u.getData().begin() + 8)); - } - else if(type == Tuple::NESTED) { + } else if (type == Tuple::NESTED) { str += tupleToString(tuple.getNested(i)); - } - else { + } else { ASSERT(false); } - if(i < tuple.size() - 1) { + if (i < tuple.size() - 1) { str += ", "; } } @@ -134,27 +125,41 @@ std::string tupleToString(Tuple const& tuple) { return str; } -ACTOR Future< Standalone > getRange(Reference tr, KeySelectorRef begin, KeySelectorRef end, int limits = 0, bool snapshot = false, bool reverse = false, FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { +ACTOR Future> getRange(Reference tr, + KeySelectorRef begin, + KeySelectorRef end, + int limits = 0, + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { state KeySelector ks_begin(begin); state KeySelector ks_end(end); state Standalone results; state int iteration = 1; - loop{ - // printf("=====DB: begin:%s, end:%s, limits:%d\n", printable(begin.key).c_str(), printable(end.key).c_str(), limits); + loop { + // printf("=====DB: begin:%s, end:%s, limits:%d\n", printable(begin.key).c_str(), printable(end.key).c_str(), + // limits); state FDBStandalone r; if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) { int effective_iteration = std::min(iteration, MAX_ITERATION); int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1]; - FDBStandalone rTemp = wait(tr->getRange(ks_begin, ks_end, GetRangeLimits(limits, bytes_limit), snapshot, reverse, (FDBStreamingMode)FDB_STREAMING_MODE_EXACT)); + FDBStandalone rTemp = wait(tr->getRange(ks_begin, + ks_end, + GetRangeLimits(limits, bytes_limit), + snapshot, + reverse, + (FDBStreamingMode)FDB_STREAMING_MODE_EXACT)); r = rTemp; } else { - FDBStandalone rTemp = wait(tr->getRange(ks_begin, ks_end, limits, snapshot, reverse, streamingMode)); + FDBStandalone rTemp = + wait(tr->getRange(ks_begin, ks_end, limits, snapshot, reverse, streamingMode)); r = rTemp; } iteration += 1; // printf("=====DB: count:%d\n", r.size()); - for (auto & s : r) { - // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); + for (auto& s : r) { + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); results.push_back_deep(results.arena(), s); if (reverse) @@ -169,34 +174,45 @@ ACTOR Future< Standalone > getRange(Reference tr, K return results; } - if(limits > 0) { + if (limits > 0) { limits -= r.size(); } } } -ACTOR Future< Standalone > getRange(Reference tr, KeyRange keys, int limits = 0, bool snapshot = false, bool reverse = false, FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { +ACTOR Future> getRange(Reference tr, + KeyRange keys, + int limits = 0, + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { state Key begin(keys.begin); state Key end(keys.end); state Standalone results; state int iteration = 1; - loop{ + loop { // printf("=====DB: begin:%s, limits:%d\n", printable(begin).c_str(), limits); KeyRange keyRange(KeyRangeRef(begin, end > begin ? end : begin)); state FDBStandalone r; if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) { int effective_iteration = std::min(iteration, MAX_ITERATION); int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1]; - FDBStandalone rTemp = wait(tr->getRange(keyRange, GetRangeLimits(limits, bytes_limit), snapshot, reverse, (FDBStreamingMode)FDB_STREAMING_MODE_EXACT)); + FDBStandalone rTemp = wait(tr->getRange(keyRange, + GetRangeLimits(limits, bytes_limit), + snapshot, + reverse, + (FDBStreamingMode)FDB_STREAMING_MODE_EXACT)); r = rTemp; } else { - FDBStandalone rTemp = wait(tr->getRange(keyRange, limits, snapshot, reverse, streamingMode)); + FDBStandalone rTemp = + wait(tr->getRange(keyRange, limits, snapshot, reverse, streamingMode)); r = rTemp; } iteration += 1; // printf("=====DB: count:%d\n", r.size()); - for (auto & s : r) { - // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); + for (auto& s : r) { + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); results.push_back_deep(results.arena(), s); if (reverse) @@ -211,18 +227,18 @@ ACTOR Future< Standalone > getRange(Reference tr, K return results; } - if(limits > 0) { + if (limits > 0) { limits -= r.size(); } } } -//ACTOR static Future debugPrintRange(Reference tr, std::string subspace, std::string msg) { +// ACTOR static Future debugPrintRange(Reference tr, std::string subspace, std::string msg) { // if (!tr) // return Void(); // -// Standalone results = wait(getRange(tr, KeyRange(KeyRangeRef(subspace + '\x00', subspace + '\xff')))); -// printf("==================================================DB:%s:%s, count:%d\n", msg.c_str(), +// Standalone results = wait(getRange(tr, KeyRange(KeyRangeRef(subspace + '\x00', subspace + +//'\xff')))); printf("==================================================DB:%s:%s, count:%d\n", msg.c_str(), // StringRef(subspace).printable().c_str(), results.size()); // for (auto & s : results) { // printf("=====key:%s, value:%s\n", StringRef(s.key).printable().c_str(), StringRef(s.value).printable().c_str()); @@ -252,7 +268,7 @@ ACTOR Future stackSub(FlowTesterStack* stack) { } ACTOR Future stackConcat(FlowTesterStack* stack) { - if(stack->data.size() < 2) + if (stack->data.size() < 2) return Void(); StackItem a = stack->data.back(); @@ -282,7 +298,7 @@ ACTOR Future stackSwap(FlowTesterStack* stack) { int64_t idx = stack->data.size() - 1; int64_t idx1 = idx - Tuple::unpack(sv).getInt(0); if (idx1 < idx) { - //printf("=============SWAP:%d,%d\n", idx, stack->data.size()); + // printf("=============SWAP:%d,%d\n", idx, stack->data.size()); StackItem item = stack->data[idx]; stack->data[idx] = stack->data[idx1]; stack->data[idx1] = item; @@ -295,7 +311,8 @@ ACTOR Future printFlowTesterStack(FlowTesterStack* stack) { state int idx; for (idx = stack->data.size() - 1; idx >= 0; --idx) { Standalone value = wait(stack->data[idx].value); - // printf("==========stack item:%d, index:%d, value:%s\n", idx, stack->data[idx].index, value.printable().c_str()); + // printf("==========stack item:%d, index:%d, value:%s\n", idx, stack->data[idx].index, + // value.printable().c_str()); } return Void(); } @@ -332,8 +349,8 @@ struct EmptyStackFunc : InstructionFunc { static const char* name; static Future call(Reference const& data, Reference const& instruction) { - //wait(printFlowTesterStack(&(data->stack))); - //wait(debugPrintRange(instruction->tr, "\x01test_results", "")); + // wait(printFlowTesterStack(&(data->stack))); + // wait(debugPrintRange(instruction->tr, "\x01test_results", "")); data->stack.clear(); return Void(); } @@ -357,7 +374,7 @@ struct PopFunc : InstructionFunc { ACTOR static Future call(Reference data, Reference instruction) { state std::vector items = data->stack.pop(); - for(StackItem item : items) { + for (StackItem item : items) { wait(success(item.value)); } return Void(); @@ -391,11 +408,13 @@ REGISTER_INSTRUCTION_FUNC(ConcatFunc); struct LogStackFunc : InstructionFunc { static const char* name; - ACTOR static Future logStack(Reference data, std::map entries, Standalone prefix) { + ACTOR static Future logStack(Reference data, + std::map entries, + Standalone prefix) { loop { state Reference tr = data->db->createTransaction(); try { - for(auto it : entries) { + for (auto it : entries) { Tuple tk; tk.append(it.first); tk.append((int64_t)it.second.index); @@ -406,8 +425,7 @@ struct LogStackFunc : InstructionFunc { wait(tr->commit()); return Void(); - } - catch(Error &e) { + } catch (Error& e) { wait(tr->onError(e)); } } @@ -422,11 +440,11 @@ struct LogStackFunc : InstructionFunc { state Standalone prefix = Tuple::unpack(s1).getString(0); state std::map entries; - while(data->stack.data.size() > 0) { + while (data->stack.data.size() > 0) { state std::vector it = data->stack.pop(); ASSERT(it.size() == 1); entries[data->stack.data.size()] = it.front(); - if(entries.size() == 100) { + if (entries.size() == 100) { wait(logStack(data, entries, prefix)); entries.clear(); } @@ -443,13 +461,12 @@ REGISTER_INSTRUCTION_FUNC(LogStackFunc); // FoundationDB Operations // ACTOR Future> waitForVoid(Future f) { - try{ + try { wait(f); Tuple t; t.append(LiteralStringRef("RESULT_NOT_PRESENT")); return t.pack(); - } - catch (Error& e){ + } catch (Error& e) { // printf("FDBError1:%d\n", e.code()); Tuple t; t.append(LiteralStringRef("ERROR")); @@ -462,13 +479,12 @@ ACTOR Future> waitForVoid(Future f) { } ACTOR Future> waitForValue(Future> f) { - try{ + try { FDBStandalone value = wait(f); Tuple t; t.append(value); return t.pack(); - } - catch (Error& e){ + } catch (Error& e) { // printf("FDBError2:%d\n", e.code()); Tuple t; t.append(LiteralStringRef("ERROR")); @@ -480,8 +496,8 @@ ACTOR Future> waitForValue(Future> f } } -ACTOR Future> waitForValue(Future< Optional> > f) { - try{ +ACTOR Future> waitForValue(Future>> f) { + try { Optional> value = wait(f); Standalone str; if (value.present()) @@ -492,8 +508,7 @@ ACTOR Future> waitForValue(Future< Optional> getKey(Future> f, Stan FDBStandalone key = wait(f); Tuple t; - if(key.startsWith(prefixFilter)) { + if (key.startsWith(prefixFilter)) { t.append(key); - } - else if(key < prefixFilter) { + } else if (key < prefixFilter) { t.append(prefixFilter); - } - else { + } else { t.append(strinc(prefixFilter)); } return t.pack(); - } - catch(Error& e){ + } catch (Error& e) { // printf("FDBError4:%d\n", e.code()); Tuple t; t.append(LiteralStringRef("ERROR")); @@ -553,7 +565,7 @@ struct UseTransactionFunc : InstructionFunc { Standalone name = wait(items[0].value); data->trName = name; - if(trMap.count(data->trName) == 0) { + if (trMap.count(data->trName) == 0) { trMap[data->trName] = data->db->createTransaction(); } return Void(); @@ -592,23 +604,22 @@ struct SetFunc : InstructionFunc { Standalone sk = wait(items[0].value); state Standalone key = Tuple::unpack(sk).getString(0); // if (instruction->isDatabase) - // printf("SetDatabase:%s, isDatabase:%d\n", printable(key).c_str(), instruction->isDatabase); + // printf("SetDatabase:%s, isDatabase:%d\n", printable(key).c_str(), instruction->isDatabase); Standalone sv = wait(items[1].value); Standalone value = Tuple::unpack(sv).getString(0); - //printf("SetDatabase:%s:%s:%s\n", printable(key).c_str(), printable(sv).c_str(), printable(value).c_str()); + // printf("SetDatabase:%s:%s:%s\n", printable(key).c_str(), printable(sv).c_str(), printable(value).c_str()); Reference instructionCopy = instruction; Standalone keyCopy = key; - Future mutation = executeMutation(instruction, [instructionCopy, keyCopy, value] () -> Future { + Future mutation = executeMutation(instruction, [instructionCopy, keyCopy, value]() -> Future { instructionCopy->tr->set(keyCopy, value); return Void(); }); if (instruction->isDatabase) { data->stack.push(waitForVoid(mutation)); - } - else { + } else { wait(mutation); } @@ -629,7 +640,7 @@ struct GetFunc : InstructionFunc { Standalone sk = wait(items[0].value); state Standalone key = Tuple::unpack(sk).getString(0); - Future< Optional> > fk = instruction->tr->get(StringRef(key), instruction->isSnapshot); + Future>> fk = instruction->tr->get(StringRef(key), instruction->isSnapshot); data->stack.push(waitForValue(holdWhile(instruction->tr, fk))); return Void(); @@ -681,8 +692,9 @@ struct GetKeyFunc : InstructionFunc { Standalone s4 = wait(items[3].value); Standalone prefix = Tuple::unpack(s4).getString(0); - //printf("===================GET_KEY:%s, %ld, %ld\n", printable(key).c_str(), or_equal, offset); - Future> fk = instruction->tr->getKey(KeySelector(KeySelectorRef(key, or_equal, offset)), instruction->isSnapshot); + // printf("===================GET_KEY:%s, %ld, %ld\n", printable(key).c_str(), or_equal, offset); + Future> fk = + instruction->tr->getKey(KeySelector(KeySelectorRef(key, or_equal, offset)), instruction->isSnapshot); data->stack.push(getKey(holdWhile(instruction->tr, fk), prefix)); return Void(); @@ -734,7 +746,7 @@ struct GetApproximateSizeFunc : InstructionFunc { ACTOR static Future call(Reference data, Reference instruction) { int64_t _ = wait(instruction->tr->getApproximateSize()); - (void) _; // disable unused variable warning + (void)_; // disable unused variable warning data->stack.pushTuple(LiteralStringRef("GOT_APPROXIMATE_SIZE")); return Void(); } @@ -797,15 +809,14 @@ struct ClearFunc : InstructionFunc { Reference instructionCopy = instruction; - Future mutation = executeMutation(instruction, [instructionCopy, key] () -> Future { + Future mutation = executeMutation(instruction, [instructionCopy, key]() -> Future { instructionCopy->tr->clear(key); return Void(); }); if (instruction->isDatabase) { data->stack.push(waitForVoid(mutation)); - } - else { + } else { wait(mutation); } @@ -862,16 +873,23 @@ struct GetRangeFunc : InstructionFunc { Standalone s5 = wait(items[4].value); FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s5).getInt(0); - // printf("================GetRange: %s, %s, %d, %d, %d, %d\n", printable(begin).c_str(), printable(end).c_str(), limit, reverse, mode, instruction->isSnapshot); + // printf("================GetRange: %s, %s, %d, %d, %d, %d\n", printable(begin).c_str(), + // printable(end).c_str(), limit, reverse, mode, instruction->isSnapshot); - Standalone results = wait(getRange(instruction->tr, KeyRange(KeyRangeRef(begin, end > begin ? end : begin)), limit, instruction->isSnapshot, reverse, mode)); + Standalone results = wait(getRange(instruction->tr, + KeyRange(KeyRangeRef(begin, end > begin ? end : begin)), + limit, + instruction->isSnapshot, + reverse, + mode)); Tuple t; - for (auto & s : results) { + for (auto& s : results) { t.append(s.key); t.append(s.value); - //printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); } - //printf("=====Results Count:%d, size:%d\n", results.size(), str.size()); + // printf("=====Results Count:%d, size:%d\n", results.size(), str.size()); data->stack.push(Tuple().append(t.pack()).pack()); return Void(); @@ -900,14 +918,21 @@ struct GetRangeStartsWithFunc : InstructionFunc { Standalone s4 = wait(items[3].value); FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s4).getInt(0); - //printf("================GetRangeStartsWithFunc: %s, %d, %d, %d, %d\n", printable(prefix).c_str(), limit, reverse, mode, isSnapshot); - Standalone results = wait(getRange(instruction->tr, KeyRange(KeyRangeRef(prefix, strinc(prefix))), limit, instruction->isSnapshot, reverse, mode)); + // printf("================GetRangeStartsWithFunc: %s, %d, %d, %d, %d\n", printable(prefix).c_str(), limit, + // reverse, mode, isSnapshot); + Standalone results = wait(getRange(instruction->tr, + KeyRange(KeyRangeRef(prefix, strinc(prefix))), + limit, + instruction->isSnapshot, + reverse, + mode)); Tuple t; - //printf("=====Results Count:%d\n", results.size()); - for (auto & s : results) { + // printf("=====Results Count:%d\n", results.size()); + for (auto& s : results) { t.append(s.key); t.append(s.value); - //printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); } data->stack.push(Tuple().append(t.pack()).pack()); @@ -934,15 +959,14 @@ struct ClearRangeFunc : InstructionFunc { Reference instructionCopy = instruction; Standalone beginCopy = begin; - Future mutation = executeMutation(instruction, [instructionCopy, beginCopy, end] () -> Future { + Future mutation = executeMutation(instruction, [instructionCopy, beginCopy, end]() -> Future { instructionCopy->tr->clear(KeyRangeRef(beginCopy, end)); return Void(); }); if (instruction->isDatabase) { data->stack.push(waitForVoid(mutation)); - } - else { + } else { wait(mutation); } @@ -965,15 +989,14 @@ struct ClearRangeStartWithFunc : InstructionFunc { Reference instructionCopy = instruction; - Future mutation = executeMutation(instruction, [instructionCopy, begin] () -> Future { + Future mutation = executeMutation(instruction, [instructionCopy, begin]() -> Future { instructionCopy->tr->clear(KeyRangeRef(begin, strinc(begin))); return Void(); }); if (instruction->isDatabase) { data->stack.push(waitForVoid(mutation)); - } - else { + } else { wait(mutation); } @@ -1016,27 +1039,34 @@ struct GetRangeSelectorFunc : InstructionFunc { state int reverse = Tuple::unpack(s8).getInt(0); Standalone s9 = wait(items[8].value); - state FDBStreamingMode mode = (FDBStreamingMode) Tuple::unpack(s9).getInt(0); + state FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s9).getInt(0); Standalone s10 = wait(items[9].value); state Optional> prefix; Tuple t10 = Tuple::unpack(s10); - if(t10.getType(0) != Tuple::ElementType::NULL_TYPE) { + if (t10.getType(0) != Tuple::ElementType::NULL_TYPE) { prefix = t10.getString(0); } - //printf("================GetRangeSelectorFunc: %s, %d, %ld, %s, %d, %ld, %d, %d, %d, %d, %s\n", printable(begin).c_str(), begin_or_equal, begin_offset, - // printable(end).c_str(), end_or_equal, end_offset, + // printf("================GetRangeSelectorFunc: %s, %d, %ld, %s, %d, %ld, %d, %d, %d, %d, %s\n", + // printable(begin).c_str(), begin_or_equal, begin_offset, printable(end).c_str(), end_or_equal, end_offset, // limit, reverse, mode, instruction->isSnapshot, printable(prefix).c_str()); - Future> f = getRange(instruction->tr, KeySelectorRef(begin, begin_or_equal, begin_offset), KeySelectorRef(end, end_or_equal, end_offset), limit, instruction->isSnapshot, reverse, mode); + Future> f = getRange(instruction->tr, + KeySelectorRef(begin, begin_or_equal, begin_offset), + KeySelectorRef(end, end_or_equal, end_offset), + limit, + instruction->isSnapshot, + reverse, + mode); Standalone results = wait(holdWhile(instruction->tr, f)); Tuple t; - //printf("=====Results Count:%d\n", results.size()); - for (auto & s : results) { - if(!prefix.present() || s.key.startsWith(prefix.get())) { + // printf("=====Results Count:%d\n", results.size()); + for (auto& s : results) { + if (!prefix.present() || s.key.startsWith(prefix.get())) { t.append(s.key); t.append(s.value); - //printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), printable(StringRef(s.value)).c_str()); + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); } } @@ -1069,40 +1099,30 @@ struct TuplePackFunc : InstructionFunc { for (; i < items1.size(); ++i) { Standalone str = wait(items1[i].value); Tuple itemTuple = Tuple::unpack(str); - if(deterministicRandom()->coinflip()) { + if (deterministicRandom()->coinflip()) { Tuple::ElementType type = itemTuple.getType(0); - if(type == Tuple::NULL_TYPE) { + if (type == Tuple::NULL_TYPE) { tuple.appendNull(); - } - else if(type == Tuple::INT) { + } else if (type == Tuple::INT) { tuple << itemTuple.getInt(0); - } - else if(type == Tuple::BYTES) { + } else if (type == Tuple::BYTES) { tuple.append(itemTuple.getString(0), false); - } - else if(type == Tuple::UTF8) { + } else if (type == Tuple::UTF8) { tuple.append(itemTuple.getString(0), true); - } - else if(type == Tuple::FLOAT) { + } else if (type == Tuple::FLOAT) { tuple << itemTuple.getFloat(0); - } - else if(type == Tuple::DOUBLE) { + } else if (type == Tuple::DOUBLE) { tuple << itemTuple.getDouble(0); - } - else if(type == Tuple::BOOL) { + } else if (type == Tuple::BOOL) { tuple << itemTuple.getBool(0); - } - else if(type == Tuple::UUID) { + } else if (type == Tuple::UUID) { tuple << itemTuple.getUuid(0); - } - else if(type == Tuple::NESTED) { + } else if (type == Tuple::NESTED) { tuple.appendNested(itemTuple.getNested(0)); - } - else { + } else { ASSERT(false); } - } - else { + } else { tuple << itemTuple; } } @@ -1128,7 +1148,7 @@ struct TupleUnpackFunc : InstructionFunc { for (int i = 0; i < t.size(); ++i) { Standalone str = t.subTuple(i, i + 1).pack(); - //printf("=====value:%s\n", printable(str).c_str()); + // printf("=====value:%s\n", printable(str).c_str()); data->stack.pushTuple(str); } return Void(); @@ -1158,40 +1178,30 @@ struct TupleRangeFunc : InstructionFunc { for (; i < items1.size(); ++i) { Standalone str = wait(items1[i].value); Tuple itemTuple = Tuple::unpack(str); - if(deterministicRandom()->coinflip()) { + if (deterministicRandom()->coinflip()) { Tuple::ElementType type = itemTuple.getType(0); - if(type == Tuple::NULL_TYPE) { + if (type == Tuple::NULL_TYPE) { tuple.appendNull(); - } - else if(type == Tuple::INT) { + } else if (type == Tuple::INT) { tuple << itemTuple.getInt(0); - } - else if(type == Tuple::BYTES) { + } else if (type == Tuple::BYTES) { tuple.append(itemTuple.getString(0), false); - } - else if(type == Tuple::UTF8) { + } else if (type == Tuple::UTF8) { tuple.append(itemTuple.getString(0), true); - } - else if(type == Tuple::FLOAT) { + } else if (type == Tuple::FLOAT) { tuple << itemTuple.getFloat(0); - } - else if(type == Tuple::DOUBLE) { + } else if (type == Tuple::DOUBLE) { tuple << itemTuple.getDouble(0); - } - else if(type == Tuple::BOOL) { + } else if (type == Tuple::BOOL) { tuple << itemTuple.getBool(0); - } - else if(type == Tuple::UUID) { + } else if (type == Tuple::UUID) { tuple << itemTuple.getUuid(0); - } - else if(type == Tuple::NESTED) { + } else if (type == Tuple::NESTED) { tuple.appendNested(itemTuple.getNested(0)); - } - else { + } else { ASSERT(false); } - } - else { + } else { tuple << itemTuple; } } @@ -1224,13 +1234,13 @@ struct TupleSortFunc : InstructionFunc { state std::vector tuples; state size_t i = 0; - for(; i < items1.size(); i++) { + for (; i < items1.size(); i++) { Standalone value = wait(items1[i].value); tuples.push_back(Tuple::unpack(value)); } std::sort(tuples.begin(), tuples.end()); - for(Tuple const& t : tuples) { + for (Tuple const& t : tuples) { data->stack.push(t.pack()); } @@ -1536,16 +1546,17 @@ struct AtomicOPFunc : InstructionFunc { Standalone keyCopy = key; Standalone valueCopy = value; - // printf("=========ATOMIC_OP:%s:%s:%s\n", printable(op).c_str(), printable(key).c_str(), printable(value).c_str()); - Future mutation = executeMutation(instruction, [instructionCopy, keyCopy, valueCopy, atomicOp] () -> Future { - instructionCopy->tr->atomicOp(keyCopy, valueCopy, atomicOp); - return Void(); - }); + // printf("=========ATOMIC_OP:%s:%s:%s\n", printable(op).c_str(), printable(key).c_str(), + // printable(value).c_str()); + Future mutation = + executeMutation(instruction, [instructionCopy, keyCopy, valueCopy, atomicOp]() -> Future { + instructionCopy->tr->atomicOp(keyCopy, valueCopy, atomicOp); + return Void(); + }); if (instruction->isDatabase) { data->stack.push(waitForVoid(mutation)); - } - else { + } else { wait(mutation); } @@ -1564,27 +1575,25 @@ struct UnitTestsFunc : InstructionFunc { ASSERT(!data->api->evaluatePredicate(FDBErrorPredicate::FDB_ERROR_PREDICATE_RETRYABLE, Error(10))); ASSERT(API::isAPIVersionSelected()); - state API *fdb = API::getInstance(); + state API* fdb = API::getInstance(); ASSERT(fdb->getAPIVersion() <= FDB_API_VERSION); try { API::selectAPIVersion(fdb->getAPIVersion() + 1); ASSERT(false); - } - catch(Error &e) { + } catch (Error& e) { ASSERT(e.code() == error_code_api_version_already_set); } try { API::selectAPIVersion(fdb->getAPIVersion() - 1); ASSERT(false); - } - catch(Error &e) { + } catch (Error& e) { ASSERT(e.code() == error_code_api_version_already_set); } API::selectAPIVersion(fdb->getAPIVersion()); const uint64_t locationCacheSize = 100001; const uint64_t maxWatches = 10001; - const uint64_t timeout = 60*1000; + const uint64_t timeout = 60 * 1000; const uint64_t noTimeout = 0; const uint64_t retryLimit = 50; const uint64_t noRetryLimit = -1; @@ -1592,19 +1601,30 @@ struct UnitTestsFunc : InstructionFunc { const uint64_t sizeLimit = 100000; const uint64_t maxFieldLength = 1000; - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_LOCATION_CACHE_SIZE, Optional(StringRef((const uint8_t*)&locationCacheSize, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MAX_WATCHES, Optional(StringRef((const uint8_t*)&maxWatches, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_DATACENTER_ID, Optional(LiteralStringRef("dc_id"))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MACHINE_ID, Optional(LiteralStringRef("machine_id"))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_LOCATION_CACHE_SIZE, + Optional(StringRef((const uint8_t*)&locationCacheSize, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MAX_WATCHES, + Optional(StringRef((const uint8_t*)&maxWatches, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_DATACENTER_ID, + Optional(LiteralStringRef("dc_id"))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MACHINE_ID, + Optional(LiteralStringRef("machine_id"))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_ENABLE); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_DISABLE); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, Optional(StringRef((const uint8_t*)&maxFieldLength, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, Optional(StringRef((const uint8_t*)&timeout, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, Optional(StringRef((const uint8_t*)&noTimeout, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_MAX_RETRY_DELAY, Optional(StringRef((const uint8_t*)&maxRetryDelay, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, Optional(StringRef((const uint8_t*)&sizeLimit, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, Optional(StringRef((const uint8_t*)&retryLimit, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, Optional(StringRef((const uint8_t*)&noRetryLimit, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, + Optional(StringRef((const uint8_t*)&maxFieldLength, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, + Optional(StringRef((const uint8_t*)&timeout, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, + Optional(StringRef((const uint8_t*)&noTimeout, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_MAX_RETRY_DELAY, + Optional(StringRef((const uint8_t*)&maxRetryDelay, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, + Optional(StringRef((const uint8_t*)&sizeLimit, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, + Optional(StringRef((const uint8_t*)&retryLimit, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, + Optional(StringRef((const uint8_t*)&noRetryLimit, 8))); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_CAUSAL_READ_RISKY); data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_INCLUDE_PORT_IN_ADDRESS); @@ -1617,22 +1637,26 @@ struct UnitTestsFunc : InstructionFunc { tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_SYSTEM_KEYS); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, Optional(StringRef((const uint8_t*)&maxFieldLength, 8))); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TIMEOUT, Optional(StringRef((const uint8_t*)&timeout, 8))); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_RETRY_LIMIT, Optional(StringRef((const uint8_t*)&retryLimit, 8))); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_MAX_RETRY_DELAY, Optional(StringRef((const uint8_t*)&maxRetryDelay, 8))); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, + Optional(StringRef((const uint8_t*)&maxFieldLength, 8))); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TIMEOUT, + Optional(StringRef((const uint8_t*)&timeout, 8))); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_RETRY_LIMIT, + Optional(StringRef((const uint8_t*)&retryLimit, 8))); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_MAX_RETRY_DELAY, + Optional(StringRef((const uint8_t*)&maxRetryDelay, 8))); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_USED_DURING_COMMIT_PROTECTION_DISABLE); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_ENABLE, Optional(LiteralStringRef("my_transaction"))); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_ENABLE, + Optional(LiteralStringRef("my_transaction"))); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_LOCK_AWARE); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_LOCK_AWARE); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_INCLUDE_PORT_IN_ADDRESS); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_REPORT_CONFLICTING_KEYS); - Optional > _ = wait(tr->get(LiteralStringRef("\xff"))); + Optional> _ = wait(tr->get(LiteralStringRef("\xff"))); tr->cancel(); return Void(); - } }; const char* UnitTestsFunc::name = "UNIT_TESTS"; @@ -1649,8 +1673,7 @@ ACTOR static Future getInstructions(Reference data, String Standalone results = wait(getRange(tr, testSpec.range())); data->instructions = results; return Void(); - } - catch(Error &e) { + } catch (Error& e) { wait(tr->onError(e)); } } @@ -1668,8 +1691,8 @@ ACTOR static Future doInstructions(Reference data) { state bool isDirectory = op.startsWith(LiteralStringRef("DIRECTORY_")); try { - if(LOG_INSTRUCTIONS) { - if(op != LiteralStringRef("SWAP") && op != LiteralStringRef("PUSH")) { + if (LOG_INSTRUCTIONS) { + if (op != LiteralStringRef("SWAP") && op != LiteralStringRef("PUSH")) { printf("%zu. %s\n", idx, tupleToString(opTuple).c_str()); fflush(stdout); } @@ -1681,18 +1704,19 @@ ACTOR static Future doInstructions(Reference data) { op = op.substr(0, op.size() - 9); // printf("[==========]%ld/%ld:%s:%s: isDatabase:%d, isSnapshot:%d, stack count:%ld\n", - // idx, data->instructions.size(), StringRef(data->instructions[idx].key).printable().c_str(), StringRef(data->instructions[idx].value).printable().c_str(), - // isDatabase, isSnapshot, data->stack.data.size()); + // idx, data->instructions.size(), StringRef(data->instructions[idx].key).printable().c_str(), + // StringRef(data->instructions[idx].value).printable().c_str(), isDatabase, isSnapshot, + // data->stack.data.size()); - //wait(printFlowTesterStack(&(data->stack))); - //wait(debugPrintRange(instruction->tr, "\x01test_results", "")); + // wait(printFlowTesterStack(&(data->stack))); + // wait(debugPrintRange(instruction->tr, "\x01test_results", "")); - state Reference instruction = Reference(new InstructionData(isDatabase, isSnapshot, data->instructions[idx].value, Reference())); + state Reference instruction = Reference( + new InstructionData(isDatabase, isSnapshot, data->instructions[idx].value, Reference())); if (isDatabase) { state Reference tr = data->db->createTransaction(); instruction->tr = tr; - } - else { + } else { instruction->tr = trMap[data->trName]; } @@ -1701,20 +1725,18 @@ ACTOR static Future doInstructions(Reference data) { data->stack.index = idx; wait(InstructionFunc::call(op.toString(), data, instruction)); - } - catch (Error& e) { - if(LOG_ERRORS) { + } catch (Error& e) { + if (LOG_ERRORS) { printf("Error: %s (%d)\n", e.name(), e.code()); fflush(stdout); } - if(isDirectory) { - if(opsThatCreateDirectories.count(op.toString())) { + if (isDirectory) { + if (opsThatCreateDirectories.count(op.toString())) { data->directoryData.directoryList.push_back(DirectoryOrSubspace()); } data->stack.pushTuple(LiteralStringRef("DIRECTORY_ERROR")); - } - else { + } else { data->stack.pushError(e.code()); } } @@ -1730,8 +1752,7 @@ ACTOR static Future runTest(Reference data, ReferencesubThreads)); - } - catch (Error& e) { + } catch (Error& e) { TraceEvent(SevError, "FlowTesterDataRunError").error(e); } @@ -1778,15 +1799,14 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi try { API::getInstance(); ASSERT(false); - } - catch(Error& e) { + } catch (Error& e) { ASSERT(e.code() == error_code_api_version_unset); } - API *fdb = API::selectAPIVersion(apiVersion); + API* fdb = API::selectAPIVersion(apiVersion); ASSERT(API::isAPIVersionSelected()); ASSERT(fdb->getAPIVersion() == apiVersion); - //fdb->setNetworkOption(FDBNetworkOption::FDB_NET_OPTION_TRACE_ENABLE); + // fdb->setNetworkOption(FDBNetworkOption::FDB_NET_OPTION_TRACE_ENABLE); // We have to start the fdb_flow network and thread separately! fdb->setupNetwork(); @@ -1801,23 +1821,21 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi // Stopping the network returns from g_network->run() and allows // the program to terminate g_network->stop(); - } - catch(Error &e) { + } catch (Error& e) { TraceEvent("ErrorRunningTest").error(e); - if(LOG_ERRORS) { + if (LOG_ERRORS) { printf("Flow tester encountered error: %s\n", e.name()); fflush(stdout); } flushAndExit(1); } - } ACTOR void _test_versionstamp() { try { g_network = newNet2(TLSConfig()); - API *fdb = FDB::API::selectAPIVersion(630); + API* fdb = FDB::API::selectAPIVersion(630); fdb->setupNetwork(); startThread(networkThread, fdb); @@ -1827,7 +1845,9 @@ ACTOR void _test_versionstamp() { state Future> ftrVersion = tr->getVersionstamp(); - tr->atomicOp(LiteralStringRef("foo"), LiteralStringRef("blahblahbl\x00\x00\x00\x00"), FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE); + tr->atomicOp(LiteralStringRef("foo"), + LiteralStringRef("blahblahbl\x00\x00\x00\x00"), + FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE); wait(tr->commit()); // should use retry loop @@ -1842,8 +1862,7 @@ ACTOR void _test_versionstamp() { fprintf(stderr, "%s\n", trVersion.printable().c_str()); g_network->stop(); - } - catch (Error &e) { + } catch (Error& e) { TraceEvent("ErrorRunningTest").error(e); if (LOG_ERRORS) { printf("Flow tester encountered error: %s\n", e.name()); @@ -1853,7 +1872,7 @@ ACTOR void _test_versionstamp() { } } -int main( int argc, char** argv ) { +int main(int argc, char** argv) { try { platformInit(); registerCrashHandler(); @@ -1884,13 +1903,11 @@ int main( int argc, char** argv ) { g_network->run(); flushAndExit(FDB_EXIT_SUCCESS); - } - catch (Error& e) { + } catch (Error& e) { fprintf(stderr, "Error: %s\n", e.name()); TraceEvent(SevError, "MainError").error(e); flushAndExit(FDB_EXIT_MAIN_ERROR); - } - catch (std::exception& e) { + } catch (std::exception& e) { fprintf(stderr, "std::exception: %s\n", e.what()); TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); flushAndExit(FDB_EXIT_MAIN_EXCEPTION); diff --git a/bindings/flow/tester/Tester.actor.h b/bindings/flow/tester/Tester.actor.h index 6f0596684f..df1d0f5551 100644 --- a/bindings/flow/tester/Tester.actor.h +++ b/bindings/flow/tester/Tester.actor.h @@ -18,12 +18,13 @@ * limitations under the License. */ -// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version. +// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source +// version. #if defined(NO_INTELLISENSE) && !defined(FDB_FLOW_TESTER_TESTER_ACTOR_G_H) - #define FDB_FLOW_TESTER_TESTER_ACTOR_G_H - #include "Tester.actor.g.h" +#define FDB_FLOW_TESTER_TESTER_ACTOR_G_H +#include "Tester.actor.g.h" #elif !defined(FDB_FLOW_TESTER_TESTER_ACTOR_H) - #define FDB_FLOW_TESTER_TESTER_ACTOR_H +#define FDB_FLOW_TESTER_TESTER_ACTOR_H #pragma once @@ -34,7 +35,7 @@ #include "bindings/flow/IDirectory.h" #include "bindings/flow/Subspace.h" #include "bindings/flow/DirectoryLayer.h" -#include "flow/actorcompiler.h" // This must be the last #include. +#include "flow/actorcompiler.h" // This must be the last #include. constexpr bool LOG_ALL = false; constexpr bool LOG_INSTRUCTIONS = LOG_ALL || false; @@ -56,19 +57,13 @@ struct FlowTesterStack { uint32_t index; std::vector data; - void push(Future> value) { - data.push_back(StackItem(index, value)); - } + void push(Future> value) { data.push_back(StackItem(index, value)); } - void push(Standalone value) { - push(Future>(value)); - } + void push(Standalone value) { push(Future>(value)); } - void push(const StackItem& item) { - data.push_back(item); - } + void push(const StackItem& item) { data.push_back(item); } - void pushTuple(StringRef value, bool utf8=false) { + void pushTuple(StringRef value, bool utf8 = false) { FDB::Tuple t; t.append(value, utf8); data.push_back(StackItem(index, t.pack())); @@ -101,9 +96,7 @@ struct FlowTesterStack { data.push_back(data.back()); } - void clear() { - data.clear(); - } + void clear() { data.clear(); } }; struct InstructionData : public ReferenceCounted { @@ -113,21 +106,21 @@ struct InstructionData : public ReferenceCounted { Reference tr; InstructionData(bool _isDatabase, bool _isSnapshot, StringRef _instruction, Reference _tr) - : isDatabase(_isDatabase) - , isSnapshot(_isSnapshot) - , instruction(_instruction) - , tr(_tr) {} + : isDatabase(_isDatabase), isSnapshot(_isSnapshot), instruction(_instruction), tr(_tr) {} }; struct FlowTesterData; -struct InstructionFunc : IDispatched(Reference data, Reference instruction)>> { +struct InstructionFunc + : IDispatched(Reference data, Reference instruction)>> { static Future call(std::string op, Reference data, Reference instruction) { ASSERT(data); ASSERT(instruction); auto it = dispatches().find(op); - if(it == dispatches().end()) { + if (it == dispatches().end()) { fprintf(stderr, "Unrecognized instruction: %s\n", op.c_str()); ASSERT(false); } @@ -143,24 +136,20 @@ struct DirectoryOrSubspace { DirectoryOrSubspace() {} DirectoryOrSubspace(Reference directory) : directory(directory) {} - DirectoryOrSubspace(FDB::Subspace *subspace) : subspace(subspace) {} - DirectoryOrSubspace(Reference dirSubspace) : directory(dirSubspace), subspace(dirSubspace.getPtr()) {} + DirectoryOrSubspace(FDB::Subspace* subspace) : subspace(subspace) {} + DirectoryOrSubspace(Reference dirSubspace) + : directory(dirSubspace), subspace(dirSubspace.getPtr()) {} - bool valid() { - return directory.present() || subspace.present(); - } + bool valid() { return directory.present() || subspace.present(); } std::string typeString() { - if(directory.present() && subspace.present()) { + if (directory.present() && subspace.present()) { return "DirectorySubspace"; - } - else if(directory.present()) { + } else if (directory.present()) { return "IDirectory"; - } - else if(subspace.present()) { + } else if (subspace.present()) { return "Subspace"; - } - else { + } else { return "InvalidDirectory"; } } @@ -190,8 +179,8 @@ struct DirectoryTesterData { template void push(T item) { directoryList.push_back(DirectoryOrSubspace(item)); - if(LOG_DIRS) { - printf("Pushed %s at %lu\n", directoryList.back().typeString().c_str(), directoryList.size()-1); + if (LOG_DIRS) { + printf("Pushed %s at %lu\n", directoryList.back().typeString().c_str(), directoryList.size() - 1); fflush(stdout); } } @@ -200,7 +189,7 @@ struct DirectoryTesterData { }; struct FlowTesterData : public ReferenceCounted { - FDB::API *api; + FDB::API* api; Reference db; Standalone instructions; Standalone trName; @@ -211,12 +200,11 @@ struct FlowTesterData : public ReferenceCounted { std::vector> subThreads; Future processInstruction(Reference instruction) { - return InstructionFunc::call(instruction->instruction.toString(), Reference::addRef(this), instruction); + return InstructionFunc::call( + instruction->instruction.toString(), Reference::addRef(this), instruction); } - FlowTesterData(FDB::API *api) { - this->api = api; - } + FlowTesterData(FDB::API* api) { this->api = api; } }; std::string tupleToString(FDB::Tuple const& tuple); @@ -226,16 +214,14 @@ Future()().getValue())> executeMutation(Reference()().getValue()) result = wait(func()); - if(instruction->isDatabase) { + if (instruction->isDatabase) { wait(instruction->tr->commit()); } return result; - } - catch(Error &e) { - if(instruction->isDatabase) { + } catch (Error& e) { + if (instruction->isDatabase) { wait(instruction->tr->onError(e)); - } - else { + } else { throw; } } diff --git a/bindings/java/JavaWorkload.cpp b/bindings/java/JavaWorkload.cpp index 7b45d808a2..50dbccf304 100644 --- a/bindings/java/JavaWorkload.cpp +++ b/bindings/java/JavaWorkload.cpp @@ -280,7 +280,8 @@ struct JVM { w.name = name; w.signature = sig; w.fnPtr = std::get<2>(t); - log->trace(info, "PreparedNativeMethod", + log->trace(info, + "PreparedNativeMethod", { { "Name", w.name }, { "Signature", w.signature }, { "Ptr", std::to_string(reinterpret_cast(w.fnPtr)) } }); @@ -362,7 +363,8 @@ struct JVM { { "getOption", "(JLjava/lang/String;Z)Z", reinterpret_cast(&getOptionBool) }, { "getOption", "(JLjava/lang/String;J)J", reinterpret_cast(&getOptionLong) }, { "getOption", "(JLjava/lang/String;D)D", reinterpret_cast(&getOptionDouble) }, - { "getOption", "(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;", + { "getOption", + "(JLjava/lang/String;Ljava/lang/String;)Ljava/lang/String;", reinterpret_cast(&getOptionString) }, { "getClientID", "(J)I", reinterpret_cast(&getClientID) }, { "getClientCount", "(J)I", reinterpret_cast(&getClientCount) }, @@ -391,7 +393,8 @@ struct JVM { auto impl = env->GetLongField(res, field); checkException(); if (impl != jContext) { - log->trace(error, "ContextNotCorrect", + log->trace(error, + "ContextNotCorrect", { { "Expected", std::to_string(jContext) }, { "Impl", std::to_string(impl) } }); std::terminate(); } @@ -471,14 +474,16 @@ struct JVM { } jobject createDatabase(jobject workload, FDBDatabase* db) { - auto executor = - env->CallObjectMethod(workload, getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"), - "getExecutor", "()Ljava/util/concurrent/Executor;")); + auto executor = env->CallObjectMethod(workload, + getMethod(getClass("com/apple/foundationdb/testing/AbstractWorkload"), + "getExecutor", + "()Ljava/util/concurrent/Executor;")); auto databaseClass = getClass("com/apple/foundationdb/FDBDatabase"); jlong databasePtr = reinterpret_cast(db); - jobject javaDatabase = - env->NewObject(databaseClass, getMethod(databaseClass, "", "(JLjava/util/concurrent/Executor;)V"), - databasePtr, executor); + jobject javaDatabase = env->NewObject(databaseClass, + getMethod(databaseClass, "", "(JLjava/util/concurrent/Executor;)V"), + databasePtr, + executor); env->DeleteLocalRef(executor); return javaDatabase; } @@ -491,9 +496,10 @@ struct JVM { jPromise = createPromise(std::move(promise)); env->CallVoidMethod( workload, - getMethod(clazz, method, - "(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"), - jdb, jPromise); + getMethod( + clazz, method, "(Lcom/apple/foundationdb/Database;Lcom/apple/foundationdb/testing/Promise;)V"), + jdb, + jPromise); env->DeleteLocalRef(jdb); env->DeleteLocalRef(jPromise); jPromise = nullptr; @@ -515,7 +521,7 @@ struct JavaWorkload : FDBWorkload { bool failed = false; jobject workload = nullptr; JavaWorkload(const std::shared_ptr& jvm, FDBLogger& log, const std::string& name) - : jvm(jvm), log(log), name(name) { + : jvm(jvm), log(log), name(name) { boost::replace_all(this->name, ".", "/"); } ~JavaWorkload() { @@ -588,9 +594,7 @@ struct JavaWorkload : FDBWorkload { log.trace(error, "CheckFailedWithJNIError", { { "Error", e.toString() }, { "Location", e.location() } }); } } - void getMetrics(std::vector& out) const override { - jvm->getMetrics(workload, name, out); - } + void getMetrics(std::vector& out) const override { jvm->getMetrics(workload, name, out); } }; struct JavaWorkloadFactory : FDBWorkloadFactory { diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp index 15dd51154e..378ecb3790 100644 --- a/bindings/java/fdbJNI.cpp +++ b/bindings/java/fdbJNI.cpp @@ -25,15 +25,16 @@ #include -#define JNI_NULL nullptr +#define JNI_NULL nullptr #if defined(__GNUG__) #undef JNIEXPORT -#define JNIEXPORT __attribute__ ((visibility ("default"))) +#define JNIEXPORT __attribute__((visibility("default"))) #endif -static JavaVM* g_jvm = nullptr; -static thread_local JNIEnv* g_thread_jenv = nullptr; // Defined for the network thread once it is running, and for any thread that has called registerCallback +static JavaVM* g_jvm = nullptr; +static thread_local JNIEnv* g_thread_jenv = + nullptr; // Defined for the network thread once it is running, and for any thread that has called registerCallback static thread_local jmethodID g_IFutureCallback_call_methodID = JNI_NULL; static thread_local bool is_external = false; static jclass range_result_summary_class; @@ -42,23 +43,23 @@ static jclass string_class; static jmethodID range_result_init; static jmethodID range_result_summary_init; -void detachIfExternalThread(void *ignore) { - if(is_external && g_thread_jenv != nullptr) { +void detachIfExternalThread(void* ignore) { + if (is_external && g_thread_jenv != nullptr) { g_thread_jenv = nullptr; g_IFutureCallback_call_methodID = JNI_NULL; g_jvm->DetachCurrentThread(); } } -void throwOutOfMem(JNIEnv *jenv) { - const char *className = "java/lang/OutOfMemoryError"; - jclass illegalArgClass = jenv->FindClass( className ); +void throwOutOfMem(JNIEnv* jenv) { + const char* className = "java/lang/OutOfMemoryError"; + jclass illegalArgClass = jenv->FindClass(className); - if(jenv->ExceptionOccurred()) + if (jenv->ExceptionOccurred()) return; - if( jenv->ThrowNew( illegalArgClass, nullptr ) != 0 ) { - if( !jenv->ExceptionOccurred() ) { + if (jenv->ThrowNew(illegalArgClass, nullptr) != 0) { + if (!jenv->ExceptionOccurred()) { jenv->FatalError("Could not throw OutOfMemoryError"); } else { // This means that an exception is pending. We do not know what it is, but are sure @@ -67,45 +68,45 @@ void throwOutOfMem(JNIEnv *jenv) { } } -static jthrowable getThrowable(JNIEnv *jenv, fdb_error_t e, const char* msg = nullptr) { +static jthrowable getThrowable(JNIEnv* jenv, fdb_error_t e, const char* msg = nullptr) { jclass excepClass = jenv->FindClass("com/apple/foundationdb/FDBException"); - if(jenv->ExceptionOccurred()) + if (jenv->ExceptionOccurred()) return JNI_NULL; jmethodID excepCtor = jenv->GetMethodID(excepClass, "", "(Ljava/lang/String;I)V"); - if(jenv->ExceptionOccurred()) + if (jenv->ExceptionOccurred()) return JNI_NULL; - const char *fdb_message = msg ? msg : fdb_get_error(e); + const char* fdb_message = msg ? msg : fdb_get_error(e); jstring m = jenv->NewStringUTF(fdb_message); - if(jenv->ExceptionOccurred()) + if (jenv->ExceptionOccurred()) return JNI_NULL; jthrowable t = (jthrowable)jenv->NewObject(excepClass, excepCtor, m, e); - if(jenv->ExceptionOccurred()) + if (jenv->ExceptionOccurred()) return JNI_NULL; return t; } -void throwNamedException(JNIEnv *jenv, const char *class_full_name, const char* message ) { - jclass exceptionClass = jenv->FindClass( class_full_name ); - if(jenv->ExceptionOccurred()) +void throwNamedException(JNIEnv* jenv, const char* class_full_name, const char* message) { + jclass exceptionClass = jenv->FindClass(class_full_name); + if (jenv->ExceptionOccurred()) return; - if( jenv->ThrowNew( exceptionClass, message ) != 0 ) { - if(jenv->ExceptionOccurred()) + if (jenv->ThrowNew(exceptionClass, message) != 0) { + if (jenv->ExceptionOccurred()) return; jenv->FatalError("FDB: Error throwing exception"); } } -void throwRuntimeEx(JNIEnv *jenv, const char* message) { - throwNamedException( jenv, "java/lang/RuntimeException", message ); +void throwRuntimeEx(JNIEnv* jenv, const char* message) { + throwNamedException(jenv, "java/lang/RuntimeException", message); } -void throwParamNotNull(JNIEnv *jenv) { - throwNamedException( jenv, "java/lang/IllegalArgumentException", "Argument cannot be null" ); +void throwParamNotNull(JNIEnv* jenv) { + throwNamedException(jenv, "java/lang/IllegalArgumentException", "Argument cannot be null"); } #ifdef __cplusplus @@ -114,25 +115,25 @@ extern "C" { // If the methods are not found, exceptions are thrown and this will return false. // Returns TRUE on success, false otherwise. -static bool findCallbackMethods(JNIEnv *jenv) { +static bool findCallbackMethods(JNIEnv* jenv) { jclass cls = jenv->FindClass("java/lang/Runnable"); - if(jenv->ExceptionOccurred()) + if (jenv->ExceptionOccurred()) return false; g_IFutureCallback_call_methodID = jenv->GetMethodID(cls, "run", "()V"); - if(jenv->ExceptionOccurred()) + if (jenv->ExceptionOccurred()) return false; return true; } -static void callCallback( FDBFuture* f, void* data ) { +static void callCallback(FDBFuture* f, void* data) { if (g_thread_jenv == nullptr) { // We are on an external thread and must attach to the JVM. // The shutdown hook will later detach this thread. is_external = true; - if( g_jvm != nullptr && g_jvm->AttachCurrentThreadAsDaemon((void **) &g_thread_jenv, nullptr) == JNI_OK ) { - if( !findCallbackMethods( g_thread_jenv ) ) { + if (g_jvm != nullptr && g_jvm->AttachCurrentThreadAsDaemon((void**)&g_thread_jenv, nullptr) == JNI_OK) { + if (!findCallbackMethods(g_thread_jenv)) { g_thread_jenv->FatalError("FDB: Could not find callback method.\n"); } } else { @@ -143,117 +144,128 @@ static void callCallback( FDBFuture* f, void* data ) { } jobject callback = (jobject)data; - g_thread_jenv->CallVoidMethod( callback, g_IFutureCallback_call_methodID ); + g_thread_jenv->CallVoidMethod(callback, g_IFutureCallback_call_methodID); g_thread_jenv->DeleteGlobalRef(callback); } // Attempts to throw 't', attempts to shut down the JVM if this fails. -void safeThrow( JNIEnv *jenv, jthrowable t ) { - if( jenv->Throw( t ) != 0 ) { +void safeThrow(JNIEnv* jenv, jthrowable t) { + if (jenv->Throw(t) != 0) { jenv->FatalError("FDB: Unable to throw exception"); } } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1registerCallback(JNIEnv *jenv, jobject cls, jlong future, jobject callback) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1registerCallback(JNIEnv* jenv, + jobject cls, + jlong future, + jobject callback) { // SOMEDAY: Do this on module load instead. Can we cache method ids across threads? - if( !g_IFutureCallback_call_methodID ) { - if( !findCallbackMethods( jenv ) ) { + if (!g_IFutureCallback_call_methodID) { + if (!findCallbackMethods(jenv)) { return; } } - if( !future || !callback ) { + if (!future || !callback) { throwParamNotNull(jenv); return; } - FDBFuture *f = (FDBFuture *)future; + FDBFuture* f = (FDBFuture*)future; // This is documented as not throwing, but simply returning null on OOM. // As belt and suspenders, we will check for pending exceptions and then, // if there are none and the result is null, we'll throw our own OOM. - callback = jenv->NewGlobalRef( callback ); - if( !callback ) { - if( !jenv->ExceptionOccurred() ) + callback = jenv->NewGlobalRef(callback); + if (!callback) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); return; } // Here we cache a thread-local reference to jenv g_thread_jenv = jenv; - fdb_error_t err = fdb_future_set_callback( f, &callCallback, callback ); - if( err ) { - jenv->DeleteGlobalRef( callback ); - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_future_set_callback(f, &callCallback, callback); + if (err) { + jenv->DeleteGlobalRef(callback); + safeThrow(jenv, getThrowable(jenv, err)); } } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1blockUntilReady(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1blockUntilReady(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return; } - FDBFuture *sav = (FDBFuture *)future; + FDBFuture* sav = (FDBFuture*)future; - fdb_error_t err = fdb_future_block_until_ready( sav ); - if( err ) - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_future_block_until_ready(sav); + if (err) + safeThrow(jenv, getThrowable(jenv, err)); } -JNIEXPORT jthrowable JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1getError(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT jthrowable JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1getError(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return JNI_NULL; } - FDBFuture *sav = (FDBFuture *)future; - fdb_error_t err = fdb_future_get_error( sav ); - if( err ) - return getThrowable( jenv, err ); + FDBFuture* sav = (FDBFuture*)future; + fdb_error_t err = fdb_future_get_error(sav); + if (err) + return getThrowable(jenv, err); else return JNI_NULL; } -JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1isReady(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1isReady(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return JNI_FALSE; } - FDBFuture *var = (FDBFuture *)future; + FDBFuture* var = (FDBFuture*)future; return (jboolean)fdb_future_is_ready(var); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1dispose(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1dispose(JNIEnv* jenv, jobject, jlong future) { + if (!future) { throwParamNotNull(jenv); return; } - FDBFuture *var = (FDBFuture *)future; + FDBFuture* var = (FDBFuture*)future; fdb_future_destroy(var); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1cancel(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1cancel(JNIEnv* jenv, jobject, jlong future) { + if (!future) { throwParamNotNull(jenv); return; } - FDBFuture *var = (FDBFuture *)future; + FDBFuture* var = (FDBFuture*)future; fdb_future_cancel(var); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1releaseMemory(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_NativeFuture_Future_1releaseMemory(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return; } - FDBFuture *var = (FDBFuture *)future; + FDBFuture* var = (FDBFuture*)future; fdb_future_release_memory(var); } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FutureInt64_FutureInt64_1get(JNIEnv *jenv, jobject, jlong future) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FutureInt64_FutureInt64_1get(JNIEnv* jenv, jobject, jlong future) { if (!future) { throwParamNotNull(jenv); return 0; } - FDBFuture *f = (FDBFuture *)future; + FDBFuture* f = (FDBFuture*)future; int64_t value = 0; fdb_error_t err = fdb_future_get_int64(f, &value); @@ -265,40 +277,42 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FutureInt64_FutureInt64_1get return (jlong)value; } -JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureStrings_1get(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureStrings_1get(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return JNI_NULL; } - FDBFuture *f = (FDBFuture *)future; + FDBFuture* f = (FDBFuture*)future; - const char **strings; + const char** strings; int count; - fdb_error_t err = fdb_future_get_string_array( f, &strings, &count ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_future_get_string_array(f, &strings, &count); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return JNI_NULL; } - if( jenv->ExceptionOccurred() ) + if (jenv->ExceptionOccurred()) return JNI_NULL; jobjectArray arr = jenv->NewObjectArray(count, string_class, JNI_NULL); - if( !arr ) { - if( !jenv->ExceptionOccurred() ) + if (!arr) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); return JNI_NULL; } - for(int i = 0; i < count; i++) { - jstring str = jenv->NewStringUTF( strings[i] ); - if( !str ) { - if( !jenv->ExceptionOccurred() ) + for (int i = 0; i < count; i++) { + jstring str = jenv->NewStringUTF(strings[i]); + if (!str) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); return JNI_NULL; } - jenv->SetObjectArrayElement( arr, i, str ); - if( jenv->ExceptionOccurred() ) + jenv->SetObjectArrayElement(arr, i, str); + if (jenv->ExceptionOccurred()) return JNI_NULL; } @@ -306,205 +320,220 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureString } // SOMEDAY: explore doing this more efficiently with Direct ByteBuffers -JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1get(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1get(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return JNI_NULL; } - - FDBFuture *f = (FDBFuture *)future; - const FDBKeyValue *kvs; + FDBFuture* f = (FDBFuture*)future; + + const FDBKeyValue* kvs; int count; fdb_bool_t more; - fdb_error_t err = fdb_future_get_keyvalue_array( f, &kvs, &count, &more ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_future_get_keyvalue_array(f, &kvs, &count, &more); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return JNI_NULL; } int totalKeyValueSize = 0; - for(int i = 0; i < count; i++) { + for (int i = 0; i < count; i++) { totalKeyValueSize += kvs[i].key_length + kvs[i].value_length; } jbyteArray keyValueArray = jenv->NewByteArray(totalKeyValueSize); - if( !keyValueArray ) { - if( !jenv->ExceptionOccurred() ) + if (!keyValueArray) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); return JNI_NULL; } - uint8_t *keyvalues_barr = (uint8_t *)jenv->GetByteArrayElements(keyValueArray, JNI_NULL); + uint8_t* keyvalues_barr = (uint8_t*)jenv->GetByteArrayElements(keyValueArray, JNI_NULL); if (!keyvalues_barr) { - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + throwRuntimeEx(jenv, "Error getting handle to native resources"); return JNI_NULL; } jintArray lengthArray = jenv->NewIntArray(count * 2); - if( !lengthArray ) { - if( !jenv->ExceptionOccurred() ) + if (!lengthArray) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); - jenv->ReleaseByteArrayElements(keyValueArray, (jbyte *)keyvalues_barr, 0); + jenv->ReleaseByteArrayElements(keyValueArray, (jbyte*)keyvalues_barr, 0); return JNI_NULL; } - jint *length_barr = jenv->GetIntArrayElements(lengthArray, JNI_NULL); - if( !length_barr ) { - if( !jenv->ExceptionOccurred() ) + jint* length_barr = jenv->GetIntArrayElements(lengthArray, JNI_NULL); + if (!length_barr) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); - jenv->ReleaseByteArrayElements(keyValueArray, (jbyte *)keyvalues_barr, 0); + jenv->ReleaseByteArrayElements(keyValueArray, (jbyte*)keyvalues_barr, 0); return JNI_NULL; } int offset = 0; - for(int i = 0; i < count; i++) { + for (int i = 0; i < count; i++) { memcpy(keyvalues_barr + offset, kvs[i].key, kvs[i].key_length); - length_barr[ i * 2 ] = kvs[i].key_length; + length_barr[i * 2] = kvs[i].key_length; offset += kvs[i].key_length; memcpy(keyvalues_barr + offset, kvs[i].value, kvs[i].value_length); - length_barr[ (i * 2) + 1 ] = kvs[i].value_length; + length_barr[(i * 2) + 1] = kvs[i].value_length; offset += kvs[i].value_length; } - jenv->ReleaseByteArrayElements(keyValueArray, (jbyte *)keyvalues_barr, 0); + jenv->ReleaseByteArrayElements(keyValueArray, (jbyte*)keyvalues_barr, 0); jenv->ReleaseIntArrayElements(lengthArray, length_barr, 0); jobject result = jenv->NewObject(range_result_class, range_result_init, keyValueArray, lengthArray, (jboolean)more); - if( jenv->ExceptionOccurred() ) + if (jenv->ExceptionOccurred()) return JNI_NULL; return result; } // SOMEDAY: explore doing this more efficiently with Direct ByteBuffers -JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureResult_FutureResult_1get(JNIEnv *jenv, jobject, jlong future) { - if( !future ) { +JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureResult_FutureResult_1get(JNIEnv* jenv, + jobject, + jlong future) { + if (!future) { throwParamNotNull(jenv); return JNI_NULL; } - FDBFuture *f = (FDBFuture *)future; + FDBFuture* f = (FDBFuture*)future; fdb_bool_t present; - const uint8_t *value; + const uint8_t* value; int length; fdb_error_t err = fdb_future_get_value(f, &present, &value, &length); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return JNI_NULL; } - if( !present ) + if (!present) return JNI_NULL; jbyteArray result = jenv->NewByteArray(length); - if( !result ) { - if( !jenv->ExceptionOccurred() ) + if (!result) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); return JNI_NULL; } - jenv->SetByteArrayRegion(result, 0, length, (const jbyte *)value); + jenv->SetByteArrayRegion(result, 0, length, (const jbyte*)value); return result; } -JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureKey_FutureKey_1get(JNIEnv * jenv, jclass, jlong future) { - if( !future ) { +JNIEXPORT jbyteArray JNICALL Java_com_apple_foundationdb_FutureKey_FutureKey_1get(JNIEnv* jenv, jclass, jlong future) { + if (!future) { throwParamNotNull(jenv); return JNI_NULL; } - FDBFuture *f = (FDBFuture *)future; + FDBFuture* f = (FDBFuture*)future; - const uint8_t *value; + const uint8_t* value; int length; fdb_error_t err = fdb_future_get_key(f, &value, &length); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return JNI_NULL; } jbyteArray result = jenv->NewByteArray(length); - if( !result ) { - if( !jenv->ExceptionOccurred() ) + if (!result) { + if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); return JNI_NULL; } - jenv->SetByteArrayRegion(result, 0, length, (const jbyte *)value); + jenv->SetByteArrayRegion(result, 0, length, (const jbyte*)value); return result; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1createTransaction(JNIEnv *jenv, jobject, jlong dbPtr) { - if( !dbPtr ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1createTransaction(JNIEnv* jenv, + jobject, + jlong dbPtr) { + if (!dbPtr) { throwParamNotNull(jenv); return 0; } - FDBDatabase *database = (FDBDatabase *)dbPtr; - FDBTransaction *tr; + FDBDatabase* database = (FDBDatabase*)dbPtr; + FDBTransaction* tr; fdb_error_t err = fdb_database_create_transaction(database, &tr); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return 0; } return (jlong)tr; } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1dispose(JNIEnv *jenv, jobject, jlong dPtr) { - if( !dPtr ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1dispose(JNIEnv* jenv, jobject, jlong dPtr) { + if (!dPtr) { throwParamNotNull(jenv); return; } - fdb_database_destroy( (FDBDatabase *)dPtr ); + fdb_database_destroy((FDBDatabase*)dPtr); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1setOption(JNIEnv *jenv, jobject, jlong dPtr, jint code, jbyteArray value) { - if( !dPtr ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1setOption(JNIEnv* jenv, + jobject, + jlong dPtr, + jint code, + jbyteArray value) { + if (!dPtr) { throwParamNotNull(jenv); return; } - FDBDatabase *c = (FDBDatabase *)dPtr; - uint8_t *barr = nullptr; + FDBDatabase* c = (FDBDatabase*)dPtr; + uint8_t* barr = nullptr; int size = 0; - if(value != JNI_NULL) { - barr = (uint8_t *)jenv->GetByteArrayElements( value, JNI_NULL ); + if (value != JNI_NULL) { + barr = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL); if (!barr) { - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - size = jenv->GetArrayLength( value ); + size = jenv->GetArrayLength(value); } - fdb_error_t err = fdb_database_set_option( c, (FDBDatabaseOption)code, barr, size ); - if(value != JNI_NULL) - jenv->ReleaseByteArrayElements( value, (jbyte *)barr, JNI_ABORT ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_database_set_option(c, (FDBDatabaseOption)code, barr, size); + if (value != JNI_NULL) + jenv->ReleaseByteArrayElements(value, (jbyte*)barr, JNI_ABORT); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); } } -JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIEnv *jenv, jobject, jint predicate, jint code) { +JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIEnv* jenv, + jobject, + jint predicate, + jint code) { return (jboolean)fdb_error_predicate(predicate, code); } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Database_1create(JNIEnv *jenv, jobject, jstring clusterFileName) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Database_1create(JNIEnv* jenv, + jobject, + jstring clusterFileName) { const char* fileName = nullptr; - if(clusterFileName != JNI_NULL) { + if (clusterFileName != JNI_NULL) { fileName = jenv->GetStringUTFChars(clusterFileName, JNI_NULL); - if(jenv->ExceptionOccurred()) { + if (jenv->ExceptionOccurred()) { return 0; } } - FDBDatabase *db; + FDBDatabase* db; fdb_error_t err = fdb_create_database(fileName, &db); - if(clusterFileName != JNI_NULL) { + if (clusterFileName != JNI_NULL) { jenv->ReleaseStringUTFChars(clusterFileName, fileName); } - if(err) { + if (err) { safeThrow(jenv, getThrowable(jenv, err)); return 0; } @@ -512,102 +541,142 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDB_Database_1create(JNIEnv return (jlong)db; } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setVersion(JNIEnv *jenv, jobject, jlong tPtr, jlong version) { - if( !tPtr ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setVersion(JNIEnv* jenv, + jobject, + jlong tPtr, + jlong version) { + if (!tPtr) { throwParamNotNull(jenv); return; } - FDBTransaction *tr = (FDBTransaction *)tPtr; - fdb_transaction_set_read_version( tr, version ); + FDBTransaction* tr = (FDBTransaction*)tPtr; + fdb_transaction_set_read_version(tr, version); } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getReadVersion(JNIEnv *jenv, jobject, jlong tPtr) { - if( !tPtr ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getReadVersion(JNIEnv* jenv, + jobject, + jlong tPtr) { + if (!tPtr) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; - FDBFuture *f = fdb_transaction_get_read_version( tr ); + FDBTransaction* tr = (FDBTransaction*)tPtr; + FDBFuture* f = fdb_transaction_get_read_version(tr); return (jlong)f; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1get(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes, jboolean snapshot) { - if( !tPtr || !keyBytes ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1get(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray keyBytes, + jboolean snapshot) { + if (!tPtr || !keyBytes) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( keyBytes, JNI_NULL ); - if(!barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); + if (!barr) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - FDBFuture *f = fdb_transaction_get( tr, barr, jenv->GetArrayLength( keyBytes ), (fdb_bool_t)snapshot ); - jenv->ReleaseByteArrayElements( keyBytes, (jbyte *)barr, JNI_ABORT ); + FDBFuture* f = fdb_transaction_get(tr, barr, jenv->GetArrayLength(keyBytes), (fdb_bool_t)snapshot); + jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barr, JNI_ABORT); return (jlong)f; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKey(JNIEnv *jenv, jobject, jlong tPtr, - jbyteArray keyBytes, jboolean orEqual, jint offset, jboolean snapshot) { - if( !tPtr || !keyBytes ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKey(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray keyBytes, + jboolean orEqual, + jint offset, + jboolean snapshot) { + if (!tPtr || !keyBytes) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( keyBytes, JNI_NULL ); - if(!barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); + if (!barr) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - FDBFuture *f = fdb_transaction_get_key( tr, barr, jenv->GetArrayLength( keyBytes ), orEqual, offset, (fdb_bool_t)snapshot ); - jenv->ReleaseByteArrayElements( keyBytes, (jbyte *)barr, JNI_ABORT ); + FDBFuture* f = + fdb_transaction_get_key(tr, barr, jenv->GetArrayLength(keyBytes), orEqual, offset, (fdb_bool_t)snapshot); + jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barr, JNI_ABORT); return (jlong)f; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRange - (JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBeginBytes, jboolean orEqualBegin, jint offsetBegin, - jbyteArray keyEndBytes, jboolean orEqualEnd, jint offsetEnd, jint rowLimit, jint targetBytes, - jint streamingMode, jint iteration, jboolean snapshot, jboolean reverse) { - if( !tPtr || !keyBeginBytes || !keyEndBytes ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRange(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray keyBeginBytes, + jboolean orEqualBegin, + jint offsetBegin, + jbyteArray keyEndBytes, + jboolean orEqualEnd, + jint offsetEnd, + jint rowLimit, + jint targetBytes, + jint streamingMode, + jint iteration, + jboolean snapshot, + jboolean reverse) { + if (!tPtr || !keyBeginBytes || !keyEndBytes) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *barrBegin = (uint8_t *)jenv->GetByteArrayElements( keyBeginBytes, JNI_NULL ); + uint8_t* barrBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL); if (!barrBegin) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - uint8_t *barrEnd = (uint8_t *)jenv->GetByteArrayElements( keyEndBytes, JNI_NULL ); + uint8_t* barrEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL); if (!barrEnd) { - jenv->ReleaseByteArrayElements( keyBeginBytes, (jbyte *)barrBegin, JNI_ABORT ); - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - FDBFuture *f = fdb_transaction_get_range( tr, - barrBegin, jenv->GetArrayLength( keyBeginBytes ), orEqualBegin, offsetBegin, - barrEnd, jenv->GetArrayLength( keyEndBytes ), orEqualEnd, offsetEnd, rowLimit, - targetBytes, (FDBStreamingMode)streamingMode, iteration, snapshot, reverse); - jenv->ReleaseByteArrayElements( keyBeginBytes, (jbyte *)barrBegin, JNI_ABORT ); - jenv->ReleaseByteArrayElements( keyEndBytes, (jbyte *)barrEnd, JNI_ABORT ); + FDBFuture* f = fdb_transaction_get_range(tr, + barrBegin, + jenv->GetArrayLength(keyBeginBytes), + orEqualBegin, + offsetBegin, + barrEnd, + jenv->GetArrayLength(keyEndBytes), + orEqualEnd, + offsetEnd, + rowLimit, + targetBytes, + (FDBStreamingMode)streamingMode, + iteration, + snapshot, + reverse); + jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT); + jenv->ReleaseByteArrayElements(keyEndBytes, (jbyte*)barrEnd, JNI_ABORT); return (jlong)f; } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1getDirect( - JNIEnv* jenv, jobject, jlong future, jobject jbuffer, jint bufferCapacity) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1getDirect(JNIEnv* jenv, + jobject, + jlong future, + jobject jbuffer, + jint bufferCapacity) { - if( !future ) { + if (!future) { throwParamNotNull(jenv); return; } @@ -620,12 +689,12 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1 } FDBFuture* f = (FDBFuture*)future; - const FDBKeyValue *kvs; + const FDBKeyValue* kvs; int count; fdb_bool_t more; - fdb_error_t err = fdb_future_get_keyvalue_array( f, &kvs, &count, &more ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_future_get_keyvalue_array(f, &kvs, &count, &more); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return; } @@ -635,8 +704,8 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1 // => sizeof(jint) to store key length per KV pair // => sizeof(jint) to store value length per KV pair int totalCapacityNeeded = 2 * sizeof(jint); - for(int i = 0; i < count; i++) { - totalCapacityNeeded += kvs[i].key_length + kvs[i].value_length + 2*sizeof(jint); + for (int i = 0; i < count; i++) { + totalCapacityNeeded += kvs[i].key_length + kvs[i].value_length + 2 * sizeof(jint); if (bufferCapacity < totalCapacityNeeded) { count = i; /* Only fit first `i` K/V pairs */ more = true; @@ -666,195 +735,220 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1 } } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getEstimatedRangeSizeBytes(JNIEnv *jenv, jobject, jlong tPtr, - jbyteArray beginKeyBytes, jbyteArray endKeyBytes) { - if( !tPtr || !beginKeyBytes || !endKeyBytes) { +JNIEXPORT jlong JNICALL +Java_com_apple_foundationdb_FDBTransaction_Transaction_1getEstimatedRangeSizeBytes(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray beginKeyBytes, + jbyteArray endKeyBytes) { + if (!tPtr || !beginKeyBytes || !endKeyBytes) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *startKey = (uint8_t *)jenv->GetByteArrayElements( beginKeyBytes, JNI_NULL ); - if(!startKey) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + uint8_t* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL); + if (!startKey) { + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - uint8_t *endKey = (uint8_t *)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL); + uint8_t* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL); if (!endKey) { - jenv->ReleaseByteArrayElements( beginKeyBytes, (jbyte *)startKey, JNI_ABORT ); - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - FDBFuture *f = fdb_transaction_get_estimated_range_size_bytes( tr, startKey, jenv->GetArrayLength( beginKeyBytes ), endKey, jenv->GetArrayLength( endKeyBytes ) ); - jenv->ReleaseByteArrayElements( beginKeyBytes, (jbyte *)startKey, JNI_ABORT ); - jenv->ReleaseByteArrayElements( endKeyBytes, (jbyte *)endKey, JNI_ABORT ); + FDBFuture* f = fdb_transaction_get_estimated_range_size_bytes( + tr, startKey, jenv->GetArrayLength(beginKeyBytes), endKey, jenv->GetArrayLength(endKeyBytes)); + jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT); + jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKey, JNI_ABORT); return (jlong)f; } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1set(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes, jbyteArray valueBytes) { - if( !tPtr || !keyBytes || !valueBytes ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1set(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray keyBytes, + jbyteArray valueBytes) { + if (!tPtr || !keyBytes || !valueBytes) { throwParamNotNull(jenv); return; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *barrKey = (uint8_t *)jenv->GetByteArrayElements( keyBytes, JNI_NULL ); + uint8_t* barrKey = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); if (!barrKey) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - uint8_t *barrValue = (uint8_t *)jenv->GetByteArrayElements( valueBytes, JNI_NULL ); + uint8_t* barrValue = (uint8_t*)jenv->GetByteArrayElements(valueBytes, JNI_NULL); if (!barrValue) { - jenv->ReleaseByteArrayElements( keyBytes, (jbyte *)barrKey, JNI_ABORT ); - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barrKey, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - fdb_transaction_set( tr, - barrKey, jenv->GetArrayLength( keyBytes ), - barrValue, jenv->GetArrayLength( valueBytes ) ); - jenv->ReleaseByteArrayElements( keyBytes, (jbyte *)barrKey, JNI_ABORT ); - jenv->ReleaseByteArrayElements( valueBytes, (jbyte *)barrValue, JNI_ABORT ); + fdb_transaction_set(tr, barrKey, jenv->GetArrayLength(keyBytes), barrValue, jenv->GetArrayLength(valueBytes)); + jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barrKey, JNI_ABORT); + jenv->ReleaseByteArrayElements(valueBytes, (jbyte*)barrValue, JNI_ABORT); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes) { - if( !tPtr || !keyBytes ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray keyBytes) { + if (!tPtr || !keyBytes) { throwParamNotNull(jenv); return; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( keyBytes, JNI_NULL ); + uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); if (!barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - fdb_transaction_clear( tr, barr, jenv->GetArrayLength( keyBytes ) ); - jenv->ReleaseByteArrayElements( keyBytes, (jbyte *)barr, JNI_ABORT ); + fdb_transaction_clear(tr, barr, jenv->GetArrayLength(keyBytes)); + jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barr, JNI_ABORT); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B_3B(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBeginBytes, jbyteArray keyEndBytes) { - if( !tPtr || !keyBeginBytes || !keyEndBytes ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1clear__J_3B_3B(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray keyBeginBytes, + jbyteArray keyEndBytes) { + if (!tPtr || !keyBeginBytes || !keyEndBytes) { throwParamNotNull(jenv); return; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *barrKeyBegin = (uint8_t *)jenv->GetByteArrayElements( keyBeginBytes, JNI_NULL ); + uint8_t* barrKeyBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL); if (!barrKeyBegin) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - uint8_t *barrKeyEnd = (uint8_t *)jenv->GetByteArrayElements( keyEndBytes, JNI_NULL ); + uint8_t* barrKeyEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL); if (!barrKeyEnd) { - jenv->ReleaseByteArrayElements( keyBeginBytes, (jbyte *)barrKeyBegin, JNI_ABORT ); - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrKeyBegin, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - fdb_transaction_clear_range( tr, - barrKeyBegin, jenv->GetArrayLength( keyBeginBytes ), - barrKeyEnd, jenv->GetArrayLength( keyEndBytes ) ); - jenv->ReleaseByteArrayElements( keyBeginBytes, (jbyte *)barrKeyBegin, JNI_ABORT ); - jenv->ReleaseByteArrayElements( keyEndBytes, (jbyte *)barrKeyEnd, JNI_ABORT ); + fdb_transaction_clear_range( + tr, barrKeyBegin, jenv->GetArrayLength(keyBeginBytes), barrKeyEnd, jenv->GetArrayLength(keyEndBytes)); + jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrKeyBegin, JNI_ABORT); + jenv->ReleaseByteArrayElements(keyEndBytes, (jbyte*)barrKeyEnd, JNI_ABORT); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1mutate(JNIEnv *jenv, jobject, jlong tPtr, jint code, - jbyteArray key, jbyteArray value ) { - if( !tPtr || !key || !value ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1mutate(JNIEnv* jenv, + jobject, + jlong tPtr, + jint code, + jbyteArray key, + jbyteArray value) { + if (!tPtr || !key || !value) { throwParamNotNull(jenv); return; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *barrKey = (uint8_t *)jenv->GetByteArrayElements( key, JNI_NULL ); + uint8_t* barrKey = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL); if (!barrKey) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - uint8_t *barrValue = (uint8_t *)jenv->GetByteArrayElements( value, JNI_NULL ); + uint8_t* barrValue = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL); if (!barrValue) { - jenv->ReleaseByteArrayElements( key, (jbyte *)barrKey, JNI_ABORT ); - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + jenv->ReleaseByteArrayElements(key, (jbyte*)barrKey, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - fdb_transaction_atomic_op( tr, - barrKey, jenv->GetArrayLength( key ), - barrValue, jenv->GetArrayLength( value ), - (FDBMutationType)code); + fdb_transaction_atomic_op( + tr, barrKey, jenv->GetArrayLength(key), barrValue, jenv->GetArrayLength(value), (FDBMutationType)code); - jenv->ReleaseByteArrayElements( key, (jbyte *)barrKey, JNI_ABORT ); - jenv->ReleaseByteArrayElements( value, (jbyte *)barrValue, JNI_ABORT ); + jenv->ReleaseByteArrayElements(key, (jbyte*)barrKey, JNI_ABORT); + jenv->ReleaseByteArrayElements(value, (jbyte*)barrValue, JNI_ABORT); } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1commit(JNIEnv *jenv, jobject, jlong tPtr) { - if( !tPtr ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1commit(JNIEnv* jenv, + jobject, + jlong tPtr) { + if (!tPtr) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; - FDBFuture *f = fdb_transaction_commit( tr ); + FDBTransaction* tr = (FDBTransaction*)tPtr; + FDBFuture* f = fdb_transaction_commit(tr); return (jlong)f; } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setOption(JNIEnv *jenv, jobject, jlong tPtr, jint code, jbyteArray value) { - if( !tPtr ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1setOption(JNIEnv* jenv, + jobject, + jlong tPtr, + jint code, + jbyteArray value) { + if (!tPtr) { throwParamNotNull(jenv); return; } - FDBTransaction *tr = (FDBTransaction *)tPtr; - uint8_t *barr = nullptr; + FDBTransaction* tr = (FDBTransaction*)tPtr; + uint8_t* barr = nullptr; int size = 0; - if(value != JNI_NULL) { - barr = (uint8_t *)jenv->GetByteArrayElements( value, JNI_NULL ); + if (value != JNI_NULL) { + barr = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL); if (!barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - size = jenv->GetArrayLength( value ); + size = jenv->GetArrayLength(value); } - fdb_error_t err = fdb_transaction_set_option( tr, (FDBTransactionOption)code, barr, size ); - if(value != JNI_NULL) - jenv->ReleaseByteArrayElements( value, (jbyte *)barr, JNI_ABORT ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_transaction_set_option(tr, (FDBTransactionOption)code, barr, size); + if (value != JNI_NULL) + jenv->ReleaseByteArrayElements(value, (jbyte*)barr, JNI_ABORT); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); } } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getCommittedVersion(JNIEnv *jenv, jobject, jlong tPtr) { - if( !tPtr ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getCommittedVersion(JNIEnv* jenv, + jobject, + jlong tPtr) { + if (!tPtr) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; int64_t version; - fdb_error_t err = fdb_transaction_get_committed_version( tr, &version ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + fdb_error_t err = fdb_transaction_get_committed_version(tr, &version); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); return 0; } return (jlong)version; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getApproximateSize(JNIEnv *jenv, jobject, jlong tPtr) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getApproximateSize(JNIEnv* jenv, + jobject, + jlong tPtr) { if (!tPtr) { throwParamNotNull(jenv); return 0; @@ -863,227 +957,256 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 return (jlong)f; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getVersionstamp(JNIEnv *jenv, jobject, jlong tPtr) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getVersionstamp(JNIEnv* jenv, + jobject, + jlong tPtr) { if (!tPtr) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; - FDBFuture *f = fdb_transaction_get_versionstamp(tr); + FDBTransaction* tr = (FDBTransaction*)tPtr; + FDBFuture* f = fdb_transaction_get_versionstamp(tr); return (jlong)f; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKeyLocations(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray key) { - if( !tPtr || !key ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getKeyLocations(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray key) { + if (!tPtr || !key) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( key, JNI_NULL ); + uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL); if (!barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - int size = jenv->GetArrayLength( key ); + int size = jenv->GetArrayLength(key); - FDBFuture *f = fdb_transaction_get_addresses_for_key( tr, barr, size ); + FDBFuture* f = fdb_transaction_get_addresses_for_key(tr, barr, size); - jenv->ReleaseByteArrayElements( key, (jbyte *)barr, JNI_ABORT ); + jenv->ReleaseByteArrayElements(key, (jbyte*)barr, JNI_ABORT); return (jlong)f; } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1onError(JNIEnv *jenv, jobject, jlong tPtr, jint errorCode) { - if( !tPtr ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1onError(JNIEnv* jenv, + jobject, + jlong tPtr, + jint errorCode) { + if (!tPtr) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; - FDBFuture *f = fdb_transaction_on_error( tr, (fdb_error_t)errorCode ); + FDBTransaction* tr = (FDBTransaction*)tPtr; + FDBFuture* f = fdb_transaction_on_error(tr, (fdb_error_t)errorCode); return (jlong)f; } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1dispose(JNIEnv *jenv, jobject, jlong tPtr) { - if( !tPtr ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1dispose(JNIEnv* jenv, + jobject, + jlong tPtr) { + if (!tPtr) { throwParamNotNull(jenv); return; } - fdb_transaction_destroy( (FDBTransaction *)tPtr ); + fdb_transaction_destroy((FDBTransaction*)tPtr); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1reset(JNIEnv *jenv, jobject, jlong tPtr) { - if( !tPtr ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1reset(JNIEnv* jenv, + jobject, + jlong tPtr) { + if (!tPtr) { throwParamNotNull(jenv); return; } - fdb_transaction_reset( (FDBTransaction *)tPtr ); + fdb_transaction_reset((FDBTransaction*)tPtr); } -JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1watch(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray key) { - if( !tPtr || !key ) { +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1watch(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray key) { + if (!tPtr || !key) { throwParamNotNull(jenv); return 0; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *barr = (uint8_t *)jenv->GetByteArrayElements( key, JNI_NULL ); + uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL); if (!barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - int size = jenv->GetArrayLength( key ); - FDBFuture *f = fdb_transaction_watch( tr, barr, size ); + int size = jenv->GetArrayLength(key); + FDBFuture* f = fdb_transaction_watch(tr, barr, size); - jenv->ReleaseByteArrayElements( key, (jbyte *)barr, JNI_ABORT ); + jenv->ReleaseByteArrayElements(key, (jbyte*)barr, JNI_ABORT); return (jlong)f; } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1cancel(JNIEnv *jenv, jobject, jlong tPtr) { - if( !tPtr ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1cancel(JNIEnv* jenv, + jobject, + jlong tPtr) { + if (!tPtr) { throwParamNotNull(jenv); return; } - fdb_transaction_cancel( (FDBTransaction *)tPtr ); + fdb_transaction_cancel((FDBTransaction*)tPtr); } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1addConflictRange( - JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBegin, jbyteArray keyEnd, jint conflictType) { - if( !tPtr || !keyBegin || !keyEnd ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1addConflictRange(JNIEnv* jenv, + jobject, + jlong tPtr, + jbyteArray keyBegin, + jbyteArray keyEnd, + jint conflictType) { + if (!tPtr || !keyBegin || !keyEnd) { throwParamNotNull(jenv); return; } - FDBTransaction *tr = (FDBTransaction *)tPtr; + FDBTransaction* tr = (FDBTransaction*)tPtr; - uint8_t *begin_barr = (uint8_t *)jenv->GetByteArrayElements( keyBegin, JNI_NULL ); + uint8_t* begin_barr = (uint8_t*)jenv->GetByteArrayElements(keyBegin, JNI_NULL); if (!begin_barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - int begin_size = jenv->GetArrayLength( keyBegin ); + int begin_size = jenv->GetArrayLength(keyBegin); - uint8_t *end_barr = (uint8_t *)jenv->GetByteArrayElements( keyEnd, JNI_NULL ); + uint8_t* end_barr = (uint8_t*)jenv->GetByteArrayElements(keyEnd, JNI_NULL); if (!end_barr) { - jenv->ReleaseByteArrayElements( keyBegin, (jbyte *)begin_barr, JNI_ABORT ); - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + jenv->ReleaseByteArrayElements(keyBegin, (jbyte*)begin_barr, JNI_ABORT); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - int end_size = jenv->GetArrayLength( keyEnd ); + int end_size = jenv->GetArrayLength(keyEnd); - fdb_error_t err = fdb_transaction_add_conflict_range( tr, begin_barr, begin_size, end_barr, end_size, (FDBConflictRangeType)conflictType ); + fdb_error_t err = fdb_transaction_add_conflict_range( + tr, begin_barr, begin_size, end_barr, end_size, (FDBConflictRangeType)conflictType); - jenv->ReleaseByteArrayElements( keyBegin, (jbyte *)begin_barr, JNI_ABORT ); - jenv->ReleaseByteArrayElements( keyEnd, (jbyte *)end_barr, JNI_ABORT ); + jenv->ReleaseByteArrayElements(keyBegin, (jbyte*)begin_barr, JNI_ABORT); + jenv->ReleaseByteArrayElements(keyEnd, (jbyte*)end_barr, JNI_ABORT); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); } } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Select_1API_1version(JNIEnv *jenv, jclass, jint version) { - fdb_error_t err = fdb_select_api_version( (int)version ); - if( err ) { - if( err == 2203 ) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Select_1API_1version(JNIEnv* jenv, jclass, jint version) { + fdb_error_t err = fdb_select_api_version((int)version); + if (err) { + if (err == 2203) { int maxSupportedVersion = fdb_get_max_api_version(); char errorStr[1024]; - if(FDB_API_VERSION > maxSupportedVersion) { - snprintf(errorStr, sizeof(errorStr), "This version of the FoundationDB Java binding is not supported by the installed " - "FoundationDB C library. The binding requires a library that supports API version " - "%d, but the installed library supports a maximum version of %d.", - FDB_API_VERSION, maxSupportedVersion); - } - else { - snprintf(errorStr, sizeof(errorStr), "API version %d is not supported by the installed FoundationDB C library.", version); + if (FDB_API_VERSION > maxSupportedVersion) { + snprintf(errorStr, + sizeof(errorStr), + "This version of the FoundationDB Java binding is not supported by the installed " + "FoundationDB C library. The binding requires a library that supports API version " + "%d, but the installed library supports a maximum version of %d.", + FDB_API_VERSION, + maxSupportedVersion); + } else { + snprintf(errorStr, + sizeof(errorStr), + "API version %d is not supported by the installed FoundationDB C library.", + version); } - safeThrow( jenv, getThrowable( jenv, err, errorStr ) ); - } - else { - safeThrow( jenv, getThrowable( jenv, err ) ); + safeThrow(jenv, getThrowable(jenv, err, errorStr)); + } else { + safeThrow(jenv, getThrowable(jenv, err)); } } } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1setOption(JNIEnv *jenv, jobject, jint code, jbyteArray value) { - uint8_t *barr = nullptr; +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1setOption(JNIEnv* jenv, + jobject, + jint code, + jbyteArray value) { + uint8_t* barr = nullptr; int size = 0; - if(value != JNI_NULL) { - barr = (uint8_t *)jenv->GetByteArrayElements( value, JNI_NULL ); + if (value != JNI_NULL) { + barr = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL); if (!barr) { - if( !jenv->ExceptionOccurred() ) - throwRuntimeEx( jenv, "Error getting handle to native resources" ); + if (!jenv->ExceptionOccurred()) + throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - size = jenv->GetArrayLength( value ); + size = jenv->GetArrayLength(value); } fdb_error_t err = fdb_network_set_option((FDBNetworkOption)code, barr, size); - if(value != JNI_NULL) - jenv->ReleaseByteArrayElements( value, (jbyte *)barr, JNI_ABORT ); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (value != JNI_NULL) + jenv->ReleaseByteArrayElements(value, (jbyte*)barr, JNI_ABORT); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); } } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1setup(JNIEnv *jenv, jobject) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1setup(JNIEnv* jenv, jobject) { fdb_error_t err = fdb_setup_network(); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); } } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1run(JNIEnv *jenv, jobject) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1run(JNIEnv* jenv, jobject) { // initialize things for the callbacks on the network thread g_thread_jenv = jenv; - if( !g_IFutureCallback_call_methodID ) { - if( !findCallbackMethods( jenv ) ) + if (!g_IFutureCallback_call_methodID) { + if (!findCallbackMethods(jenv)) return; } - fdb_error_t hookErr = fdb_add_network_thread_completion_hook( &detachIfExternalThread, nullptr ); - if( hookErr ) { - safeThrow( jenv, getThrowable( jenv, hookErr ) ); + fdb_error_t hookErr = fdb_add_network_thread_completion_hook(&detachIfExternalThread, nullptr); + if (hookErr) { + safeThrow(jenv, getThrowable(jenv, hookErr)); } fdb_error_t err = fdb_run_network(); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); } } -JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1stop(JNIEnv *jenv, jobject) { +JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1stop(JNIEnv* jenv, jobject) { fdb_error_t err = fdb_stop_network(); - if( err ) { - safeThrow( jenv, getThrowable( jenv, err ) ); + if (err) { + safeThrow(jenv, getThrowable(jenv, err)); } } -jint JNI_OnLoad(JavaVM *vm, void *reserved) { - JNIEnv *env; +jint JNI_OnLoad(JavaVM* vm, void* reserved) { + JNIEnv* env; g_jvm = vm; if (vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK) { return JNI_ERR; } else { jclass local_range_result_class = env->FindClass("com/apple/foundationdb/RangeResult"); range_result_init = env->GetMethodID(local_range_result_class, "", "([B[IZ)V"); - range_result_class = (jclass) (env)->NewGlobalRef(local_range_result_class); + range_result_class = (jclass)(env)->NewGlobalRef(local_range_result_class); jclass local_range_result_summary_class = env->FindClass("com/apple/foundationdb/RangeResultSummary"); range_result_summary_init = env->GetMethodID(local_range_result_summary_class, "", "([BIZ)V"); - range_result_summary_class = (jclass) (env)->NewGlobalRef(local_range_result_summary_class); + range_result_summary_class = (jclass)(env)->NewGlobalRef(local_range_result_summary_class); jclass local_string_class = env->FindClass("java/lang/String"); - string_class = (jclass) (env)->NewGlobalRef(local_string_class); + string_class = (jclass)(env)->NewGlobalRef(local_string_class); return JNI_VERSION_1_6; } } // Is automatically called once the Classloader is destroyed -void JNI_OnUnload(JavaVM *vm, void *reserved) { +void JNI_OnUnload(JavaVM* vm, void* reserved) { JNIEnv* env; if (vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK) { return; @@ -1104,4 +1227,3 @@ void JNI_OnUnload(JavaVM *vm, void *reserved) { #ifdef __cplusplus } #endif - diff --git a/contrib/monitoring/actor_flamegraph.cpp b/contrib/monitoring/actor_flamegraph.cpp index 4e1c168338..58bd0d2c86 100644 --- a/contrib/monitoring/actor_flamegraph.cpp +++ b/contrib/monitoring/actor_flamegraph.cpp @@ -24,7 +24,8 @@ struct Error { struct Actor { template - explicit Actor(std::unordered_map& results, unsigned long id, Str&& name) : results(results), id(id), name(std::forward(name)) {} + explicit Actor(std::unordered_map& results, unsigned long id, Str&& name) + : results(results), id(id), name(std::forward(name)) {} Actor(const Actor&) = delete; ~Actor() { collect(); } std::unordered_map& results; @@ -40,12 +41,12 @@ struct Actor { for (auto i = stack.begin(); i != stack.end();) { int num = 0; auto name = *i; - for (; i != stack.end() && *i == name ; ++i) { + for (; i != stack.end() && *i == name; ++i) { ++num; } ss << name; if (num > 1) { - ss << " ("<< num << ')'; + ss << " (" << num << ')'; } ss << ';'; } diff --git a/fdbbackup/FileConverter.actor.cpp b/fdbbackup/FileConverter.actor.cpp index 3556b3ae98..ddea0f30cf 100644 --- a/fdbbackup/FileConverter.actor.cpp +++ b/fdbbackup/FileConverter.actor.cpp @@ -142,13 +142,17 @@ struct MutationFilesReadProgress : public ReferenceCounted f, int index) : fd(f), idx(index), offset(0), eof(false) {} bool operator<(const FileProgress& rhs) const { - if (rhs.mutations.empty()) return true; - if (mutations.empty()) return false; + if (rhs.mutations.empty()) + return true; + if (mutations.empty()) + return false; return mutations[0].version < rhs.mutations[0].version; } bool operator<=(const FileProgress& rhs) const { - if (rhs.mutations.empty()) return true; - if (mutations.empty()) return false; + if (rhs.mutations.empty()) + return true; + if (mutations.empty()) + return false; return mutations[0].version <= rhs.mutations[0].version; } bool empty() { return eof && mutations.empty(); } @@ -163,11 +167,13 @@ struct MutationFilesReadProgress : public ReferenceCounted() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version(); + if (reader.consume() != PARTITIONED_MLOG_VERSION) + throw restore_unsupported_file_version(); while (1) { // If eof reached or first key len bytes is 0xFF then end of block was reached. - if (reader.eof() || *reader.rptr == 0xFF) break; + if (reader.eof() || *reader.rptr == 0xFF) + break; // Deserialize messages written in saveMutationsToFile(). msgVersion = bigEndian64(reader.consume()); @@ -188,8 +194,8 @@ struct MutationFilesReadProgress : public ReferenceCounted= minVersion) { - mutations.emplace_back(LogMessageVersion(msgVersion, sub), StringRef(message, msgSize), - buf.arena()); + mutations.emplace_back( + LogMessageVersion(msgVersion, sub), StringRef(message, msgSize), buf.arena()); inserted++; } } @@ -226,7 +232,8 @@ struct MutationFilesReadProgress : public ReferenceCountedempty()) return true; + if (!fp->empty()) + return true; } return false; } @@ -246,7 +253,8 @@ struct MutationFilesReadProgress : public ReferenceCounted& a, const Reference& b) { return (*a) < (*b); }); while (!fileProgress.empty() && fileProgress.back()->empty()) { fileProgress.pop_back(); @@ -313,11 +321,15 @@ struct MutationFilesReadProgress : public ReferenceCounted= minVersion and saves these mutations. // Skip mutations >= maxVersion. - ACTOR static Future decodeToVersion(Reference fp, Version minVersion, Version maxVersion, + ACTOR static Future decodeToVersion(Reference fp, + Version minVersion, + Version maxVersion, LogFile file) { - if (fp->empty()) return Void(); + if (fp->empty()) + return Void(); - if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion) return Void(); + if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion) + return Void(); state int64_t len; try { @@ -331,13 +343,15 @@ struct MutationFilesReadProgress : public ReferenceCounted buf = makeString(len); int rLen = wait(fp->fd->read(mutateString(buf), len, fp->offset)); - if (len != rLen) throw restore_bad_read(); + if (len != rLen) + throw restore_bad_read(); TraceEvent("ReadFile") .detail("Name", fp->fd->getFilename()) .detail("Length", rLen) .detail("Offset", fp->offset); - if (fp->decodeBlock(buf, rLen, minVersion, maxVersion)) break; + if (fp->decodeBlock(buf, rLen, minVersion, maxVersion)) + break; } return Void(); } catch (Error& e) { @@ -396,7 +410,8 @@ struct LogFileWriter { wait(self->file->appendStringRefWithLen(v)); // At this point we should be in whatever the current block is or the block size is too small - if (self->file->size() > self->blockEnd) throw backup_bad_block_size(); + if (self->file->size() > self->blockEnd) + throw backup_bad_block_size(); return Void(); } @@ -540,7 +555,7 @@ int parseCommandLine(ConvertParams* param, CSimpleOpt* args) { return FDB_EXIT_SUCCESS; } -} // namespace file_converter +} // namespace file_converter int main(int argc, char** argv) { try { diff --git a/fdbbackup/FileConverter.h b/fdbbackup/FileConverter.h index e01566b889..8038bc6a22 100644 --- a/fdbbackup/FileConverter.h +++ b/fdbbackup/FileConverter.h @@ -59,6 +59,6 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP }, { OPT_HELP, "--help", SO_NONE }, SO_END_OF_OPTIONS }; -} // namespace file_converter +} // namespace file_converter -#endif // FDBBACKUP_FILECONVERTER_H +#endif // FDBBACKUP_FILECONVERTER_H diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp index f294f724c9..40434ac920 100644 --- a/fdbbackup/FileDecoder.actor.cpp +++ b/fdbbackup/FileDecoder.actor.cpp @@ -179,7 +179,8 @@ std::vector decode_value(const StringRef& value) { std::vector mutations; while (1) { - if (reader.eof()) break; + if (reader.eof()) + break; // Deserialization of a MutationRef, which was packed by MutationListRef::push_back_deep() uint32_t type, p1len, p2len; @@ -220,8 +221,7 @@ struct VersionedMutations { struct DecodeProgress { DecodeProgress() = default; template - DecodeProgress(const LogFile& file, U &&values) - : file(file), keyValues(std::forward(values)) {} + DecodeProgress(const LogFile& file, U&& values) : file(file), keyValues(std::forward(values)) {} // If there are no more mutations to pull from the file. // However, we could have unfinished version in the buffer when EOF is true, @@ -229,7 +229,9 @@ struct DecodeProgress { // should call getUnfinishedBuffer() to get these left data. bool finished() { return (eof && keyValues.empty()) || (leftover && !keyValues.empty()); } - std::vector>&& getUnfinishedBuffer() && { return std::move(keyValues); } + std::vector>&& getUnfinishedBuffer() && { + return std::move(keyValues); + } // Returns all mutations of the next version in a batch. Future getNextBatch() { return getNextBatchImpl(this); } @@ -267,7 +269,8 @@ struct DecodeProgress { int idx = 1; // next kv pair in "keyValues" int bufSize = std::get<3>(tuple).size(); for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) { - if (idx == self->keyValues.size()) break; + if (idx == self->keyValues.size()) + break; auto next_tuple = self->keyValues[idx]; if (std::get<1>(tuple) != std::get<1>(next_tuple)) { @@ -333,12 +336,14 @@ struct DecodeProgress { try { // Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION - if (reader.consume() != BACKUP_AGENT_MLOG_VERSION) throw restore_unsupported_file_version(); + if (reader.consume() != BACKUP_AGENT_MLOG_VERSION) + throw restore_unsupported_file_version(); // Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte. while (1) { // If eof reached or first key len bytes is 0xFF then end of block was reached. - if (reader.eof() || *reader.rptr == 0xFF) break; + if (reader.eof() || *reader.rptr == 0xFF) + break; // Read key and value. If anything throws then there is a problem. uint32_t kLen = reader.consumeNetworkUInt32(); @@ -357,13 +362,15 @@ struct DecodeProgress { // Make sure any remaining bytes in the block are 0xFF for (auto b : reader.remainder()) { - if (b != 0xFF) throw restore_corrupted_data_padding(); + if (b != 0xFF) + throw restore_corrupted_data_padding(); } // The (version, part) in a block can be out of order, i.e., (3, 0) // can be followed by (4, 0), and then (3, 1). So we need to sort them // first by version, and then by part number. - std::sort(keyValues.begin(), keyValues.end(), + std::sort(keyValues.begin(), + keyValues.end(), [](const std::tuple& a, const std::tuple& b) { return std::get<1>(a) == std::get<1>(b) ? std::get<2>(a) < std::get<2>(b) @@ -428,7 +435,8 @@ ACTOR Future decode_logs(DecodeParams params) { state BackupFileList listing = wait(container->dumpFileList()); // remove partitioned logs - listing.logs.erase(std::remove_if(listing.logs.begin(), listing.logs.end(), + listing.logs.erase(std::remove_if(listing.logs.begin(), + listing.logs.end(), [](const LogFile& file) { std::string prefix("plogs/"); return file.fileName.substr(0, prefix.size()) == prefix; @@ -447,7 +455,8 @@ ACTOR Future decode_logs(DecodeParams params) { // Previous file's unfinished version data state std::vector> left; for (; i < logs.size(); i++) { - if (logs[i].fileSize == 0) continue; + if (logs[i].fileSize == 0) + continue; state DecodeProgress progress(logs[i], std::move(left)); wait(progress.openFile(container)); diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 6717978f27..f1f77c40c6 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -46,7 +46,7 @@ #include #include #include -#include // std::transform +#include // std::transform #include #include #include @@ -60,7 +60,7 @@ using std::endl; #endif #include -#ifdef __linux__ +#ifdef __linux__ #include #ifdef ALLOC_INSTRUMENTATION #include @@ -70,8 +70,7 @@ using std::endl; #include "fdbclient/versions.h" #include "flow/SimpleOpt.h" -#include "flow/actorcompiler.h" // This must be the last #include. - +#include "flow/actorcompiler.h" // This must be the last #include. // Type of program being executed enum enumProgramExe { @@ -103,41 +102,74 @@ enum enumBackupType { BACKUP_CLEANUP }; -enum enumDBType { - DB_UNDEFINED=0, DB_START, DB_STATUS, DB_SWITCH, DB_ABORT, DB_PAUSE, DB_RESUME -}; +enum enumDBType { DB_UNDEFINED = 0, DB_START, DB_STATUS, DB_SWITCH, DB_ABORT, DB_PAUSE, DB_RESUME }; // New fast restore reuses the type from legacy slow restore -enum enumRestoreType { - RESTORE_UNKNOWN, RESTORE_START, RESTORE_STATUS, RESTORE_ABORT, RESTORE_WAIT -}; +enum enumRestoreType { RESTORE_UNKNOWN, RESTORE_START, RESTORE_STATUS, RESTORE_ABORT, RESTORE_WAIT }; // enum { // Backup constants - OPT_DESTCONTAINER, OPT_SNAPSHOTINTERVAL, OPT_ERRORLIMIT, OPT_NOSTOPWHENDONE, - OPT_EXPIRE_BEFORE_VERSION, OPT_EXPIRE_BEFORE_DATETIME, OPT_EXPIRE_DELETE_BEFORE_DAYS, - OPT_EXPIRE_RESTORABLE_AFTER_VERSION, OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, OPT_EXPIRE_MIN_RESTORABLE_DAYS, - OPT_BASEURL, OPT_BLOB_CREDENTIALS, OPT_DESCRIBE_DEEP, OPT_DESCRIBE_TIMESTAMPS, - OPT_DUMP_BEGIN, OPT_DUMP_END, OPT_JSON, OPT_DELETE_DATA, OPT_MIN_CLEANUP_SECONDS, + OPT_DESTCONTAINER, + OPT_SNAPSHOTINTERVAL, + OPT_ERRORLIMIT, + OPT_NOSTOPWHENDONE, + OPT_EXPIRE_BEFORE_VERSION, + OPT_EXPIRE_BEFORE_DATETIME, + OPT_EXPIRE_DELETE_BEFORE_DAYS, + OPT_EXPIRE_RESTORABLE_AFTER_VERSION, + OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, + OPT_EXPIRE_MIN_RESTORABLE_DAYS, + OPT_BASEURL, + OPT_BLOB_CREDENTIALS, + OPT_DESCRIBE_DEEP, + OPT_DESCRIBE_TIMESTAMPS, + OPT_DUMP_BEGIN, + OPT_DUMP_END, + OPT_JSON, + OPT_DELETE_DATA, + OPT_MIN_CLEANUP_SECONDS, OPT_USE_PARTITIONED_LOG, // Backup and Restore constants - OPT_TAGNAME, OPT_BACKUPKEYS, OPT_WAITFORDONE, OPT_BACKUPKEYS_FILTER, + OPT_TAGNAME, + OPT_BACKUPKEYS, + OPT_WAITFORDONE, + OPT_BACKUPKEYS_FILTER, // Backup Modify - OPT_MOD_ACTIVE_INTERVAL, OPT_MOD_VERIFY_UID, + OPT_MOD_ACTIVE_INTERVAL, + OPT_MOD_VERIFY_UID, // Restore constants - OPT_RESTORECONTAINER, OPT_RESTORE_VERSION, OPT_RESTORE_TIMESTAMP, OPT_PREFIX_ADD, OPT_PREFIX_REMOVE, OPT_RESTORE_CLUSTERFILE_DEST, OPT_RESTORE_CLUSTERFILE_ORIG, + OPT_RESTORECONTAINER, + OPT_RESTORE_VERSION, + OPT_RESTORE_TIMESTAMP, + OPT_PREFIX_ADD, + OPT_PREFIX_REMOVE, + OPT_RESTORE_CLUSTERFILE_DEST, + OPT_RESTORE_CLUSTERFILE_ORIG, // Shared constants - OPT_CLUSTERFILE, OPT_QUIET, OPT_DRYRUN, OPT_FORCE, - OPT_HELP, OPT_DEVHELP, OPT_VERSION, OPT_PARENTPID, OPT_CRASHONERROR, - OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR, OPT_TRACE, OPT_TRACE_DIR, - OPT_KNOB, OPT_TRACE_LOG_GROUP, OPT_MEMLIMIT, OPT_LOCALITY, + OPT_CLUSTERFILE, + OPT_QUIET, + OPT_DRYRUN, + OPT_FORCE, + OPT_HELP, + OPT_DEVHELP, + OPT_VERSION, + OPT_PARENTPID, + OPT_CRASHONERROR, + OPT_NOBUFSTDOUT, + OPT_BUFSTDOUTERR, + OPT_TRACE, + OPT_TRACE_DIR, + OPT_KNOB, + OPT_TRACE_LOG_GROUP, + OPT_MEMLIMIT, + OPT_LOCALITY, - //DB constants + // DB constants OPT_SOURCE_CLUSTER, OPT_DEST_CLUSTER, OPT_CLEANUP, @@ -178,78 +210,78 @@ CSimpleOpt::SOption g_rgAgentOptions[] = { CSimpleOpt::SOption g_rgBackupStartOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_WAITFORDONE, "-w", SO_NONE }, - { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, - { OPT_NOSTOPWHENDONE, "-z", SO_NONE }, - { OPT_NOSTOPWHENDONE, "--no-stop-when-done",SO_NONE }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_WAITFORDONE, "-w", SO_NONE }, + { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, + { OPT_NOSTOPWHENDONE, "-z", SO_NONE }, + { OPT_NOSTOPWHENDONE, "--no-stop-when-done", SO_NONE }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, // Enable "-p" option after GA // { OPT_USE_PARTITIONED_LOG, "-p", SO_NONE }, - { OPT_USE_PARTITIONED_LOG, "--partitioned_log_experimental", SO_NONE }, - { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP }, - { OPT_SNAPSHOTINTERVAL, "--snapshot_interval", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, - { OPT_DRYRUN, "-n", SO_NONE }, - { OPT_DRYRUN, "--dryrun", SO_NONE }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_USE_PARTITIONED_LOG, "--partitioned_log_experimental", SO_NONE }, + { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP }, + { OPT_SNAPSHOTINTERVAL, "--snapshot_interval", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, + { OPT_DRYRUN, "-n", SO_NONE }, + { OPT_DRYRUN, "--dryrun", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupModifyOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_MOD_VERIFY_UID, "--verify_uid", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_MOD_VERIFY_UID, "--verify_uid", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP }, { OPT_SNAPSHOTINTERVAL, "--snapshot_interval", SO_REQ_SEP }, { OPT_MOD_ACTIVE_INTERVAL, "--active_snapshot_interval", SO_REQ_SEP }, @@ -258,350 +290,350 @@ CSimpleOpt::SOption g_rgBackupModifyOptions[] = { CSimpleOpt::SOption g_rgBackupStatusOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_ERRORLIMIT, "-e", SO_REQ_SEP }, - { OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_JSON, "--json", SO_NONE}, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_ERRORLIMIT, "-e", SO_REQ_SEP }, + { OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_JSON, "--json", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupAbortOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupCleanupOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_DELETE_DATA, "--delete_data", SO_NONE }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_DELETE_DATA, "--delete_data", SO_NONE }, { OPT_MIN_CLEANUP_SECONDS, "--min_cleanup_seconds", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupDiscontinueOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_WAITFORDONE, "-w", SO_NONE }, - { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_WAITFORDONE, "-w", SO_NONE }, + { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupWaitOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_NOSTOPWHENDONE, "-z", SO_NONE }, - { OPT_NOSTOPWHENDONE, "--no-stop-when-done",SO_NONE }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_NOSTOPWHENDONE, "-z", SO_NONE }, + { OPT_NOSTOPWHENDONE, "--no-stop-when-done", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupPauseOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupExpireOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_FORCE, "-f", SO_NONE }, - { OPT_FORCE, "--force", SO_NONE }, - { OPT_EXPIRE_RESTORABLE_AFTER_VERSION, "--restorable_after_version", SO_REQ_SEP }, - { OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, "--restorable_after_timestamp", SO_REQ_SEP }, - { OPT_EXPIRE_BEFORE_VERSION, "--expire_before_version", SO_REQ_SEP }, - { OPT_EXPIRE_BEFORE_DATETIME, "--expire_before_timestamp", SO_REQ_SEP }, - { OPT_EXPIRE_MIN_RESTORABLE_DAYS, "--min_restorable_days", SO_REQ_SEP }, - { OPT_EXPIRE_DELETE_BEFORE_DAYS, "--delete_before_days", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_FORCE, "-f", SO_NONE }, + { OPT_FORCE, "--force", SO_NONE }, + { OPT_EXPIRE_RESTORABLE_AFTER_VERSION, "--restorable_after_version", SO_REQ_SEP }, + { OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, "--restorable_after_timestamp", SO_REQ_SEP }, + { OPT_EXPIRE_BEFORE_VERSION, "--expire_before_version", SO_REQ_SEP }, + { OPT_EXPIRE_BEFORE_DATETIME, "--expire_before_timestamp", SO_REQ_SEP }, + { OPT_EXPIRE_MIN_RESTORABLE_DAYS, "--min_restorable_days", SO_REQ_SEP }, + { OPT_EXPIRE_DELETE_BEFORE_DAYS, "--delete_before_days", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupDeleteOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupDescribeOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_DESCRIBE_DEEP, "--deep", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_DESCRIBE_DEEP, "--deep", SO_NONE }, { OPT_DESCRIBE_TIMESTAMPS, "--version_timestamps", SO_NONE }, - { OPT_JSON, "--json", SO_NONE}, + { OPT_JSON, "--json", SO_NONE }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupDumpOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_DUMP_BEGIN, "--begin", SO_REQ_SEP }, - { OPT_DUMP_END, "--end", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_DUMP_BEGIN, "--begin", SO_REQ_SEP }, + { OPT_DUMP_END, "--end", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupListOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_BASEURL, "-b", SO_REQ_SEP }, - { OPT_BASEURL, "--base_url", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_BASEURL, "-b", SO_REQ_SEP }, + { OPT_BASEURL, "--base_url", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgBackupQueryOptions[] = { @@ -641,240 +673,240 @@ CSimpleOpt::SOption g_rgBackupQueryOptions[] = { // g_rgRestoreOptions is used by fdbrestore and fastrestore_tool CSimpleOpt::SOption g_rgRestoreOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_RESTORE_CLUSTERFILE_DEST, "--dest_cluster_file", SO_REQ_SEP }, - { OPT_RESTORE_CLUSTERFILE_ORIG, "--orig_cluster_file", SO_REQ_SEP }, - { OPT_RESTORE_TIMESTAMP, "--timestamp", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_RESTORECONTAINER,"-r", SO_REQ_SEP }, - { OPT_PREFIX_ADD, "--add_prefix", SO_REQ_SEP }, - { OPT_PREFIX_REMOVE, "--remove_prefix", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, - { OPT_WAITFORDONE, "-w", SO_NONE }, - { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, - { OPT_RESTORE_VERSION, "--version", SO_REQ_SEP }, - { OPT_RESTORE_VERSION, "-v", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_DRYRUN, "-n", SO_NONE }, - { OPT_DRYRUN, "--dryrun", SO_NONE }, - { OPT_FORCE, "-f", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_RESTORE_CLUSTERFILE_DEST, "--dest_cluster_file", SO_REQ_SEP }, + { OPT_RESTORE_CLUSTERFILE_ORIG, "--orig_cluster_file", SO_REQ_SEP }, + { OPT_RESTORE_TIMESTAMP, "--timestamp", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_RESTORECONTAINER, "-r", SO_REQ_SEP }, + { OPT_PREFIX_ADD, "--add_prefix", SO_REQ_SEP }, + { OPT_PREFIX_REMOVE, "--remove_prefix", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, + { OPT_WAITFORDONE, "-w", SO_NONE }, + { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, + { OPT_RESTORE_VERSION, "--version", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "-v", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_DRYRUN, "-n", SO_NONE }, + { OPT_DRYRUN, "--dryrun", SO_NONE }, + { OPT_FORCE, "-f", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgDBAgentOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_LOCALITY, "--locality_", SO_REQ_SEP }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_LOCALITY, "--locality_", SO_REQ_SEP }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgDBStartOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgDBStatusOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_ERRORLIMIT, "-e", SO_REQ_SEP }, - { OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_ERRORLIMIT, "-e", SO_REQ_SEP }, + { OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgDBSwitchOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_FORCE, "-f", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_FORCE, "-f", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgDBAbortOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_CLEANUP, "--cleanup", SO_NONE }, - { OPT_DSTONLY, "--dstonly", SO_NONE }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_CLEANUP, "--cleanup", SO_NONE }, + { OPT_DSTONLY, "--dstonly", SO_NONE }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; CSimpleOpt::SOption g_rgDBPauseOptions[] = { #ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, #endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS + SO_END_OF_OPTIONS }; const KeyRef exeAgent = LiteralStringRef("backup_agent"); @@ -887,7 +919,7 @@ const KeyRef exeDatabaseBackup = LiteralStringRef("fdbdr"); extern const char* getSourceVersion(); #ifdef _WIN32 -void parentWatcher(void *parentHandle) { +void parentWatcher(void* parentHandle) { HANDLE parent = (HANDLE)parentHandle; int signal = WaitForSingleObject(parent, INFINITE); CloseHandle(parentHandle); @@ -901,22 +933,25 @@ void parentWatcher(void *parentHandle) { static void printVersion() { printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); printf("source version %s\n", getSourceVersion()); - printf("protocol %llx\n", (long long) currentProtocolVersion.version()); + printf("protocol %llx\n", (long long)currentProtocolVersion.version()); } -const char *BlobCredentialInfo = - " BLOB CREDENTIALS\n" - " Blob account secret keys can optionally be omitted from blobstore:// URLs, in which case they will be\n" - " loaded, if possible, from 1 or more blob credentials definition files.\n\n" - " These files can be specified with the --blob_credentials argument described above or via the environment variable\n" - " FDB_BLOB_CREDENTIALS, whose value is a colon-separated list of files. The command line takes priority over\n" - " over the environment but all files from both sources are used.\n\n" - " At connect time, the specified files are read in order and the first matching account specification (user@host)\n" - " will be used to obtain the secret key.\n\n" - " The JSON schema is:\n" - " { \"accounts\" : { \"user@host\" : { \"secret\" : \"SECRETKEY\" }, \"user2@host2\" : { \"secret\" : \"SECRET\" } } }\n"; +const char* BlobCredentialInfo = + " BLOB CREDENTIALS\n" + " Blob account secret keys can optionally be omitted from blobstore:// URLs, in which case they will be\n" + " loaded, if possible, from 1 or more blob credentials definition files.\n\n" + " These files can be specified with the --blob_credentials argument described above or via the environment " + "variable\n" + " FDB_BLOB_CREDENTIALS, whose value is a colon-separated list of files. The command line takes priority over\n" + " over the environment but all files from both sources are used.\n\n" + " At connect time, the specified files are read in order and the first matching account specification " + "(user@host)\n" + " will be used to obtain the secret key.\n\n" + " The JSON schema is:\n" + " { \"accounts\" : { \"user@host\" : { \"secret\" : \"SECRETKEY\" }, \"user2@host2\" : { \"secret\" : " + "\"SECRET\" } } }\n"; -static void printHelpTeaser( const char *name ) { +static void printHelpTeaser(const char* name) { fprintf(stderr, "Try `%s --help' for more information.\n", name); } @@ -924,22 +959,23 @@ static void printAgentUsage(bool devhelp) { printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); printf("Usage: %s [OPTIONS]\n\n", exeAgent.toString().c_str()); printf(" -C CONNFILE The path of a file containing the connection string for the\n" - " FoundationDB cluster. The default is first the value of the\n" - " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" - " then `%s'.\n", platform::getDefaultClusterFilePath().c_str()); + " FoundationDB cluster. The default is first the value of the\n" + " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" + " then `%s'.\n", + platform::getDefaultClusterFilePath().c_str()); printf(" --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifes the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n"); + " --logdir PATH Specifes the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); printf(" --loggroup LOG_GROUP\n" " Sets the LogGroup field with the specified value for all\n" " events in the trace output (defaults to `default').\n"); printf(" --trace_format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); printf(" -m SIZE, --memory SIZE\n" - " Memory limit. The default value is 8GiB. When specified\n" - " without a unit, MiB is assumed.\n"); + " Memory limit. The default value is 8GiB. When specified\n" + " without a unit, MiB is assumed.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif @@ -964,7 +1000,7 @@ static void printAgentUsage(bool devhelp) { void printBackupContainerInfo() { printf(" Backup URL forms:\n\n"); std::vector formats = IBackupContainer::getURLFormats(); - for(auto &f : formats) + for (auto& f : formats) printf(" %s\n", f.c_str()); printf("\n"); } @@ -975,47 +1011,66 @@ static void printBackupUsage(bool devhelp) { "list | query | cleanup) [OPTIONS]\n\n", exeBackup.toString().c_str()); printf(" -C CONNFILE The path of a file containing the connection string for the\n" - " FoundationDB cluster. The default is first the value of the\n" - " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" - " then `%s'.\n", platform::getDefaultClusterFilePath().c_str()); + " FoundationDB cluster. The default is first the value of the\n" + " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" + " then `%s'.\n", + platform::getDefaultClusterFilePath().c_str()); printf(" -d, --destcontainer URL\n" " The Backup container URL for start, modify, describe, query, expire, and delete " "operations.\n"); printBackupContainerInfo(); printf(" -b, --base_url BASEURL\n" - " Base backup URL for list operations. This looks like a Backup URL but without a backup name.\n"); + " Base backup URL for list operations. This looks like a Backup URL but without a backup " + "name.\n"); printf(" --blob_credentials FILE\n" - " File containing blob credentials in JSON format. Can be specified multiple times for multiple files. See below for more details.\n"); + " File containing blob credentials in JSON format. Can be specified multiple times for " + "multiple files. See below for more details.\n"); printf(" --expire_before_timestamp DATETIME\n" - " Datetime cutoff for expire operations. Requires a cluster file and will use version/timestamp metadata\n" - " in the database to obtain a cutoff version very close to the timestamp given in %s.\n", BackupAgentBase::timeFormat().c_str()); + " Datetime cutoff for expire operations. Requires a cluster file and will use " + "version/timestamp metadata\n" + " in the database to obtain a cutoff version very close to the timestamp given in %s.\n", + BackupAgentBase::timeFormat().c_str()); printf(" --expire_before_version VERSION\n" - " Version cutoff for expire operations. Deletes data files containing no data at or after VERSION.\n"); + " Version cutoff for expire operations. Deletes data files containing no data at or after " + "VERSION.\n"); printf(" --delete_before_days NUM_DAYS\n" - " Another way to specify version cutoff for expire operations. Deletes data files containing no data at or after a\n" - " version approximately NUM_DAYS days worth of versions prior to the latest log version in the backup.\n"); + " Another way to specify version cutoff for expire operations. Deletes data files " + "containing no data at or after a\n" + " version approximately NUM_DAYS days worth of versions prior to the latest log version in " + "the backup.\n"); printf(" -qrv --query_restore_version VERSION\n" " For query operations, set target version for restoring a backup. Set -1 for maximum\n" " restorable version (default) and -2 for minimum restorable version.\n"); - printf(" --query_restore_timestamp DATETIME\n" - " For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", BackupAgentBase::timeFormat().c_str()); - printf(" and it will be converted to a version from that time using metadata in the cluster file.\n"); + printf( + " --query_restore_timestamp DATETIME\n" + " For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", + BackupAgentBase::timeFormat().c_str()); + printf( + " and it will be converted to a version from that time using metadata in the cluster file.\n"); printf(" --restorable_after_timestamp DATETIME\n" - " For expire operations, set minimum acceptable restorability to the version equivalent of DATETIME and later.\n"); + " For expire operations, set minimum acceptable restorability to the version equivalent of " + "DATETIME and later.\n"); printf(" --restorable_after_version VERSION\n" - " For expire operations, set minimum acceptable restorability to the VERSION and later.\n"); + " For expire operations, set minimum acceptable restorability to the VERSION and later.\n"); printf(" --min_restorable_days NUM_DAYS\n" - " For expire operations, set minimum acceptable restorability to approximately NUM_DAYS days worth of versions\n" - " prior to the latest log version in the backup.\n"); + " For expire operations, set minimum acceptable restorability to approximately NUM_DAYS " + "days worth of versions\n" + " prior to the latest log version in the backup.\n"); printf(" --version_timestamps\n"); - printf(" For describe operations, lookup versions in the database to obtain timestamps. A cluster file is required.\n"); - printf(" -f, --force For expire operations, force expiration even if minimum restorability would be violated.\n"); + printf(" For describe operations, lookup versions in the database to obtain timestamps. A cluster " + "file is required.\n"); + printf( + " -f, --force For expire operations, force expiration even if minimum restorability would be violated.\n"); printf(" -s, --snapshot_interval DURATION\n" - " For start or modify operations, specifies the backup's default target snapshot interval as DURATION seconds. Defaults to %d for start operations.\n", CLIENT_KNOBS->BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC); + " For start or modify operations, specifies the backup's default target snapshot interval " + "as DURATION seconds. Defaults to %d for start operations.\n", + CLIENT_KNOBS->BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC); printf(" --active_snapshot_interval DURATION\n" - " For modify operations, sets the desired interval for the backup's currently active snapshot, relative to the start of the snapshot.\n"); + " For modify operations, sets the desired interval for the backup's currently active " + "snapshot, relative to the start of the snapshot.\n"); printf(" --verify_uid UID\n" - " Specifies a UID to verify against the BackupUID of the running backup. If provided, the UID is verified in the same transaction\n" + " Specifies a UID to verify against the BackupUID of the running backup. If provided, the " + "UID is verified in the same transaction\n" " which sets the new backup parameters (if the UID matches).\n"); printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n"); printf(" -k KEYS List of key ranges to backup or to filter the backup in query operations.\n" @@ -1023,27 +1078,27 @@ static void printBackupUsage(bool devhelp) { printf(" --partitioned_log_experimental Starts with new type of backup system using partitioned logs.\n"); printf(" -n, --dryrun For backup start or restore start, performs a trial run with no actual changes made.\n"); printf(" --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifes the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n"); + " --logdir PATH Specifes the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); printf(" --loggroup LOG_GROUP\n" " Sets the LogGroup field with the specified value for all\n" " events in the trace output (defaults to `default').\n"); printf(" --trace_format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); printf(" --max_cleanup_seconds SECONDS\n" " Specifies the amount of time a backup or DR needs to be stale before cleanup will\n" " remove mutations for it. By default this is set to one hour.\n"); printf(" --delete_data\n" - " This flag will cause cleanup to remove mutations for the most stale backup or DR.\n"); + " This flag will cause cleanup to remove mutations for the most stale backup or DR.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif printf(" -v, --version Print version information and exit.\n"); printf(" -w, --wait Wait for the backup to complete (allowed with `start' and `discontinue').\n"); printf(" -z, --no-stop-when-done\n" - " Do not stop backup when restorable.\n"); + " Do not stop backup when restorable.\n"); printf(" -h, --help Display this help and exit.\n"); if (devhelp) { @@ -1054,20 +1109,19 @@ static void printBackupUsage(bool devhelp) { printf(" Specify a process after whose termination to exit.\n"); #endif printf(" --deep For describe operations, do not use cached metadata. Warning: Very slow\n"); - } printf("\n" - " KEYS FORMAT: \" \" [...]\n"); + " KEYS FORMAT: \" \" [...]\n"); printf("\n"); puts(BlobCredentialInfo); return; } -static void printRestoreUsage(bool devhelp ) { +static void printRestoreUsage(bool devhelp) { printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); printf("Usage: %s (start | status | abort | wait) [OPTIONS]\n\n", exeRestore.toString().c_str()); - //printf(" FOLDERS Paths to folders containing the backup files.\n"); + // printf(" FOLDERS Paths to folders containing the backup files.\n"); printf("Options for all commands:\n\n"); printf(" --dest_cluster_file CONNFILE\n"); printf(" The cluster file to restore data into.\n"); @@ -1092,20 +1146,23 @@ static void printRestoreUsage(bool devhelp ) { " Sets the LogGroup field with the specified value for all\n" " events in the trace output (defaults to `default').\n"); printf(" --trace_format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif printf(" -v DBVERSION The version at which the database will be restored.\n"); - printf(" --timestamp Instead of a numeric version, use this to specify a timestamp in %s\n", BackupAgentBase::timeFormat().c_str()); - printf(" and it will be converted to a version from that time using metadata in orig_cluster_file.\n"); + printf(" --timestamp Instead of a numeric version, use this to specify a timestamp in %s\n", + BackupAgentBase::timeFormat().c_str()); + printf( + " and it will be converted to a version from that time using metadata in orig_cluster_file.\n"); printf(" --orig_cluster_file CONNFILE\n"); - printf(" The cluster file for the original database from which the backup was created. The original database\n"); + printf(" The cluster file for the original database from which the backup was created. The " + "original database\n"); printf(" is only needed to convert a --timestamp argument to a database version.\n"); printf(" -h, --help Display this help and exit.\n"); - if( devhelp ) { + if (devhelp) { #ifdef _WIN32 printf(" -q Disable error dialog on crash.\n"); printf(" --parentpid PID\n"); @@ -1114,7 +1171,7 @@ static void printRestoreUsage(bool devhelp ) { } printf("\n" - " KEYS FORMAT: \" \" [...]\n"); + " KEYS FORMAT: \" \" [...]\n"); printf("\n"); puts(BlobCredentialInfo); @@ -1133,22 +1190,22 @@ static void printDBAgentUsage(bool devhelp) { printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); printf("Usage: %s [OPTIONS]\n\n", exeDatabaseAgent.toString().c_str()); printf(" -d CONNFILE The path of a file containing the connection string for the\n" - " destination FoundationDB cluster.\n"); + " destination FoundationDB cluster.\n"); printf(" -s CONNFILE The path of a file containing the connection string for the\n" - " source FoundationDB cluster.\n"); + " source FoundationDB cluster.\n"); printf(" --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifes the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n"); + " --logdir PATH Specifes the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); printf(" --loggroup LOG_GROUP\n" " Sets the LogGroup field with the specified value for all\n" " events in the trace output (defaults to `default').\n"); printf(" --trace_format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); printf(" -m SIZE, --memory SIZE\n" - " Memory limit. The default value is 8GiB. When specified\n" - " without a unit, MiB is assumed.\n"); + " Memory limit. The default value is 8GiB. When specified\n" + " without a unit, MiB is assumed.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif @@ -1168,35 +1225,36 @@ static void printDBAgentUsage(bool devhelp) { static void printDBBackupUsage(bool devhelp) { printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); - printf("Usage: %s (start | status | switch | abort | pause | resume) [OPTIONS]\n\n", exeDatabaseBackup.toString().c_str()); + printf("Usage: %s (start | status | switch | abort | pause | resume) [OPTIONS]\n\n", + exeDatabaseBackup.toString().c_str()); printf(" -d, --destination CONNFILE\n" " The path of a file containing the connection string for the\n"); printf(" destination FoundationDB cluster.\n"); printf(" -s, --source CONNFILE\n" " The path of a file containing the connection string for the\n" - " source FoundationDB cluster.\n"); + " source FoundationDB cluster.\n"); printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n"); printf(" -k KEYS List of key ranges to backup.\n" - " If not specified, the entire database will be backed up.\n"); + " If not specified, the entire database will be backed up.\n"); printf(" --cleanup Abort will attempt to stop mutation logging on the source cluster.\n"); printf(" --dstonly Abort will not make any changes on the source cluster.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif printf(" --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifes the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n"); + " --logdir PATH Specifes the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); printf(" --loggroup LOG_GROUP\n" " Sets the LogGroup field with the specified value for all\n" " events in the trace output (defaults to `default').\n"); printf(" --trace_format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); printf(" -v, --version Print version information and exit.\n"); printf(" -h, --help Display this help and exit.\n"); printf("\n" - " KEYS FORMAT: \" \" [...]\n"); + " KEYS FORMAT: \" \" [...]\n"); if (devhelp) { #ifdef _WIN32 @@ -1210,11 +1268,9 @@ static void printDBBackupUsage(bool devhelp) { return; } -static void printUsage(enumProgramExe programExe, bool devhelp) -{ +static void printUsage(enumProgramExe programExe, bool devhelp) { - switch (programExe) - { + switch (programExe) { case EXE_AGENT: printAgentUsage(devhelp); break; @@ -1244,9 +1300,8 @@ static void printUsage(enumProgramExe programExe, bool devhelp) extern bool g_crashOnError; // Return the type of program executable based on the name of executable file -enumProgramExe getProgramType(std::string programExe) -{ - enumProgramExe enProgramExe = EXE_UNDEFINED; +enumProgramExe getProgramType(std::string programExe) { + enumProgramExe enProgramExe = EXE_UNDEFINED; // lowercase the string std::transform(programExe.begin(), programExe.end(), programExe.begin(), ::tolower); @@ -1258,55 +1313,56 @@ enumProgramExe getProgramType(std::string programExe) size_t lastSlash = programExe.find_last_of("\\"); // Ensure last dot is after last slash, if present - if ((lastSlash == std::string::npos)|| - (lastSlash < lastDot) ) - { + if ((lastSlash == std::string::npos) || (lastSlash < lastDot)) { programExe = programExe.substr(0, lastDot); } } #endif // For debugging convenience, remove .debug suffix if present. - if(StringRef(programExe).endsWith(LiteralStringRef(".debug"))) + if (StringRef(programExe).endsWith(LiteralStringRef(".debug"))) programExe = programExe.substr(0, programExe.size() - 6); // Check if backup agent - if ((programExe.length() >= exeAgent.size()) && - (programExe.compare(programExe.length()-exeAgent.size(), exeAgent.size(), (const char*) exeAgent.begin()) == 0) ) - { + if ((programExe.length() >= exeAgent.size()) && + (programExe.compare(programExe.length() - exeAgent.size(), exeAgent.size(), (const char*)exeAgent.begin()) == + 0)) { enProgramExe = EXE_AGENT; } // Check if backup - else if ((programExe.length() >= exeBackup.size()) && - (programExe.compare(programExe.length() - exeBackup.size(), exeBackup.size(), (const char*)exeBackup.begin()) == 0)) - { + else if ((programExe.length() >= exeBackup.size()) && + (programExe.compare( + programExe.length() - exeBackup.size(), exeBackup.size(), (const char*)exeBackup.begin()) == 0)) { enProgramExe = EXE_BACKUP; } // Check if restore - else if ((programExe.length() >= exeRestore.size()) && - (programExe.compare(programExe.length() - exeRestore.size(), exeRestore.size(), (const char*)exeRestore.begin()) == 0)) - { + else if ((programExe.length() >= exeRestore.size()) && + (programExe.compare( + programExe.length() - exeRestore.size(), exeRestore.size(), (const char*)exeRestore.begin()) == 0)) { enProgramExe = EXE_RESTORE; } // Check if restore else if ((programExe.length() >= exeFastRestoreTool.size()) && - (programExe.compare(programExe.length() - exeFastRestoreTool.size(), exeFastRestoreTool.size(), + (programExe.compare(programExe.length() - exeFastRestoreTool.size(), + exeFastRestoreTool.size(), (const char*)exeFastRestoreTool.begin()) == 0)) { enProgramExe = EXE_FASTRESTORE_TOOL; } // Check if db agent else if ((programExe.length() >= exeDatabaseAgent.size()) && - (programExe.compare(programExe.length() - exeDatabaseAgent.size(), exeDatabaseAgent.size(), + (programExe.compare(programExe.length() - exeDatabaseAgent.size(), + exeDatabaseAgent.size(), (const char*)exeDatabaseAgent.begin()) == 0)) { enProgramExe = EXE_DR_AGENT; } // Check if db backup else if ((programExe.length() >= exeDatabaseBackup.size()) && - (programExe.compare(programExe.length() - exeDatabaseBackup.size(), exeDatabaseBackup.size(), + (programExe.compare(programExe.length() - exeDatabaseBackup.size(), + exeDatabaseBackup.size(), (const char*)exeDatabaseBackup.begin()) == 0)) { enProgramExe = EXE_DB_BACKUP; } @@ -1314,15 +1370,14 @@ enumProgramExe getProgramType(std::string programExe) return enProgramExe; } -enumBackupType getBackupType(std::string backupType) -{ - enumBackupType enBackupType = BACKUP_UNDEFINED; +enumBackupType getBackupType(std::string backupType) { + enumBackupType enBackupType = BACKUP_UNDEFINED; // lowercase the string std::transform(backupType.begin(), backupType.end(), backupType.begin(), ::tolower); static std::map values; - if(values.empty()) { + if (values.empty()) { values["start"] = BACKUP_START; values["status"] = BACKUP_STATUS; values["abort"] = BACKUP_ABORT; @@ -1341,29 +1396,32 @@ enumBackupType getBackupType(std::string backupType) } auto i = values.find(backupType); - if(i != values.end()) + if (i != values.end()) enBackupType = i->second; return enBackupType; } enumRestoreType getRestoreType(std::string name) { - if(name == "start") return RESTORE_START; - if(name == "abort") return RESTORE_ABORT; - if(name == "status") return RESTORE_STATUS; - if(name == "wait") return RESTORE_WAIT; + if (name == "start") + return RESTORE_START; + if (name == "abort") + return RESTORE_ABORT; + if (name == "status") + return RESTORE_STATUS; + if (name == "wait") + return RESTORE_WAIT; return RESTORE_UNKNOWN; } -enumDBType getDBType(std::string dbType) -{ +enumDBType getDBType(std::string dbType) { enumDBType enBackupType = DB_UNDEFINED; // lowercase the string std::transform(dbType.begin(), dbType.end(), dbType.begin(), ::tolower); static std::map values; - if(values.empty()) { + if (values.empty()) { values["start"] = DB_START; values["status"] = DB_STATUS; values["switch"] = DB_SWITCH; @@ -1373,13 +1431,18 @@ enumDBType getDBType(std::string dbType) } auto i = values.find(dbType); - if(i != values.end()) + if (i != values.end()) enBackupType = i->second; return enBackupType; } -ACTOR Future getLayerStatus(Reference tr, std::string name, std::string id, enumProgramExe exe, Database dest, bool snapshot = false) { +ACTOR Future getLayerStatus(Reference tr, + std::string name, + std::string id, + enumProgramExe exe, + Database dest, + bool snapshot = false) { // This process will write a document that looks like this: // { backup : { $expires : {}, version: } // so that the value under 'backup' will eventually expire to null and thus be ignored by @@ -1388,9 +1451,9 @@ ACTOR Future getLayerStatus(Reference tr state Version readVer = wait(tr->getReadVersion()); - state json_spirit::mValue layersRootValue; // Will contain stuff that goes into the doc at the layers status root - JSONDoc layersRoot(layersRootValue); // Convenient mutator / accessor for the layers root - JSONDoc op = layersRoot.subDoc(name); // Operator object for the $expires operation + state json_spirit::mValue layersRootValue; // Will contain stuff that goes into the doc at the layers status root + JSONDoc layersRoot(layersRootValue); // Convenient mutator / accessor for the layers root + JSONDoc op = layersRoot.subDoc(name); // Operator object for the $expires operation // Create the $expires key which is where the rest of the status output will go state JSONDoc layerRoot = op.subDoc("$expires"); @@ -1403,15 +1466,15 @@ ACTOR Future getLayerStatus(Reference tr state JSONDoc o = layerRoot.subDoc("instances." + id); o.create("version") = FDB_VT_VERSION; - o.create("id") = id; + o.create("id") = id; o.create("last_updated") = now(); - o.create("memory_usage") = (int64_t)getMemoryUsage(); + o.create("memory_usage") = (int64_t)getMemoryUsage(); o.create("resident_size") = (int64_t)getResidentMemoryUsage(); o.create("main_thread_cpu_seconds") = getProcessorTimeThread(); - o.create("process_cpu_seconds") = getProcessorTimeProcess(); + o.create("process_cpu_seconds") = getProcessorTimeProcess(); o.create("configured_workers") = CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT; - if(exe == EXE_AGENT) { + if (exe == EXE_AGENT) { static BlobStoreEndpoint::Stats last_stats; static double last_ts = 0; BlobStoreEndpoint::Stats current_stats = BlobStoreEndpoint::s_stats; @@ -1419,14 +1482,14 @@ ACTOR Future getLayerStatus(Reference tr blobstats.create("total") = current_stats.getJSON(); BlobStoreEndpoint::Stats diff = current_stats - last_stats; json_spirit::mObject diffObj = diff.getJSON(); - if(last_ts > 0) + if (last_ts > 0) diffObj["bytes_per_second"] = double(current_stats.bytes_sent - last_stats.bytes_sent) / (now() - last_ts); blobstats.create("recent") = diffObj; last_stats = current_stats; last_ts = now(); JSONDoc totalBlobStats = layerRoot.subDoc("blob_recent_io"); - for(auto &p : diffObj) + for (auto& p : diffObj) totalBlobStats.create(p.first + ".$sum") = p.second; state FileBackupAgent fba; @@ -1454,19 +1517,22 @@ ACTOR Future getLayerStatus(Reference tr tagLastRestorableVersions.push_back(fba.getLastRestorable(tr, StringRef(tag->tagName), snapshot)); } - wait( waitForAll(tagLastRestorableVersions) && waitForAll(tagStates) && waitForAll(tagContainers) && waitForAll(tagRangeBytes) && waitForAll(tagLogBytes) && success(fBackupPaused)); + wait(waitForAll(tagLastRestorableVersions) && waitForAll(tagStates) && waitForAll(tagContainers) && + waitForAll(tagRangeBytes) && waitForAll(tagLogBytes) && success(fBackupPaused)); JSONDoc tagsRoot = layerRoot.subDoc("tags.$latest"); layerRoot.create("tags.timestamp") = now(); - layerRoot.create("total_workers.$sum") = fBackupPaused.get().present() ? 0 : CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT; + layerRoot.create("total_workers.$sum") = + fBackupPaused.get().present() ? 0 : CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT; layerRoot.create("paused.$latest") = fBackupPaused.get().present(); int j = 0; for (KeyBackedTag eachTag : backupTags) { Version last_restorable_version = tagLastRestorableVersions[j].get(); - double last_restorable_seconds_behind = ((double)readVer - last_restorable_version) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND; + double last_restorable_seconds_behind = + ((double)readVer - last_restorable_version) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND; BackupAgentBase::enumState status = (BackupAgentBase::enumState)tagStates[j].get(); - const char *statusText = fba.getStateText(status); + const char* statusText = fba.getStateText(status); // The object for this backup tag inside this instance's subdocument JSONDoc tagRoot = tagsRoot.subDoc(eachTag.tagName); @@ -1474,7 +1540,8 @@ ACTOR Future getLayerStatus(Reference tr tagRoot.create("current_status") = statusText; tagRoot.create("last_restorable_version") = tagLastRestorableVersions[j].get(); tagRoot.create("last_restorable_seconds_behind") = last_restorable_seconds_behind; - tagRoot.create("running_backup") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || status == BackupAgentBase::STATE_RUNNING); + tagRoot.create("running_backup") = + (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || status == BackupAgentBase::STATE_RUNNING); tagRoot.create("running_backup_is_restorable") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL); tagRoot.create("range_bytes_written") = tagRangeBytes[j].get(); tagRoot.create("mutation_log_bytes_written") = tagLogBytes[j].get(); @@ -1482,8 +1549,7 @@ ACTOR Future getLayerStatus(Reference tr j++; } - } - else if(exe == EXE_DR_AGENT) { + } else if (exe == EXE_DR_AGENT) { state DatabaseBackupAgent dba; state Reference tr2(new ReadYourWritesTransaction(dest)); tr2->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); @@ -1496,7 +1562,7 @@ ACTOR Future getLayerStatus(Reference tr state Future> fDRPaused = tr->get(dba.taskBucket->getPauseKey(), snapshot); state std::vector drTagUids; - for(int i = 0; i < tagNames.size(); i++) { + for (int i = 0; i < tagNames.size(); i++) { backupVersion.push_back(tr2->get(tagNames[i].value.withPrefix(applyMutationsBeginRange.begin), snapshot)); UID tagUID = BinaryReader::fromStringRef(tagNames[i].value, Unversioned()); drTagUids.push_back(tagUID); @@ -1505,7 +1571,8 @@ ACTOR Future getLayerStatus(Reference tr tagLogBytesDR.push_back(dba.getLogBytesWritten(tr2, tagUID, snapshot)); } - wait(waitForAll(backupStatus) && waitForAll(backupVersion) && waitForAll(tagRangeBytesDR) && waitForAll(tagLogBytesDR) && success(fDRPaused)); + wait(waitForAll(backupStatus) && waitForAll(backupVersion) && waitForAll(tagRangeBytesDR) && + waitForAll(tagLogBytesDR) && success(fDRPaused)); JSONDoc tagsRoot = layerRoot.subDoc("tags.$latest"); layerRoot.create("tags.timestamp") = now(); @@ -1518,14 +1585,17 @@ ACTOR Future getLayerStatus(Reference tr BackupAgentBase::enumState status = (BackupAgentBase::enumState)backupStatus[i].get(); JSONDoc tagRoot = tagsRoot.create(tagName); - tagRoot.create("running_backup") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || status == BackupAgentBase::STATE_RUNNING); + tagRoot.create("running_backup") = + (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || status == BackupAgentBase::STATE_RUNNING); tagRoot.create("running_backup_is_restorable") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL); tagRoot.create("range_bytes_written") = tagRangeBytesDR[i].get(); tagRoot.create("mutation_log_bytes_written") = tagLogBytesDR[i].get(); tagRoot.create("mutation_stream_id") = drTagUids[i].toString(); if (backupVersion[i].get().present()) { - double seconds_behind = ((double)readVer - BinaryReader::fromStringRef(backupVersion[i].get().get(), Unversioned())) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND; + double seconds_behind = ((double)readVer - BinaryReader::fromStringRef( + backupVersion[i].get().get(), Unversioned())) / + CLIENT_KNOBS->CORE_VERSIONSPERSECOND; tagRoot.create("seconds_behind") = seconds_behind; //TraceEvent("BackupMetrics").detail("SecondsBehind", seconds_behind); } @@ -1541,11 +1611,15 @@ ACTOR Future getLayerStatus(Reference tr // Check for unparseable or expired statuses and delete them. // First checks the first doc in the key range, and if it is valid, alive and not "me" then // returns. Otherwise, checks the rest of the range as well. -ACTOR Future cleanupStatus(Reference tr, std::string rootKey, std::string name, std::string id, int limit = 1) { +ACTOR Future cleanupStatus(Reference tr, + std::string rootKey, + std::string name, + std::string id, + int limit = 1) { state Standalone docs = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, true)); state bool readMore = false; state int i; - for(i = 0; i < docs.size(); ++i) { + for (i = 0; i < docs.size(); ++i) { json_spirit::mValue docValue; try { json_spirit::read_string(docs[i].value.toString(), docValue); @@ -1554,22 +1628,22 @@ ACTOR Future cleanupStatus(Reference tr, std::s JSONDoc::expires_reference_version = tr->getReadVersion().get(); // Evaluate the operators in the document, which will reduce to nothing if it is expired. doc.cleanOps(); - if(!doc.has(name + ".last_updated")) + if (!doc.has(name + ".last_updated")) throw Error(); // Alive and valid. // If limit == 1 and id is present then read more - if(limit == 1 && doc.has(name + ".instances." + id)) + if (limit == 1 && doc.has(name + ".instances." + id)) readMore = true; - } catch(Error &e) { + } catch (Error& e) { // If doc can't be parsed or isn't alive, delete it. TraceEvent(SevWarn, "RemovedDeadBackupLayerStatus").detail("Key", docs[i].key); tr->clear(docs[i].key); // If limit is 1 then read more. - if(limit == 1) + if (limit == 1) readMore = true; } - if(readMore) { + if (readMore) { limit = 10000; Standalone docs2 = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, true)); docs = std::move(docs2); @@ -1588,10 +1662,11 @@ ACTOR Future getLayerStatus(Database src, std::string root try { tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr.setOption(FDBTransactionOptions::LOCK_AWARE); - state Standalone kvPairs = wait(tr.getRange(KeyRangeRef(rootKey, strinc(rootKey)), CLIENT_KNOBS->ROW_LIMIT_UNLIMITED)); + state Standalone kvPairs = + wait(tr.getRange(KeyRangeRef(rootKey, strinc(rootKey)), CLIENT_KNOBS->ROW_LIMIT_UNLIMITED)); json_spirit::mObject statusDoc; JSONDoc modifier(statusDoc); - for(auto &kv : kvPairs) { + for (auto& kv : kvPairs) { json_spirit::mValue docValue; json_spirit::read_string(kv.value.toString(), docValue); modifier.absorb(docValue); @@ -1599,34 +1674,39 @@ ACTOR Future getLayerStatus(Database src, std::string root JSONDoc::expires_reference_version = (uint64_t)tr.getReadVersion().get(); modifier.cleanOps(); return statusDoc; - } - catch (Error& e) { + } catch (Error& e) { wait(tr.onError(e)); } } } -// Read layer status for this layer and get the total count of agent processes (instances) then adjust the poll delay based on that and BACKUP_AGGREGATE_POLL_RATE -ACTOR Future updateAgentPollRate(Database src, std::string rootKey, std::string name, double *pollDelay) { +// Read layer status for this layer and get the total count of agent processes (instances) then adjust the poll delay +// based on that and BACKUP_AGGREGATE_POLL_RATE +ACTOR Future updateAgentPollRate(Database src, std::string rootKey, std::string name, double* pollDelay) { loop { try { json_spirit::mObject status = wait(getLayerStatus(src, rootKey)); int64_t processes = 0; // If instances count is present and greater than 0 then update pollDelay - if(JSONDoc(status).tryGet(name + ".instances_running", processes) && processes > 0) { + if (JSONDoc(status).tryGet(name + ".instances_running", processes) && processes > 0) { // The aggregate poll rate is the target poll rate for all agent processes in the cluster - // The poll rate (polls/sec) for a single processes is aggregate poll rate / processes, and pollDelay is the inverse of that + // The poll rate (polls/sec) for a single processes is aggregate poll rate / processes, and pollDelay is + // the inverse of that *pollDelay = (double)processes / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE; } - } catch(Error &e) { + } catch (Error& e) { TraceEvent(SevWarn, "BackupAgentPollRateUpdateError").error(e); } wait(delay(CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE_UPDATE_INTERVAL)); } } -ACTOR Future statusUpdateActor(Database statusUpdateDest, std::string name, enumProgramExe exe, double *pollDelay, Database taskDest = Database(), - std::string id = nondeterministicRandom()->randomUniqueID().toString()) { +ACTOR Future statusUpdateActor(Database statusUpdateDest, + std::string name, + enumProgramExe exe, + double* pollDelay, + Database taskDest = Database(), + std::string id = nondeterministicRandom()->randomUniqueID().toString()) { state std::string metaKey = layerStatusMetaPrefixRange.begin.toString() + "json/" + name; state std::string rootKey = backupStatusPrefixRange.begin.toString() + name + "/json"; state std::string instanceKey = rootKey + "/" + "agent-" + id; @@ -1641,8 +1721,7 @@ ACTOR Future statusUpdateActor(Database statusUpdateDest, std::string name tr->set(metaKey, rootKey); wait(tr->commit()); break; - } - catch (Error& e) { + } catch (Error& e) { wait(tr->onError(e)); } } @@ -1651,7 +1730,7 @@ ACTOR Future statusUpdateActor(Database statusUpdateDest, std::string name loop { tr->reset(); try { - loop{ + loop { try { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); @@ -1661,19 +1740,20 @@ ACTOR Future statusUpdateActor(Database statusUpdateDest, std::string name tr->set(instanceKey, statusdoc); wait(tr->commit()); break; - } - catch (Error& e) { + } catch (Error& e) { wait(tr->onError(e)); } } - wait(delay(CLIENT_KNOBS->BACKUP_STATUS_DELAY * ( ( 1.0 - CLIENT_KNOBS->BACKUP_STATUS_JITTER ) + 2 * deterministicRandom()->random01() * CLIENT_KNOBS->BACKUP_STATUS_JITTER ))); + wait(delay(CLIENT_KNOBS->BACKUP_STATUS_DELAY * + ((1.0 - CLIENT_KNOBS->BACKUP_STATUS_JITTER) + + 2 * deterministicRandom()->random01() * CLIENT_KNOBS->BACKUP_STATUS_JITTER))); - // Now that status was written at least once by this process (and hopefully others), start the poll rate control updater if it wasn't started yet - if(!pollRateUpdater.isValid() && pollDelay != nullptr) + // Now that status was written at least once by this process (and hopefully others), start the poll rate + // control updater if it wasn't started yet + if (!pollRateUpdater.isValid() && pollDelay != nullptr) pollRateUpdater = updateAgentPollRate(statusUpdateDest, rootKey, name, pollDelay); - } - catch (Error& e) { + } catch (Error& e) { TraceEvent(SevWarnAlways, "UnableToWriteStatus").error(e); wait(delay(10.0)); } @@ -1692,15 +1772,14 @@ ACTOR Future runDBAgent(Database src, Database dest) { try { wait(backupAgent.run(dest, &pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT)); break; - } - catch (Error& e) { + } catch (Error& e) { if (e.code() == error_code_operation_cancelled) throw; TraceEvent(SevError, "DA_runAgent").error(e); fprintf(stderr, "ERROR: DR agent encountered fatal error `%s'\n", e.what()); - wait( delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY) ); + wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY)); } } @@ -1717,24 +1796,25 @@ ACTOR Future runAgent(Database db) { try { wait(backupAgent.run(db, &pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT)); break; - } - catch (Error& e) { + } catch (Error& e) { if (e.code() == error_code_operation_cancelled) throw; TraceEvent(SevError, "BA_runAgent").error(e); fprintf(stderr, "ERROR: backup agent encountered fatal error `%s'\n", e.what()); - wait( delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY) ); + wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY)); } } return Void(); } -ACTOR Future submitDBBackup(Database src, Database dest, Standalone> backupRanges, std::string tagName) { - try - { +ACTOR Future submitDBBackup(Database src, + Database dest, + Standalone> backupRanges, + std::string tagName) { + try { state DatabaseBackupAgent backupAgent(src); // Backup everything, if no ranges were specified @@ -1742,36 +1822,34 @@ ACTOR Future submitDBBackup(Database src, Database dest, Standalone submitDBBackup(Database src, Database dest, Standalone submitBackup(Database db, std::string url, int snapshotIntervalSeconds, - Standalone> backupRanges, std::string tagName, bool dryRun, - bool waitForCompletion, bool stopWhenDone, bool usePartitionedLog) { +ACTOR Future submitBackup(Database db, + std::string url, + int snapshotIntervalSeconds, + Standalone> backupRanges, + std::string tagName, + bool dryRun, + bool waitForCompletion, + bool stopWhenDone, + bool usePartitionedLog) { try { state FileBackupAgent backupAgent; @@ -1807,7 +1891,8 @@ ACTOR Future submitBackup(Database db, std::string url, int snapshotInterv } if (waitForCompletion) { - printf("Submitted and now waiting for the backup on tag `%s' to complete. (DRY RUN)\n", printable(StringRef(tagName)).c_str()); + printf("Submitted and now waiting for the backup on tag `%s' to complete. (DRY RUN)\n", + printable(StringRef(tagName)).c_str()); } else { @@ -1815,54 +1900,56 @@ ACTOR Future submitBackup(Database db, std::string url, int snapshotInterv bool agentRunning = wait(backupAgent.checkActive(db)); if (!agentRunning) { - printf("The backup on tag `%s' was successfully submitted but no backup agents are responding. (DRY RUN)\n", printable(StringRef(tagName)).c_str()); + printf("The backup on tag `%s' was successfully submitted but no backup agents are responding. " + "(DRY RUN)\n", + printable(StringRef(tagName)).c_str()); // Throw an error that will not display any additional information throw actor_cancelled(); - } - else { - printf("The backup on tag `%s' was successfully submitted. (DRY RUN)\n", printable(StringRef(tagName)).c_str()); + } else { + printf("The backup on tag `%s' was successfully submitted. (DRY RUN)\n", + printable(StringRef(tagName)).c_str()); } } } else { - wait(backupAgent.submitBackup(db, KeyRef(url), snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone, - usePartitionedLog)); + wait(backupAgent.submitBackup( + db, KeyRef(url), snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone, usePartitionedLog)); // Wait for the backup to complete, if requested if (waitForCompletion) { - printf("Submitted and now waiting for the backup on tag `%s' to complete.\n", printable(StringRef(tagName)).c_str()); + printf("Submitted and now waiting for the backup on tag `%s' to complete.\n", + printable(StringRef(tagName)).c_str()); wait(success(backupAgent.waitBackup(db, tagName))); - } - else { + } else { // Check if a backup agent is running bool agentRunning = wait(backupAgent.checkActive(db)); if (!agentRunning) { - printf("The backup on tag `%s' was successfully submitted but no backup agents are responding.\n", printable(StringRef(tagName)).c_str()); + printf("The backup on tag `%s' was successfully submitted but no backup agents are responding.\n", + printable(StringRef(tagName)).c_str()); // Throw an error that will not display any additional information throw actor_cancelled(); - } - else { - printf("The backup on tag `%s' was successfully submitted.\n", printable(StringRef(tagName)).c_str()); + } else { + printf("The backup on tag `%s' was successfully submitted.\n", + printable(StringRef(tagName)).c_str()); } } } } catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + if (e.code() == error_code_actor_cancelled) throw; - switch (e.code()) - { - case error_code_backup_error: - fprintf(stderr, "ERROR: An error was encountered during submission\n"); + switch (e.code()) { + case error_code_backup_error: + fprintf(stderr, "ERROR: An error was encountered during submission\n"); break; - case error_code_backup_duplicate: - fprintf(stderr, "ERROR: A backup is already running on tag `%s'\n", printable(StringRef(tagName)).c_str()); + case error_code_backup_duplicate: + fprintf(stderr, "ERROR: A backup is already running on tag `%s'\n", printable(StringRef(tagName)).c_str()); break; - default: - fprintf(stderr, "ERROR: %s\n", e.what()); + default: + fprintf(stderr, "ERROR: %s\n", e.what()); break; } @@ -1872,9 +1959,12 @@ ACTOR Future submitBackup(Database db, std::string url, int snapshotInterv return Void(); } -ACTOR Future switchDBBackup(Database src, Database dest, Standalone> backupRanges, std::string tagName, bool forceAction) { - try - { +ACTOR Future switchDBBackup(Database src, + Database dest, + Standalone> backupRanges, + std::string tagName, + bool forceAction) { + try { state DatabaseBackupAgent backupAgent(src); // Backup everything, if no ranges were specified @@ -1882,24 +1972,22 @@ ACTOR Future switchDBBackup(Database src, Database dest, Standalone switchDBBackup(Database src, Database dest, Standalone statusDBBackup(Database src, Database dest, std::string tagName, int errorLimit) { - try - { + try { state DatabaseBackupAgent backupAgent(src); - std::string statusText = wait(backupAgent.getStatus(dest, errorLimit, StringRef(tagName))); + std::string statusText = wait(backupAgent.getStatus(dest, errorLimit, StringRef(tagName))); printf("%s\n", statusText.c_str()); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -1928,15 +2014,14 @@ ACTOR Future statusDBBackup(Database src, Database dest, std::string tagNa } ACTOR Future statusBackup(Database db, std::string tagName, bool showErrors, bool json) { - try - { + try { state FileBackupAgent backupAgent; - std::string statusText = wait(json ? backupAgent.getStatusJSON(db, tagName) : backupAgent.getStatus(db, showErrors, tagName)); + std::string statusText = + wait(json ? backupAgent.getStatusJSON(db, tagName) : backupAgent.getStatus(db, showErrors, tagName)); printf("%s\n", statusText.c_str()); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -1946,28 +2031,25 @@ ACTOR Future statusBackup(Database db, std::string tagName, bool showError } ACTOR Future abortDBBackup(Database src, Database dest, std::string tagName, bool partial, bool dstOnly) { - try - { + try { state DatabaseBackupAgent backupAgent(src); wait(backupAgent.abortBackup(dest, Key(tagName), partial, false, dstOnly)); wait(backupAgent.unlockBackup(dest, Key(tagName))); printf("The DR on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str()); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; - switch (e.code()) - { - case error_code_backup_error: - fprintf(stderr, "ERROR: An error was encountered during submission\n"); + switch (e.code()) { + case error_code_backup_error: + fprintf(stderr, "ERROR: An error was encountered during submission\n"); break; - case error_code_backup_unneeded: - fprintf(stderr, "ERROR: A DR was not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); + case error_code_backup_unneeded: + fprintf(stderr, "ERROR: A DR was not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); break; - default: - fprintf(stderr, "ERROR: %s\n", e.what()); + default: + fprintf(stderr, "ERROR: %s\n", e.what()); break; } throw; @@ -1977,27 +2059,24 @@ ACTOR Future abortDBBackup(Database src, Database dest, std::string tagNam } ACTOR Future abortBackup(Database db, std::string tagName) { - try - { + try { state FileBackupAgent backupAgent; wait(backupAgent.abortBackup(db, tagName)); printf("The backup on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str()); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; - switch (e.code()) - { - case error_code_backup_error: - fprintf(stderr, "ERROR: An error was encountered during submission\n"); + switch (e.code()) { + case error_code_backup_error: + fprintf(stderr, "ERROR: An error was encountered during submission\n"); break; - case error_code_backup_unneeded: - fprintf(stderr, "ERROR: A backup was not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); + case error_code_backup_unneeded: + fprintf(stderr, "ERROR: A backup was not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); break; - default: - fprintf(stderr, "ERROR: %s\n", e.what()); + default: + fprintf(stderr, "ERROR: %s\n", e.what()); break; } throw; @@ -2007,12 +2086,10 @@ ACTOR Future abortBackup(Database db, std::string tagName) { } ACTOR Future cleanupMutations(Database db, bool deleteData) { - try - { + try { wait(cleanupBackup(db, deleteData)); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -2022,17 +2099,16 @@ ACTOR Future cleanupMutations(Database db, bool deleteData) { } ACTOR Future waitBackup(Database db, std::string tagName, bool stopWhenDone) { - try - { + try { state FileBackupAgent backupAgent; int status = wait(backupAgent.waitBackup(db, tagName, stopWhenDone)); - printf("The backup on tag `%s' %s.\n", printable(StringRef(tagName)).c_str(), - BackupAgentBase::getStateText((BackupAgentBase::enumState) status)); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + printf("The backup on tag `%s' %s.\n", + printable(StringRef(tagName)).c_str(), + BackupAgentBase::getStateText((BackupAgentBase::enumState)status)); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -2042,38 +2118,37 @@ ACTOR Future waitBackup(Database db, std::string tagName, bool stopWhenDon } ACTOR Future discontinueBackup(Database db, std::string tagName, bool waitForCompletion) { - try - { + try { state FileBackupAgent backupAgent; wait(backupAgent.discontinueBackup(db, StringRef(tagName))); // Wait for the backup to complete, if requested if (waitForCompletion) { - printf("Discontinued and now waiting for the backup on tag `%s' to complete.\n", printable(StringRef(tagName)).c_str()); + printf("Discontinued and now waiting for the backup on tag `%s' to complete.\n", + printable(StringRef(tagName)).c_str()); wait(success(backupAgent.waitBackup(db, tagName))); - } - else { + } else { printf("The backup on tag `%s' was successfully discontinued.\n", printable(StringRef(tagName)).c_str()); } - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; - switch (e.code()) - { - case error_code_backup_error: - fprintf(stderr, "ERROR: An encounter was error during submission\n"); + switch (e.code()) { + case error_code_backup_error: + fprintf(stderr, "ERROR: An encounter was error during submission\n"); break; - case error_code_backup_unneeded: - fprintf(stderr, "ERROR: A backup in not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); + case error_code_backup_unneeded: + fprintf(stderr, "ERROR: A backup in not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); break; - case error_code_backup_duplicate: - fprintf(stderr, "ERROR: The backup on tag `%s' is already discontinued\n", printable(StringRef(tagName)).c_str()); + case error_code_backup_duplicate: + fprintf(stderr, + "ERROR: The backup on tag `%s' is already discontinued\n", + printable(StringRef(tagName)).c_str()); break; - default: - fprintf(stderr, "ERROR: %s\n", e.what()); + default: + fprintf(stderr, "ERROR: %s\n", e.what()); break; } throw; @@ -2087,9 +2162,8 @@ ACTOR Future changeBackupResumed(Database db, bool pause) { FileBackupAgent backupAgent; wait(backupAgent.changePause(db, pause)); printf("All backup agents have been %s.\n", pause ? "paused" : "resumed"); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -2103,9 +2177,8 @@ ACTOR Future changeDBBackupResumed(Database src, Database dest, bool pause state DatabaseBackupAgent backupAgent(src); wait(backupAgent.taskBucket->changePause(dest, pause)); printf("All DR agents have been %s.\n", pause ? "paused" : "resumed"); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -2114,7 +2187,7 @@ ACTOR Future changeDBBackupResumed(Database src, Database dest, bool pause return Void(); } -Reference openBackupContainer(const char *name, std::string destinationContainer) { +Reference openBackupContainer(const char* name, std::string destinationContainer) { // Error, if no dest container was specified if (destinationContainer.empty()) { fprintf(stderr, "ERROR: No backup destination was specified.\n"); @@ -2125,10 +2198,9 @@ Reference openBackupContainer(const char *name, std::string de Reference c; try { c = IBackupContainer::openContainer(destinationContainer); - } - catch (Error& e) { + } catch (Error& e) { std::string msg = format("ERROR: '%s' on URL '%s'", e.what(), destinationContainer.c_str()); - if(e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { + if (e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { msg += format(": %s", IBackupContainer::lastOpenError.c_str()); } fprintf(stderr, "%s\n", msg.c_str()); @@ -2139,12 +2211,23 @@ Reference openBackupContainer(const char *name, std::string de return c; } -ACTOR Future runRestore(Database db, std::string originalClusterFile, std::string tagName, std::string container, Standalone> ranges, Version targetVersion, std::string targetTimestamp, bool performRestore, bool verbose, bool waitForDone, std::string addPrefix, std::string removePrefix) { - if(ranges.empty()) { +ACTOR Future runRestore(Database db, + std::string originalClusterFile, + std::string tagName, + std::string container, + Standalone> ranges, + Version targetVersion, + std::string targetTimestamp, + bool performRestore, + bool verbose, + bool waitForDone, + std::string addPrefix, + std::string removePrefix) { + if (ranges.empty()) { ranges.push_back_deep(ranges.arena(), normalKeys); } - if(targetVersion != invalidVersion && !targetTimestamp.empty()) { + if (targetVersion != invalidVersion && !targetTimestamp.empty()) { fprintf(stderr, "Restore target version and target timestamp cannot both be specified\n"); throw restore_error(); } @@ -2152,14 +2235,17 @@ ACTOR Future runRestore(Database db, std::string originalClusterFile, std: state Optional origDb; // Resolve targetTimestamp if given - if(!targetTimestamp.empty()) { - if(originalClusterFile.empty()) { - fprintf(stderr, "An original cluster file must be given in order to resolve restore target timestamp '%s'\n", targetTimestamp.c_str()); + if (!targetTimestamp.empty()) { + if (originalClusterFile.empty()) { + fprintf(stderr, + "An original cluster file must be given in order to resolve restore target timestamp '%s'\n", + targetTimestamp.c_str()); throw restore_error(); } - if(!fileExists(originalClusterFile)) { - fprintf(stderr, "Original source database cluster file '%s' does not exist.\n", originalClusterFile.c_str()); + if (!fileExists(originalClusterFile)) { + fprintf( + stderr, "Original source database cluster file '%s' does not exist.\n", originalClusterFile.c_str()); throw restore_error(); } @@ -2175,45 +2261,55 @@ ACTOR Future runRestore(Database db, std::string originalClusterFile, std: state Reference bc = openBackupContainer(exeRestore.toString().c_str(), container); // If targetVersion is unset then use the maximum restorable version from the backup description - if(targetVersion == invalidVersion) { - if(verbose) - printf("No restore target version given, will use maximum restorable version from backup description.\n"); + if (targetVersion == invalidVersion) { + if (verbose) + printf( + "No restore target version given, will use maximum restorable version from backup description.\n"); BackupDescription desc = wait(bc->describeBackup()); - if(!desc.maxRestorableVersion.present()) { + if (!desc.maxRestorableVersion.present()) { fprintf(stderr, "The specified backup is not restorable to any version.\n"); throw restore_error(); } targetVersion = desc.maxRestorableVersion.get(); - if(verbose) + if (verbose) printf("Using target restore version %" PRId64 "\n", targetVersion); } if (performRestore) { - Version restoredVersion = wait(backupAgent.restore(db, origDb, KeyRef(tagName), KeyRef(container), ranges, waitForDone, targetVersion, verbose, KeyRef(addPrefix), KeyRef(removePrefix))); + Version restoredVersion = wait(backupAgent.restore(db, + origDb, + KeyRef(tagName), + KeyRef(container), + ranges, + waitForDone, + targetVersion, + verbose, + KeyRef(addPrefix), + KeyRef(removePrefix))); - if(waitForDone && verbose) { + if (waitForDone && verbose) { // If restore is now complete then report version restored printf("Restored to version %" PRId64 "\n", restoredVersion); } - } - else { + } else { state Optional rset = wait(bc->getRestoreSet(targetVersion)); - if(!rset.present()) { - fprintf(stderr, "Insufficient data to restore to version %" PRId64 ". Describe backup for more information.\n", targetVersion); + if (!rset.present()) { + fprintf(stderr, + "Insufficient data to restore to version %" PRId64 ". Describe backup for more information.\n", + targetVersion); throw restore_invalid_version(); } printf("Backup can be used to restore to version %" PRId64 "\n", targetVersion); } - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -2223,9 +2319,14 @@ ACTOR Future runRestore(Database db, std::string originalClusterFile, std: } // Fast restore agent that kicks off the restore: send restore requests to restore workers. -ACTOR Future runFastRestoreTool(Database db, std::string tagName, std::string container, - Standalone> ranges, Version dbVersion, - bool performRestore, bool verbose, bool waitForDone) { +ACTOR Future runFastRestoreTool(Database db, + std::string tagName, + std::string container, + Standalone> ranges, + Version dbVersion, + bool performRestore, + bool verbose, + bool waitForDone) { try { state FileBackupAgent backupAgent; state Version restoreVersion = invalidVersion; @@ -2238,7 +2339,8 @@ ACTOR Future runFastRestoreTool(Database db, std::string tagName, std::str ranges.push_back(ranges.arena(), normalKeys); } - printf("[INFO] runFastRestoreTool: restore_ranges:%d first range:%s\n", ranges.size(), + printf("[INFO] runFastRestoreTool: restore_ranges:%d first range:%s\n", + ranges.size(), ranges.front().toString().c_str()); TraceEvent ev("FastRestoreTool"); ev.detail("RestoreRanges", ranges.size()); @@ -2262,8 +2364,15 @@ ACTOR Future runFastRestoreTool(Database db, std::string tagName, std::str TraceEvent("FastRestoreTool") .detail("SubmitRestoreRequests", ranges.size()) .detail("RestoreUID", randomUID); - wait(backupAgent.submitParallelRestore(db, KeyRef(tagName), ranges, KeyRef(container), dbVersion, true, - randomUID, LiteralStringRef(""), LiteralStringRef(""))); + wait(backupAgent.submitParallelRestore(db, + KeyRef(tagName), + ranges, + KeyRef(container), + dbVersion, + true, + randomUID, + LiteralStringRef(""), + LiteralStringRef(""))); // TODO: Support addPrefix and removePrefix if (waitForDone) { // Wait for parallel restore to finish and unlock DB after that @@ -2313,7 +2422,8 @@ ACTOR Future runFastRestoreTool(Database db, std::string tagName, std::str printf("Restored to version %" PRId64 "%s\n", restoreVersion, (performRestore) ? "" : " (DRY RUN)"); } } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) throw; + if (e.code() == error_code_actor_cancelled) + throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; } @@ -2321,22 +2431,25 @@ ACTOR Future runFastRestoreTool(Database db, std::string tagName, std::str return Void(); } -ACTOR Future dumpBackupData(const char *name, std::string destinationContainer, Version beginVersion, Version endVersion) { +ACTOR Future dumpBackupData(const char* name, + std::string destinationContainer, + Version beginVersion, + Version endVersion) { state Reference c = openBackupContainer(name, destinationContainer); - if(beginVersion < 0 || endVersion < 0) { + if (beginVersion < 0 || endVersion < 0) { BackupDescription desc = wait(c->describeBackup()); - if(!desc.maxLogEnd.present()) { + if (!desc.maxLogEnd.present()) { fprintf(stderr, "ERROR: Backup must have log data in order to use relative begin/end versions.\n"); throw backup_invalid_info(); } - if(beginVersion < 0) { + if (beginVersion < 0) { beginVersion += desc.maxLogEnd.get(); } - if(endVersion < 0) { + if (endVersion < 0) { endVersion += desc.maxLogEnd.get(); } } @@ -2348,21 +2461,29 @@ ACTOR Future dumpBackupData(const char *name, std::string destinationConta return Void(); } -ACTOR Future expireBackupData(const char *name, std::string destinationContainer, Version endVersion, std::string endDatetime, Database db, bool force, Version restorableAfterVersion, std::string restorableAfterDatetime) { +ACTOR Future expireBackupData(const char* name, + std::string destinationContainer, + Version endVersion, + std::string endDatetime, + Database db, + bool force, + Version restorableAfterVersion, + std::string restorableAfterDatetime) { if (!endDatetime.empty()) { - Version v = wait( timeKeeperVersionFromDatetime(endDatetime, db) ); + Version v = wait(timeKeeperVersionFromDatetime(endDatetime, db)); endVersion = v; } if (!restorableAfterDatetime.empty()) { - Version v = wait( timeKeeperVersionFromDatetime(restorableAfterDatetime, db) ); + Version v = wait(timeKeeperVersionFromDatetime(restorableAfterDatetime, db)); restorableAfterVersion = v; } if (endVersion == invalidVersion) { fprintf(stderr, "ERROR: No version or date/time is specified.\n"); printHelpTeaser(name); - throw backup_error();; + throw backup_error(); + ; } try { @@ -2376,32 +2497,34 @@ ACTOR Future expireBackupData(const char *name, std::string destinationCon choose { when(wait(delay(5))) { std::string p = progress.toString(); - if(p != lastProgress) { + if (p != lastProgress) { int spaces = lastProgress.size() - p.size(); - printf("\r%s%s", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "") ); + printf("\r%s%s", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "")); lastProgress = p; } } - when(wait(expire)) { - break; - } + when(wait(expire)) { break; } } } std::string p = progress.toString(); int spaces = lastProgress.size() - p.size(); - printf("\r%s%s\n", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "") ); + printf("\r%s%s\n", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "")); - if(endVersion < 0) - printf("All data before %" PRId64 " versions (%" PRId64 " days) prior to latest backup log has been deleted.\n", -endVersion, -endVersion / ((int64_t)24 * 3600 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); + if (endVersion < 0) + printf("All data before %" PRId64 " versions (%" PRId64 + " days) prior to latest backup log has been deleted.\n", + -endVersion, + -endVersion / ((int64_t)24 * 3600 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); else printf("All data before version %" PRId64 " has been deleted.\n", endVersion); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; - if(e.code() == error_code_backup_cannot_expire) - fprintf(stderr, "ERROR: Requested expiration would be unsafe. Backup would not meet minimum restorability. Use --force to delete data anyway.\n"); + if (e.code() == error_code_backup_cannot_expire) + fprintf(stderr, + "ERROR: Requested expiration would be unsafe. Backup would not meet minimum restorability. Use " + "--force to delete data anyway.\n"); else fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -2410,7 +2533,7 @@ ACTOR Future expireBackupData(const char *name, std::string destinationCon return Void(); } -ACTOR Future deleteBackupContainer(const char *name, std::string destinationContainer) { +ACTOR Future deleteBackupContainer(const char* name, std::string destinationContainer) { try { state Reference c = openBackupContainer(name, destinationContainer); state int numDeleted = 0; @@ -2421,11 +2544,9 @@ ACTOR Future deleteBackupContainer(const char *name, std::string destinati loop { choose { - when ( wait(done) ) { - break; - } - when ( wait(delay(5)) ) { - if(numDeleted != lastUpdate) { + when(wait(done)) { break; } + when(wait(delay(5))) { + if (numDeleted != lastUpdate) { printf("\r%d...", numDeleted); lastUpdate = numDeleted; } @@ -2434,9 +2555,8 @@ ACTOR Future deleteBackupContainer(const char *name, std::string destinati } printf("\r%d objects deleted\n", numDeleted); printf("The entire container has been deleted.\n"); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -2445,16 +2565,19 @@ ACTOR Future deleteBackupContainer(const char *name, std::string destinati return Void(); } -ACTOR Future describeBackup(const char *name, std::string destinationContainer, bool deep, Optional cx, bool json) { +ACTOR Future describeBackup(const char* name, + std::string destinationContainer, + bool deep, + Optional cx, + bool json) { try { Reference c = openBackupContainer(name, destinationContainer); state BackupDescription desc = wait(c->describeBackup(deep)); - if(cx.present()) + if (cx.present()) wait(desc.resolveVersionTimes(cx.get())); printf("%s\n", (json ? desc.toJSON() : desc.toString()).c_str()); - } - catch (Error& e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; fprintf(stderr, "ERROR: %s\n", e.what()); throw; @@ -2472,9 +2595,13 @@ static void reportBackupQueryError(UID operationId, JsonBuilderObject& result, s // If restoreVersion is invalidVersion or latestVersion, use the maximum or minimum restorable version respectively for // selected key ranges. If restoreTimestamp is specified, any specified restoreVersion will be overriden to the version // resolved to that timestamp. -ACTOR Future queryBackup(const char* name, std::string destinationContainer, - Standalone> keyRangesFilter, Version restoreVersion, - std::string originalClusterFile, std::string restoreTimestamp, bool verbose) { +ACTOR Future queryBackup(const char* name, + std::string destinationContainer, + Standalone> keyRangesFilter, + Version restoreVersion, + std::string originalClusterFile, + std::string restoreTimestamp, + bool verbose) { state UID operationId = deterministicRandom()->randomUniqueID(); state JsonBuilderObject result; state std::string errorMessage; @@ -2493,14 +2620,16 @@ ACTOR Future queryBackup(const char* name, std::string destinationContaine if (!restoreTimestamp.empty()) { if (originalClusterFile.empty()) { reportBackupQueryError( - operationId, result, + operationId, + result, format("an original cluster file must be given in order to resolve restore target timestamp '%s'", restoreTimestamp.c_str())); return Void(); } if (!fileExists(originalClusterFile)) { - reportBackupQueryError(operationId, result, + reportBackupQueryError(operationId, + result, format("The specified original source database cluster file '%s' does not exist\n", originalClusterFile.c_str())); return Void(); @@ -2524,13 +2653,15 @@ ACTOR Future queryBackup(const char* name, std::string destinationContaine restoreVersion = desc.contiguousLogEnd.get(); } else { reportBackupQueryError( - operationId, result, + operationId, + result, errorMessage = format("the backup for the specified key ranges is not restorable to any version")); } } if (restoreVersion < 0 && restoreVersion != latestVersion) { - reportBackupQueryError(operationId, result, + reportBackupQueryError(operationId, + result, errorMessage = format("the specified restorable version %ld is not valid", restoreVersion)); return Void(); @@ -2598,10 +2729,9 @@ ACTOR Future listBackup(std::string baseUrl) { for (std::string container : containers) { printf("%s\n", container.c_str()); } - } - catch (Error& e) { + } catch (Error& e) { std::string msg = format("ERROR: %s", e.what()); - if(e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { + if (e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { msg += format(": %s", IBackupContainer::lastOpenError.c_str()); } fprintf(stderr, "%s\n", msg.c_str()); @@ -2622,7 +2752,7 @@ struct BackupModifyOptions { }; ACTOR Future modifyBackup(Database db, std::string tagName, BackupModifyOptions options) { - if(!options.hasChanges()) { + if (!options.hasChanges()) { fprintf(stderr, "No changes were specified, nothing to do!\n"); throw backup_error(); } @@ -2630,14 +2760,17 @@ ACTOR Future modifyBackup(Database db, std::string tagName, BackupModifyOp state KeyBackedTag tag = makeBackupTag(tagName); state Reference bc; - if(options.destURL.present()) { + if (options.destURL.present()) { bc = openBackupContainer(exeBackup.toString().c_str(), options.destURL.get()); try { wait(timeoutError(bc->create(), 30)); - } catch(Error &e) { - if(e.code() == error_code_actor_cancelled) + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) throw; - fprintf(stderr, "ERROR: Could not create backup container at '%s': %s\n", options.destURL.get().c_str(), e.what()); + fprintf(stderr, + "ERROR: Could not create backup container at '%s': %s\n", + options.destURL.get().c_str(), + e.what()); throw backup_error(); } } @@ -2650,45 +2783,50 @@ ACTOR Future modifyBackup(Database db, std::string tagName, BackupModifyOp state Optional uidFlag = wait(tag.get(db)); - if(!uidFlag.present()) { + if (!uidFlag.present()) { fprintf(stderr, "No backup exists on tag '%s'\n", tagName.c_str()); throw backup_error(); } - if(uidFlag.get().second) { + if (uidFlag.get().second) { fprintf(stderr, "Cannot modify aborted backup on tag '%s'\n", tagName.c_str()); throw backup_error(); } state BackupConfig config(uidFlag.get().first); EBackupState s = wait(config.stateEnum().getOrThrow(tr, false, backup_invalid_info())); - if(!FileBackupAgent::isRunnable(s)) { + if (!FileBackupAgent::isRunnable(s)) { fprintf(stderr, "Backup on tag '%s' is not runnable.\n", tagName.c_str()); throw backup_error(); } - if(options.verifyUID.present() && options.verifyUID.get() != uidFlag.get().first.toString()) { - fprintf(stderr, "UID verification failed, backup on tag '%s' is '%s' but '%s' was specified.\n", tagName.c_str(), uidFlag.get().first.toString().c_str(), options.verifyUID.get().c_str()); + if (options.verifyUID.present() && options.verifyUID.get() != uidFlag.get().first.toString()) { + fprintf(stderr, + "UID verification failed, backup on tag '%s' is '%s' but '%s' was specified.\n", + tagName.c_str(), + uidFlag.get().first.toString().c_str(), + options.verifyUID.get().c_str()); throw backup_error(); } - if(options.snapshotIntervalSeconds.present()) { + if (options.snapshotIntervalSeconds.present()) { config.snapshotIntervalSeconds().set(tr, options.snapshotIntervalSeconds.get()); } - if(options.activeSnapshotIntervalSeconds.present()) { + if (options.activeSnapshotIntervalSeconds.present()) { Version begin = wait(config.snapshotBeginVersion().getOrThrow(tr, false, backup_error())); - config.snapshotTargetEndVersion().set(tr, begin + ((int64_t)options.activeSnapshotIntervalSeconds.get() * CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); + config.snapshotTargetEndVersion().set(tr, + begin + ((int64_t)options.activeSnapshotIntervalSeconds.get() * + CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); } - if(options.destURL.present()) { + if (options.destURL.present()) { config.backupContainer().set(tr, bc); } wait(tr->commit()); break; - } - catch (Error& e) { + } catch (Error& e) { wait(tr->onError(e)); } } @@ -2696,8 +2834,7 @@ ACTOR Future modifyBackup(Database db, std::string tagName, BackupModifyOp return Void(); } -static std::vector> parseLine(std::string &line, bool& err, bool& partial) -{ +static std::vector> parseLine(std::string& line, bool& err, bool& partial) { err = false; partial = false; @@ -2718,8 +2855,7 @@ static std::vector> parseLine(std::string &line, bool& er buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); ret.push_back(std::move(buf)); offset = i = line.find_first_not_of(' ', i + 1); - } - else + } else i++; break; case '"': @@ -2730,12 +2866,10 @@ static std::vector> parseLine(std::string &line, bool& er break; case ' ': if (!quoted) { - buf.push_back(StringRef((uint8_t *)(line.data() + offset), - i - offset)); + buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); offset = i = line.find_first_not_of(' ', i); forcetoken = false; - } - else + } else i++; break; case '\\': @@ -2758,7 +2892,7 @@ static std::vector> parseLine(std::string &line, bool& er ret.push_back(std::move(buf)); return ret; } - char *pEnd; + char* pEnd; save = line[i + 4]; line[i + 4] = 0; ent = char(strtoul(line.data() + i + 2, &pEnd, 16)); @@ -2792,48 +2926,47 @@ static std::vector> parseLine(std::string &line, bool& er return ret; } -static void addKeyRange(std::string optionValue, Standalone>& keyRanges) -{ - bool err = false, partial = false; - int tokenArray = 0, tokenIndex = 0; +static void addKeyRange(std::string optionValue, Standalone>& keyRanges) { + bool err = false, partial = false; + int tokenArray = 0, tokenIndex = 0; auto parsed = parseLine(optionValue, err, partial); - for (auto tokens : parsed) - { + for (auto tokens : parsed) { tokenArray++; tokenIndex = 0; /* for (auto token : tokens) { - tokenIndex++; + tokenIndex++; - printf("%4d token #%2d: %s\n", tokenArray, tokenIndex, printable(token).c_str()); + printf("%4d token #%2d: %s\n", tokenArray, tokenIndex, printable(token).c_str()); } */ // Process the keys // [end] - switch (tokens.size()) - { + switch (tokens.size()) { // empty case 0: break; // single key range case 1: - keyRanges.push_back_deep(keyRanges.arena(), KeyRangeRef(tokens.at(0), strinc(tokens.at(0)))); + keyRanges.push_back_deep(keyRanges.arena(), KeyRangeRef(tokens.at(0), strinc(tokens.at(0)))); break; // full key range case 2: try { keyRanges.push_back_deep(keyRanges.arena(), KeyRangeRef(tokens.at(0), tokens.at(1))); - } - catch (Error& e) { - fprintf(stderr, "ERROR: Invalid key range `%s %s' reported error %s\n", - tokens.at(0).toString().c_str(), tokens.at(1).toString().c_str(), e.what()); + } catch (Error& e) { + fprintf(stderr, + "ERROR: Invalid key range `%s %s' reported error %s\n", + tokens.at(0).toString().c_str(), + tokens.at(1).toString().c_str(), + e.what()); throw invalid_option_value(); } break; @@ -2849,12 +2982,12 @@ static void addKeyRange(std::string optionValue, StandaloneNext()) { lastError = args->LastError(); - switch (lastError) - { + switch (lastError) { case SO_SUCCESS: break; @@ -3186,299 +3305,292 @@ int main(int argc, char* argv[]) { int optId = args->OptionId(); switch (optId) { - case OPT_HELP: - printUsage(programExe, false); - return FDB_EXIT_SUCCESS; - break; - case OPT_DEVHELP: - printUsage(programExe, true); - return FDB_EXIT_SUCCESS; - break; - case OPT_VERSION: - printVersion(); - return FDB_EXIT_SUCCESS; - break; - case OPT_NOBUFSTDOUT: - setvbuf(stdout, NULL, _IONBF, 0); - setvbuf(stderr, NULL, _IONBF, 0); - break; - case OPT_BUFSTDOUTERR: - setvbuf(stdout, NULL, _IOFBF, BUFSIZ); - setvbuf(stderr, NULL, _IOFBF, BUFSIZ); - break; - case OPT_QUIET: - quietDisplay = true; - break; - case OPT_DRYRUN: - dryRun = true; - break; - case OPT_DELETE_DATA: - deleteData = true; - break; - case OPT_MIN_CLEANUP_SECONDS: - knobs.push_back( std::make_pair( "min_cleanup_seconds", args->OptionArg() ) ); - break; - case OPT_FORCE: - forceAction = true; - break; - case OPT_TRACE: - trace = true; - break; - case OPT_TRACE_DIR: - trace = true; - traceDir = args->OptionArg(); - break; - case OPT_TRACE_FORMAT: - if (!validateTraceFormat(args->OptionArg())) { - fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args->OptionArg()); - } - traceFormat = args->OptionArg(); - break; - case OPT_TRACE_LOG_GROUP: - traceLogGroup = args->OptionArg(); - break; - case OPT_LOCALITY: { - std::string syn = args->OptionSyntax(); - if (!StringRef(syn).startsWith(LiteralStringRef("--locality_"))) { - fprintf(stderr, "ERROR: unable to parse locality key '%s'\n", syn.c_str()); - return FDB_EXIT_ERROR; - } - syn = syn.substr(11); - std::transform(syn.begin(), syn.end(), syn.begin(), ::tolower); - localities.set(Standalone(syn), Standalone(std::string(args->OptionArg()))); - break; - } - case OPT_EXPIRE_BEFORE_DATETIME: - expireDatetime = args->OptionArg(); - break; - case OPT_EXPIRE_RESTORABLE_AFTER_DATETIME: - expireRestorableAfterDatetime = args->OptionArg(); - break; - case OPT_EXPIRE_BEFORE_VERSION: - case OPT_EXPIRE_RESTORABLE_AFTER_VERSION: - case OPT_EXPIRE_MIN_RESTORABLE_DAYS: - case OPT_EXPIRE_DELETE_BEFORE_DAYS: - { - const char* a = args->OptionArg(); - long long ver = 0; - if (!sscanf(a, "%lld", &ver)) { - fprintf(stderr, "ERROR: Could not parse expiration version `%s'\n", a); - printHelpTeaser(argv[0]); - return FDB_EXIT_ERROR; - } + case OPT_HELP: + printUsage(programExe, false); + return FDB_EXIT_SUCCESS; + break; + case OPT_DEVHELP: + printUsage(programExe, true); + return FDB_EXIT_SUCCESS; + break; + case OPT_VERSION: + printVersion(); + return FDB_EXIT_SUCCESS; + break; + case OPT_NOBUFSTDOUT: + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + break; + case OPT_BUFSTDOUTERR: + setvbuf(stdout, NULL, _IOFBF, BUFSIZ); + setvbuf(stderr, NULL, _IOFBF, BUFSIZ); + break; + case OPT_QUIET: + quietDisplay = true; + break; + case OPT_DRYRUN: + dryRun = true; + break; + case OPT_DELETE_DATA: + deleteData = true; + break; + case OPT_MIN_CLEANUP_SECONDS: + knobs.push_back(std::make_pair("min_cleanup_seconds", args->OptionArg())); + break; + case OPT_FORCE: + forceAction = true; + break; + case OPT_TRACE: + trace = true; + break; + case OPT_TRACE_DIR: + trace = true; + traceDir = args->OptionArg(); + break; + case OPT_TRACE_FORMAT: + if (!validateTraceFormat(args->OptionArg())) { + fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args->OptionArg()); + } + traceFormat = args->OptionArg(); + break; + case OPT_TRACE_LOG_GROUP: + traceLogGroup = args->OptionArg(); + break; + case OPT_LOCALITY: { + std::string syn = args->OptionSyntax(); + if (!StringRef(syn).startsWith(LiteralStringRef("--locality_"))) { + fprintf(stderr, "ERROR: unable to parse locality key '%s'\n", syn.c_str()); + return FDB_EXIT_ERROR; + } + syn = syn.substr(11); + std::transform(syn.begin(), syn.end(), syn.begin(), ::tolower); + localities.set(Standalone(syn), Standalone(std::string(args->OptionArg()))); + break; + } + case OPT_EXPIRE_BEFORE_DATETIME: + expireDatetime = args->OptionArg(); + break; + case OPT_EXPIRE_RESTORABLE_AFTER_DATETIME: + expireRestorableAfterDatetime = args->OptionArg(); + break; + case OPT_EXPIRE_BEFORE_VERSION: + case OPT_EXPIRE_RESTORABLE_AFTER_VERSION: + case OPT_EXPIRE_MIN_RESTORABLE_DAYS: + case OPT_EXPIRE_DELETE_BEFORE_DAYS: { + const char* a = args->OptionArg(); + long long ver = 0; + if (!sscanf(a, "%lld", &ver)) { + fprintf(stderr, "ERROR: Could not parse expiration version `%s'\n", a); + printHelpTeaser(argv[0]); + return FDB_EXIT_ERROR; + } - // Interpret the value as days worth of versions relative to now (negative) - if(optId == OPT_EXPIRE_MIN_RESTORABLE_DAYS || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS) { - ver = -ver * 24 * 60 * 60 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND; - } + // Interpret the value as days worth of versions relative to now (negative) + if (optId == OPT_EXPIRE_MIN_RESTORABLE_DAYS || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS) { + ver = -ver * 24 * 60 * 60 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND; + } - if(optId == OPT_EXPIRE_BEFORE_VERSION || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS) - expireVersion = ver; - else - expireRestorableAfterVersion = ver; - break; + if (optId == OPT_EXPIRE_BEFORE_VERSION || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS) + expireVersion = ver; + else + expireRestorableAfterVersion = ver; + break; + } + case OPT_RESTORE_TIMESTAMP: + restoreTimestamp = args->OptionArg(); + break; + case OPT_BASEURL: + baseUrl = args->OptionArg(); + break; + case OPT_RESTORE_CLUSTERFILE_DEST: + restoreClusterFileDest = args->OptionArg(); + break; + case OPT_RESTORE_CLUSTERFILE_ORIG: + restoreClusterFileOrig = args->OptionArg(); + break; + case OPT_CLUSTERFILE: + clusterFile = args->OptionArg(); + break; + case OPT_DEST_CLUSTER: + clusterFile = args->OptionArg(); + break; + case OPT_SOURCE_CLUSTER: + sourceClusterFile = args->OptionArg(); + break; + case OPT_CLEANUP: + partial = false; + break; + case OPT_DSTONLY: + dstOnly = true; + break; + case OPT_KNOB: { + std::string syn = args->OptionSyntax(); + if (!StringRef(syn).startsWith(LiteralStringRef("--knob_"))) { + fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str()); + return FDB_EXIT_ERROR; } - case OPT_RESTORE_TIMESTAMP: - restoreTimestamp = args->OptionArg(); - break; - case OPT_BASEURL: - baseUrl = args->OptionArg(); - break; - case OPT_RESTORE_CLUSTERFILE_DEST: - restoreClusterFileDest = args->OptionArg(); - break; - case OPT_RESTORE_CLUSTERFILE_ORIG: - restoreClusterFileOrig = args->OptionArg(); - break; - case OPT_CLUSTERFILE: - clusterFile = args->OptionArg(); - break; - case OPT_DEST_CLUSTER: - clusterFile = args->OptionArg(); - break; - case OPT_SOURCE_CLUSTER: - sourceClusterFile = args->OptionArg(); - break; - case OPT_CLEANUP: - partial = false; - break; - case OPT_DSTONLY: - dstOnly = true; - break; - case OPT_KNOB: { - std::string syn = args->OptionSyntax(); - if (!StringRef(syn).startsWith(LiteralStringRef("--knob_"))) { - fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str()); - return FDB_EXIT_ERROR; - } - syn = syn.substr(7); - knobs.push_back( std::make_pair( syn, args->OptionArg() ) ); - break; - } - case OPT_BACKUPKEYS: - try { - addKeyRange(args->OptionArg(), backupKeys); - } - catch (Error &) { - printHelpTeaser(argv[0]); - return FDB_EXIT_ERROR; - } - break; - case OPT_BACKUPKEYS_FILTER: - try { - addKeyRange(args->OptionArg(), backupKeysFilter); - } - catch (Error &) { - printHelpTeaser(argv[0]); - return FDB_EXIT_ERROR; - } - break; - case OPT_DESTCONTAINER: - destinationContainer = args->OptionArg(); - // If the url starts with '/' then prepend "file://" for backwards compatibility - if(StringRef(destinationContainer).startsWith(LiteralStringRef("/"))) - destinationContainer = std::string("file://") + destinationContainer; - modifyOptions.destURL = destinationContainer; - break; - case OPT_SNAPSHOTINTERVAL: - case OPT_MOD_ACTIVE_INTERVAL: - { - const char* a = args->OptionArg(); - int seconds; - if (!sscanf(a, "%d", &seconds)) { - fprintf(stderr, "ERROR: Could not parse snapshot interval `%s'\n", a); - printHelpTeaser(argv[0]); - return FDB_EXIT_ERROR; - } - if(optId == OPT_SNAPSHOTINTERVAL) { - snapshotIntervalSeconds = seconds; - modifyOptions.snapshotIntervalSeconds = seconds; - } - else if(optId == OPT_MOD_ACTIVE_INTERVAL) { - modifyOptions.activeSnapshotIntervalSeconds = seconds; - } - break; + syn = syn.substr(7); + knobs.push_back(std::make_pair(syn, args->OptionArg())); + break; + } + case OPT_BACKUPKEYS: + try { + addKeyRange(args->OptionArg(), backupKeys); + } catch (Error&) { + printHelpTeaser(argv[0]); + return FDB_EXIT_ERROR; } - case OPT_MOD_VERIFY_UID: - modifyOptions.verifyUID = args->OptionArg(); - break; - case OPT_WAITFORDONE: - waitForDone = true; - break; - case OPT_NOSTOPWHENDONE: - stopWhenDone = false; - break; - case OPT_USE_PARTITIONED_LOG: - usePartitionedLog = true; - break; - case OPT_RESTORECONTAINER: - restoreContainer = args->OptionArg(); - // If the url starts with '/' then prepend "file://" for backwards compatibility - if(StringRef(restoreContainer).startsWith(LiteralStringRef("/"))) - restoreContainer = std::string("file://") + restoreContainer; - break; - case OPT_DESCRIBE_DEEP: - describeDeep = true; - break; - case OPT_DESCRIBE_TIMESTAMPS: - describeTimestamps = true; - break; - case OPT_PREFIX_ADD: - addPrefix = args->OptionArg(); - break; - case OPT_PREFIX_REMOVE: - removePrefix = args->OptionArg(); - break; - case OPT_ERRORLIMIT: { - const char* a = args->OptionArg(); - if (!sscanf(a, "%d", &maxErrors)) { - fprintf(stderr, "ERROR: Could not parse max number of errors `%s'\n", a); - printHelpTeaser(argv[0]); - return FDB_EXIT_ERROR; - } - break; + break; + case OPT_BACKUPKEYS_FILTER: + try { + addKeyRange(args->OptionArg(), backupKeysFilter); + } catch (Error&) { + printHelpTeaser(argv[0]); + return FDB_EXIT_ERROR; } - case OPT_RESTORE_VERSION: { - const char* a = args->OptionArg(); - long long ver = 0; - if (!sscanf(a, "%lld", &ver)) { - fprintf(stderr, "ERROR: Could not parse database version `%s'\n", a); - printHelpTeaser(argv[0]); - return FDB_EXIT_ERROR; - } - restoreVersion = ver; - break; + break; + case OPT_DESTCONTAINER: + destinationContainer = args->OptionArg(); + // If the url starts with '/' then prepend "file://" for backwards compatibility + if (StringRef(destinationContainer).startsWith(LiteralStringRef("/"))) + destinationContainer = std::string("file://") + destinationContainer; + modifyOptions.destURL = destinationContainer; + break; + case OPT_SNAPSHOTINTERVAL: + case OPT_MOD_ACTIVE_INTERVAL: { + const char* a = args->OptionArg(); + int seconds; + if (!sscanf(a, "%d", &seconds)) { + fprintf(stderr, "ERROR: Could not parse snapshot interval `%s'\n", a); + printHelpTeaser(argv[0]); + return FDB_EXIT_ERROR; } - #ifdef _WIN32 - case OPT_PARENTPID: { - auto pid_str = args->OptionArg(); - int parent_pid = atoi(pid_str); - auto pHandle = OpenProcess( SYNCHRONIZE, FALSE, parent_pid ); - if( !pHandle ) { - TraceEvent("ParentProcessOpenError").GetLastError(); - fprintf(stderr, "Could not open parent process at pid %d (error %d)", parent_pid, GetLastError()); - throw platform_error(); - } - startThread(&parentWatcher, pHandle); - break; + if (optId == OPT_SNAPSHOTINTERVAL) { + snapshotIntervalSeconds = seconds; + modifyOptions.snapshotIntervalSeconds = seconds; + } else if (optId == OPT_MOD_ACTIVE_INTERVAL) { + modifyOptions.activeSnapshotIntervalSeconds = seconds; } - #endif - case OPT_TAGNAME: - tagName = args->OptionArg(); - tagProvided = true; - break; - case OPT_CRASHONERROR: - g_crashOnError = true; - break; - case OPT_MEMLIMIT: - ti = parse_with_suffix(args->OptionArg(), "MiB"); - if (!ti.present()) { - fprintf(stderr, "ERROR: Could not parse memory limit from `%s'\n", args->OptionArg()); - printHelpTeaser(argv[0]); - flushAndExit(FDB_EXIT_ERROR); - } - memLimit = ti.get(); - break; - case OPT_BLOB_CREDENTIALS: - blobCredentials.push_back(args->OptionArg()); - break; -#ifndef TLS_DISABLED - case TLSConfig::OPT_TLS_PLUGIN: - args->OptionArg(); - break; - case TLSConfig::OPT_TLS_CERTIFICATES: - tlsCertPath = args->OptionArg(); - break; - case TLSConfig::OPT_TLS_PASSWORD: - tlsPassword = args->OptionArg(); - break; - case TLSConfig::OPT_TLS_CA_FILE: - tlsCAPath = args->OptionArg(); - break; - case TLSConfig::OPT_TLS_KEY: - tlsKeyPath = args->OptionArg(); - break; - case TLSConfig::OPT_TLS_VERIFY_PEERS: - tlsVerifyPeers = args->OptionArg(); - break; + break; + } + case OPT_MOD_VERIFY_UID: + modifyOptions.verifyUID = args->OptionArg(); + break; + case OPT_WAITFORDONE: + waitForDone = true; + break; + case OPT_NOSTOPWHENDONE: + stopWhenDone = false; + break; + case OPT_USE_PARTITIONED_LOG: + usePartitionedLog = true; + break; + case OPT_RESTORECONTAINER: + restoreContainer = args->OptionArg(); + // If the url starts with '/' then prepend "file://" for backwards compatibility + if (StringRef(restoreContainer).startsWith(LiteralStringRef("/"))) + restoreContainer = std::string("file://") + restoreContainer; + break; + case OPT_DESCRIBE_DEEP: + describeDeep = true; + break; + case OPT_DESCRIBE_TIMESTAMPS: + describeTimestamps = true; + break; + case OPT_PREFIX_ADD: + addPrefix = args->OptionArg(); + break; + case OPT_PREFIX_REMOVE: + removePrefix = args->OptionArg(); + break; + case OPT_ERRORLIMIT: { + const char* a = args->OptionArg(); + if (!sscanf(a, "%d", &maxErrors)) { + fprintf(stderr, "ERROR: Could not parse max number of errors `%s'\n", a); + printHelpTeaser(argv[0]); + return FDB_EXIT_ERROR; + } + break; + } + case OPT_RESTORE_VERSION: { + const char* a = args->OptionArg(); + long long ver = 0; + if (!sscanf(a, "%lld", &ver)) { + fprintf(stderr, "ERROR: Could not parse database version `%s'\n", a); + printHelpTeaser(argv[0]); + return FDB_EXIT_ERROR; + } + restoreVersion = ver; + break; + } +#ifdef _WIN32 + case OPT_PARENTPID: { + auto pid_str = args->OptionArg(); + int parent_pid = atoi(pid_str); + auto pHandle = OpenProcess(SYNCHRONIZE, FALSE, parent_pid); + if (!pHandle) { + TraceEvent("ParentProcessOpenError").GetLastError(); + fprintf(stderr, "Could not open parent process at pid %d (error %d)", parent_pid, GetLastError()); + throw platform_error(); + } + startThread(&parentWatcher, pHandle); + break; + } #endif - case OPT_DUMP_BEGIN: - dumpBegin = parseVersion(args->OptionArg()); - break; - case OPT_DUMP_END: - dumpEnd = parseVersion(args->OptionArg()); - break; - case OPT_JSON: - jsonOutput = true; - break; + case OPT_TAGNAME: + tagName = args->OptionArg(); + tagProvided = true; + break; + case OPT_CRASHONERROR: + g_crashOnError = true; + break; + case OPT_MEMLIMIT: + ti = parse_with_suffix(args->OptionArg(), "MiB"); + if (!ti.present()) { + fprintf(stderr, "ERROR: Could not parse memory limit from `%s'\n", args->OptionArg()); + printHelpTeaser(argv[0]); + flushAndExit(FDB_EXIT_ERROR); + } + memLimit = ti.get(); + break; + case OPT_BLOB_CREDENTIALS: + blobCredentials.push_back(args->OptionArg()); + break; +#ifndef TLS_DISABLED + case TLSConfig::OPT_TLS_PLUGIN: + args->OptionArg(); + break; + case TLSConfig::OPT_TLS_CERTIFICATES: + tlsCertPath = args->OptionArg(); + break; + case TLSConfig::OPT_TLS_PASSWORD: + tlsPassword = args->OptionArg(); + break; + case TLSConfig::OPT_TLS_CA_FILE: + tlsCAPath = args->OptionArg(); + break; + case TLSConfig::OPT_TLS_KEY: + tlsKeyPath = args->OptionArg(); + break; + case TLSConfig::OPT_TLS_VERIFY_PEERS: + tlsVerifyPeers = args->OptionArg(); + break; +#endif + case OPT_DUMP_BEGIN: + dumpBegin = parseVersion(args->OptionArg()); + break; + case OPT_DUMP_END: + dumpEnd = parseVersion(args->OptionArg()); + break; + case OPT_JSON: + jsonOutput = true; + break; } } // Process the extra arguments - for (int argLoop = 0; argLoop < args->FileCount(); argLoop++) - { - switch (programExe) - { + for (int argLoop = 0; argLoop < args->FileCount(); argLoop++) { + switch (programExe) { case EXE_AGENT: fprintf(stderr, "ERROR: Backup Agent does not support argument value `%s'\n", args->File(argLoop)); printHelpTeaser(argv[0]); @@ -3497,8 +3609,7 @@ int main(int argc, char* argv[]) { else { try { addKeyRange(args->File(argLoop), backupKeys); - } - catch (Error& ) { + } catch (Error&) { printHelpTeaser(argv[0]); return FDB_EXIT_ERROR; } @@ -3512,8 +3623,8 @@ int main(int argc, char* argv[]) { break; case EXE_FASTRESTORE_TOOL: - fprintf(stderr, "ERROR: FDB Fast Restore Tool does not support argument value `%s'\n", - args->File(argLoop)); + fprintf( + stderr, "ERROR: FDB Fast Restore Tool does not support argument value `%s'\n", args->File(argLoop)); printHelpTeaser(argv[0]); return FDB_EXIT_ERROR; break; @@ -3535,8 +3646,7 @@ int main(int argc, char* argv[]) { else { try { addKeyRange(args->File(argLoop), backupKeys); - } - catch (Error& ) { + } catch (Error&) { printHelpTeaser(argv[0]); return FDB_EXIT_ERROR; } @@ -3550,8 +3660,7 @@ int main(int argc, char* argv[]) { } // Delete the simple option object, if defined - if (args) - { + if (args) { delete args; args = NULL; } @@ -3564,22 +3673,27 @@ int main(int argc, char* argv[]) { ClientKnobs* clientKnobs = new ClientKnobs; CLIENT_KNOBS = clientKnobs; - for(auto k=knobs.begin(); k!=knobs.end(); ++k) { + for (auto k = knobs.begin(); k != knobs.end(); ++k) { try { - if (!flowKnobs->setKnob( k->first, k->second ) && - !clientKnobs->setKnob( k->first, k->second )) - { + if (!flowKnobs->setKnob(k->first, k->second) && !clientKnobs->setKnob(k->first, k->second)) { fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", k->first.c_str()); TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(k->first)); } } catch (Error& e) { if (e.code() == error_code_invalid_option_value) { - fprintf(stderr, "WARNING: Invalid value '%s' for knob option '%s'\n", k->second.c_str(), k->first.c_str()); - TraceEvent(SevWarnAlways, "InvalidKnobValue").detail("Knob", printable(k->first)).detail("Value", printable(k->second)); - } - else { + fprintf(stderr, + "WARNING: Invalid value '%s' for knob option '%s'\n", + k->second.c_str(), + k->first.c_str()); + TraceEvent(SevWarnAlways, "InvalidKnobValue") + .detail("Knob", printable(k->first)) + .detail("Value", printable(k->second)); + } else { fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", k->first.c_str(), e.what()); - TraceEvent(SevError, "FailedToSetKnob").detail("Knob", printable(k->first)).detail("Value", printable(k->second)).error(e); + TraceEvent(SevError, "FailedToSetKnob") + .detail("Knob", printable(k->first)) + .detail("Value", printable(k->second)) + .error(e); throw; } } @@ -3590,7 +3704,7 @@ int main(int argc, char* argv[]) { clientKnobs->initialize(true); if (trace) { - if(!traceLogGroup.empty()) + if (!traceLogGroup.empty()) setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(traceLogGroup)); if (traceDir.empty()) @@ -3609,8 +3723,7 @@ int main(int argc, char* argv[]) { if (tlsCertPath.size()) { try { setNetworkOption(FDBNetworkOptions::TLS_CERT_PATH, tlsCertPath); - } - catch (Error& e) { + } catch (Error& e) { fprintf(stderr, "ERROR: cannot set TLS certificate path to `%s' (%s)\n", tlsCertPath.c_str(), e.what()); return 1; } @@ -3619,8 +3732,7 @@ int main(int argc, char* argv[]) { if (tlsCAPath.size()) { try { setNetworkOption(FDBNetworkOptions::TLS_CA_PATH, tlsCAPath); - } - catch (Error& e) { + } catch (Error& e) { fprintf(stderr, "ERROR: cannot set TLS CA path to `%s' (%s)\n", tlsCAPath.c_str(), e.what()); return 1; } @@ -3631,8 +3743,7 @@ int main(int argc, char* argv[]) { setNetworkOption(FDBNetworkOptions::TLS_PASSWORD, tlsPassword); setNetworkOption(FDBNetworkOptions::TLS_KEY_PATH, tlsKeyPath); - } - catch (Error& e) { + } catch (Error& e) { fprintf(stderr, "ERROR: cannot set TLS key path to `%s' (%s)\n", tlsKeyPath.c_str(), e.what()); return 1; } @@ -3640,27 +3751,26 @@ int main(int argc, char* argv[]) { if (tlsVerifyPeers.size()) { try { setNetworkOption(FDBNetworkOptions::TLS_VERIFY_PEERS, tlsVerifyPeers); - } - catch (Error& e) { - fprintf(stderr, "ERROR: cannot set TLS peer verification to `%s' (%s)\n", tlsVerifyPeers.c_str(), e.what()); + } catch (Error& e) { + fprintf( + stderr, "ERROR: cannot set TLS peer verification to `%s' (%s)\n", tlsVerifyPeers.c_str(), e.what()); return 1; } } Error::init(); - std::set_new_handler( &platform::outOfMemory ); - setMemoryQuota( memLimit ); + std::set_new_handler(&platform::outOfMemory); + setMemoryQuota(memLimit); int total = 0; - for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) + for (auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) total += i->second; if (total) printf("%d errors:\n", total); - for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) + for (auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) if (i->second > 0) printf(" %d: %d %s\n", i->second, i->first, Error::fromCode(i->first).what()); - Reference ccf; Database db; Reference sourceCcf; @@ -3673,42 +3783,43 @@ int main(int argc, char* argv[]) { try { setupNetwork(0, true); - } - catch (Error& e) { + } catch (Error& e) { fprintf(stderr, "ERROR: %s\n", e.what()); return FDB_EXIT_ERROR; } TraceEvent("ProgramStart") - .setMaxEventLength(12000) - .detail("SourceVersion", getSourceVersion()) - .detail("Version", FDB_VT_VERSION ) - .detail("PackageName", FDB_VT_PACKAGE_NAME) - .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL)) - .setMaxFieldLength(10000) - .detail("CommandLine", commandLine) - .setMaxFieldLength(0) - .detail("MemoryLimit", memLimit) - .trackLatest("ProgramStart"); + .setMaxEventLength(12000) + .detail("SourceVersion", getSourceVersion()) + .detail("Version", FDB_VT_VERSION) + .detail("PackageName", FDB_VT_PACKAGE_NAME) + .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL)) + .setMaxFieldLength(10000) + .detail("CommandLine", commandLine) + .setMaxFieldLength(0) + .detail("MemoryLimit", memLimit) + .trackLatest("ProgramStart"); - // Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents are logged. This thread will eventually run the network, so call it now. + // Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents + // are logged. This thread will eventually run the network, so call it now. TraceEvent::setNetworkThread(); // Add blob credentials files from the environment to the list collected from the command line. - const char *blobCredsFromENV = getenv("FDB_BLOB_CREDENTIALS"); - if(blobCredsFromENV != nullptr) { + const char* blobCredsFromENV = getenv("FDB_BLOB_CREDENTIALS"); + if (blobCredsFromENV != nullptr) { StringRef t((uint8_t*)blobCredsFromENV, strlen(blobCredsFromENV)); do { StringRef file = t.eat(":"); - if(file.size() != 0) - blobCredentials.push_back(file.toString()); - } while(t.size() != 0); + if (file.size() != 0) + blobCredentials.push_back(file.toString()); + } while (t.size() != 0); } // Update the global blob credential files list - std::vector *pFiles = (std::vector *)g_network->global(INetwork::enBlobCredentialFiles); - if(pFiles != nullptr) { - for(auto &f : blobCredentials) { + std::vector* pFiles = + (std::vector*)g_network->global(INetwork::enBlobCredentialFiles); + if (pFiles != nullptr) { + for (auto& f : blobCredentials) { pFiles->push_back(f); } } @@ -3717,7 +3828,7 @@ int main(int argc, char* argv[]) { // For most modes, initCluster() will open a trace file, but some fdbbackup operations do not require // a cluster so they should use this instead. auto initTraceFile = [&]() { - if(trace) + if (trace) openTraceFile(NetworkAddress(), traceRollSize, traceMaxLogsSize, traceDir, "trace", traceLogGroup); }; @@ -3725,17 +3836,15 @@ int main(int argc, char* argv[]) { auto resolvedClusterFile = ClusterConnectionFile::lookupClusterFileName(clusterFile); try { ccf = Reference(new ClusterConnectionFile(resolvedClusterFile.first)); - } - catch (Error& e) { - if(!quiet) + } catch (Error& e) { + if (!quiet) fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedClusterFile, e).c_str()); return false; } try { db = Database::createDatabase(ccf, -1, true, localities); - } - catch (Error& e) { + } catch (Error& e) { fprintf(stderr, "ERROR: %s\n", e.what()); fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", ccf->getFilename().c_str()); return false; @@ -3744,137 +3853,156 @@ int main(int argc, char* argv[]) { return true; }; - if(sourceClusterFile.size()) { + if (sourceClusterFile.size()) { auto resolvedSourceClusterFile = ClusterConnectionFile::lookupClusterFileName(sourceClusterFile); try { - sourceCcf = Reference(new ClusterConnectionFile(resolvedSourceClusterFile.first)); - } - catch (Error& e) { + sourceCcf = + Reference(new ClusterConnectionFile(resolvedSourceClusterFile.first)); + } catch (Error& e) { fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedSourceClusterFile, e).c_str()); return FDB_EXIT_ERROR; } try { sourceDb = Database::createDatabase(sourceCcf, -1, true, localities); - } - catch (Error& e) { + } catch (Error& e) { fprintf(stderr, "ERROR: %s\n", e.what()); fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", sourceCcf->getFilename().c_str()); return FDB_EXIT_ERROR; } } - switch (programExe) - { + switch (programExe) { case EXE_AGENT: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; f = stopAfter(runAgent(db)); break; case EXE_BACKUP: - switch (backupType) - { - case BACKUP_START: - { - if(!initCluster()) + switch (backupType) { + case BACKUP_START: { + if (!initCluster()) return FDB_EXIT_ERROR; // Test out the backup url to make sure it parses. Doesn't test to make sure it's actually writeable. openBackupContainer(argv[0], destinationContainer); - f = stopAfter(submitBackup(db, destinationContainer, snapshotIntervalSeconds, backupKeys, tagName, - dryRun, waitForDone, stopWhenDone, usePartitionedLog)); + f = stopAfter(submitBackup(db, + destinationContainer, + snapshotIntervalSeconds, + backupKeys, + tagName, + dryRun, + waitForDone, + stopWhenDone, + usePartitionedLog)); break; } - case BACKUP_MODIFY: - { - if(!initCluster()) + case BACKUP_MODIFY: { + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( modifyBackup(db, tagName, modifyOptions) ); + f = stopAfter(modifyBackup(db, tagName, modifyOptions)); break; } case BACKUP_STATUS: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( statusBackup(db, tagName, true, jsonOutput) ); + f = stopAfter(statusBackup(db, tagName, true, jsonOutput)); break; case BACKUP_ABORT: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( abortBackup(db, tagName) ); + f = stopAfter(abortBackup(db, tagName)); break; case BACKUP_CLEANUP: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( cleanupMutations(db, deleteData) ); + f = stopAfter(cleanupMutations(db, deleteData)); break; case BACKUP_WAIT: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( waitBackup(db, tagName, stopWhenDone) ); + f = stopAfter(waitBackup(db, tagName, stopWhenDone)); break; case BACKUP_DISCONTINUE: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( discontinueBackup(db, tagName, waitForDone) ); + f = stopAfter(discontinueBackup(db, tagName, waitForDone)); break; case BACKUP_PAUSE: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( changeBackupResumed(db, true) ); + f = stopAfter(changeBackupResumed(db, true)); break; case BACKUP_RESUME: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( changeBackupResumed(db, false) ); + f = stopAfter(changeBackupResumed(db, false)); break; case BACKUP_EXPIRE: initTraceFile(); // Must have a usable cluster if either expire DateTime options were used - if(!expireDatetime.empty() || !expireRestorableAfterDatetime.empty()) { - if(!initCluster()) + if (!expireDatetime.empty() || !expireRestorableAfterDatetime.empty()) { + if (!initCluster()) return FDB_EXIT_ERROR; } - f = stopAfter( expireBackupData(argv[0], destinationContainer, expireVersion, expireDatetime, db, forceAction, expireRestorableAfterVersion, expireRestorableAfterDatetime) ); + f = stopAfter(expireBackupData(argv[0], + destinationContainer, + expireVersion, + expireDatetime, + db, + forceAction, + expireRestorableAfterVersion, + expireRestorableAfterDatetime)); break; case BACKUP_DELETE: initTraceFile(); - f = stopAfter( deleteBackupContainer(argv[0], destinationContainer) ); + f = stopAfter(deleteBackupContainer(argv[0], destinationContainer)); break; case BACKUP_DESCRIBE: initTraceFile(); // If timestamp lookups are desired, require a cluster file - if(describeTimestamps && !initCluster()) + if (describeTimestamps && !initCluster()) return FDB_EXIT_ERROR; - // Only pass database optionDatabase Describe will lookup version timestamps if a cluster file was given, but quietly skip them if not. - f = stopAfter( describeBackup(argv[0], destinationContainer, describeDeep, describeTimestamps ? Optional(db) : Optional(), jsonOutput) ); + // Only pass database optionDatabase Describe will lookup version timestamps if a cluster file was + // given, but quietly skip them if not. + f = stopAfter(describeBackup(argv[0], + destinationContainer, + describeDeep, + describeTimestamps ? Optional(db) : Optional(), + jsonOutput)); break; case BACKUP_LIST: initTraceFile(); - f = stopAfter( listBackup(baseUrl) ); + f = stopAfter(listBackup(baseUrl)); break; case BACKUP_QUERY: initTraceFile(); - f = stopAfter(queryBackup(argv[0], destinationContainer, backupKeysFilter, restoreVersion, - restoreClusterFileOrig, restoreTimestamp, !quietDisplay)); + f = stopAfter(queryBackup(argv[0], + destinationContainer, + backupKeysFilter, + restoreVersion, + restoreClusterFileOrig, + restoreTimestamp, + !quietDisplay)); break; case BACKUP_DUMP: initTraceFile(); - f = stopAfter( dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd) ); + f = stopAfter(dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd)); break; case BACKUP_UNDEFINED: @@ -3887,60 +4015,76 @@ int main(int argc, char* argv[]) { break; case EXE_RESTORE: - if(dryRun) { - if(restoreType != RESTORE_START) { + if (dryRun) { + if (restoreType != RESTORE_START) { fprintf(stderr, "Restore dry run only works for 'start' command\n"); return FDB_EXIT_ERROR; } // Must explicitly call trace file options handling if not calling Database::createDatabase() initTraceFile(); - } - else { - if(restoreClusterFileDest.empty()) { + } else { + if (restoreClusterFileDest.empty()) { fprintf(stderr, "Restore destination cluster file must be specified explicitly.\n"); return FDB_EXIT_ERROR; } - if(!fileExists(restoreClusterFileDest)) { - fprintf(stderr, "Restore destination cluster file '%s' does not exist.\n", restoreClusterFileDest.c_str()); + if (!fileExists(restoreClusterFileDest)) { + fprintf(stderr, + "Restore destination cluster file '%s' does not exist.\n", + restoreClusterFileDest.c_str()); return FDB_EXIT_ERROR; } try { db = Database::createDatabase(restoreClusterFileDest, Database::API_VERSION_LATEST); - } catch(Error &e) { - fprintf(stderr, "Restore destination cluster file '%s' invalid: %s\n", restoreClusterFileDest.c_str(), e.what()); + } catch (Error& e) { + fprintf(stderr, + "Restore destination cluster file '%s' invalid: %s\n", + restoreClusterFileDest.c_str(), + e.what()); return FDB_EXIT_ERROR; } } - switch(restoreType) { - case RESTORE_START: - f = stopAfter( runRestore(db, restoreClusterFileOrig, tagName, restoreContainer, backupKeys, restoreVersion, restoreTimestamp, !dryRun, !quietDisplay, waitForDone, addPrefix, removePrefix) ); - break; - case RESTORE_WAIT: - f = stopAfter( success(ba.waitRestore(db, KeyRef(tagName), true)) ); - break; - case RESTORE_ABORT: - f = stopAfter( - map(ba.abortRestore(db, KeyRef(tagName)), [tagName](FileBackupAgent::ERestoreState s) -> Void { - printf("RESTORE_ABORT Tag: %s State: %s\n", tagName.c_str(), - FileBackupAgent::restoreStateText(s).toString().c_str()); - return Void(); - })); - break; - case RESTORE_STATUS: - // If no tag is specifically provided then print all tag status, don't just use "default" - if(tagProvided) - tag = tagName; - f = stopAfter( map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void { - printf("%s\n", s.c_str()); - return Void(); - }) ); - break; - default: - throw restore_error(); + switch (restoreType) { + case RESTORE_START: + f = stopAfter(runRestore(db, + restoreClusterFileOrig, + tagName, + restoreContainer, + backupKeys, + restoreVersion, + restoreTimestamp, + !dryRun, + !quietDisplay, + waitForDone, + addPrefix, + removePrefix)); + break; + case RESTORE_WAIT: + f = stopAfter(success(ba.waitRestore(db, KeyRef(tagName), true))); + break; + case RESTORE_ABORT: + f = stopAfter( + map(ba.abortRestore(db, KeyRef(tagName)), [tagName](FileBackupAgent::ERestoreState s) -> Void { + printf("RESTORE_ABORT Tag: %s State: %s\n", + tagName.c_str(), + FileBackupAgent::restoreStateText(s).toString().c_str()); + return Void(); + })); + break; + case RESTORE_STATUS: + // If no tag is specifically provided then print all tag status, don't just use "default" + if (tagProvided) + tag = tagName; + f = stopAfter(map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void { + printf("%s\n", s.c_str()); + return Void(); + })); + break; + default: + throw restore_error(); } break; case EXE_FASTRESTORE_TOOL: @@ -3960,7 +4104,8 @@ int main(int argc, char* argv[]) { } if (!fileExists(restoreClusterFileDest)) { - fprintf(stderr, "Restore destination cluster file '%s' does not exist.\n", + fprintf(stderr, + "Restore destination cluster file '%s' does not exist.\n", restoreClusterFileDest.c_str()); return FDB_EXIT_ERROR; } @@ -3968,16 +4113,18 @@ int main(int argc, char* argv[]) { try { db = Database::createDatabase(restoreClusterFileDest, Database::API_VERSION_LATEST); } catch (Error& e) { - fprintf(stderr, "Restore destination cluster file '%s' invalid: %s\n", - restoreClusterFileDest.c_str(), e.what()); + fprintf(stderr, + "Restore destination cluster file '%s' invalid: %s\n", + restoreClusterFileDest.c_str(), + e.what()); return FDB_EXIT_ERROR; } } // TODO: We have not implemented the code commented out in this case switch (restoreType) { case RESTORE_START: - f = stopAfter(runFastRestoreTool(db, tagName, restoreContainer, backupKeys, restoreVersion, !dryRun, - !quietDisplay, waitForDone)); + f = stopAfter(runFastRestoreTool( + db, tagName, restoreContainer, backupKeys, restoreVersion, !dryRun, !quietDisplay, waitForDone)); break; case RESTORE_WAIT: printf("[TODO][ERROR] FastRestore does not support RESTORE_WAIT yet!\n"); @@ -3988,8 +4135,8 @@ int main(int argc, char* argv[]) { printf("[TODO][ERROR] FastRestore does not support RESTORE_ABORT yet!\n"); throw restore_error(); // f = stopAfter( map(ba.abortRestore(db, KeyRef(tagName)), - //[tagName](FileBackupAgent::ERestoreState s) -> Void { printf("Tag: %s State: %s\n", - //tagName.c_str(), + //[tagName](FileBackupAgent::ERestoreState s) -> Void { printf("Tag: %s State: + //%s\n", tagName.c_str(), // FileBackupAgent::restoreStateText(s).toString().c_str()); return Void(); // }) ); break; @@ -3997,7 +4144,8 @@ int main(int argc, char* argv[]) { printf("[TODO][ERROR] FastRestore does not support RESTORE_STATUS yet!\n"); throw restore_error(); // If no tag is specifically provided then print all tag status, don't just use "default" - if (tagProvided) tag = tagName; + if (tagProvided) + tag = tagName; // f = stopAfter( map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void { // printf("%s\n", s.c_str()); // return Void(); @@ -4008,32 +4156,31 @@ int main(int argc, char* argv[]) { } break; case EXE_DR_AGENT: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter( runDBAgent(sourceDb, db) ); + f = stopAfter(runDBAgent(sourceDb, db)); break; case EXE_DB_BACKUP: - if(!initCluster()) + if (!initCluster()) return FDB_EXIT_ERROR; - switch (dbType) - { + switch (dbType) { case DB_START: - f = stopAfter( submitDBBackup(sourceDb, db, backupKeys, tagName) ); + f = stopAfter(submitDBBackup(sourceDb, db, backupKeys, tagName)); break; case DB_STATUS: - f = stopAfter( statusDBBackup(sourceDb, db, tagName, maxErrors) ); + f = stopAfter(statusDBBackup(sourceDb, db, tagName, maxErrors)); break; case DB_SWITCH: - f = stopAfter( switchDBBackup(sourceDb, db, backupKeys, tagName, forceAction) ); + f = stopAfter(switchDBBackup(sourceDb, db, backupKeys, tagName, forceAction)); break; case DB_ABORT: - f = stopAfter( abortDBBackup(sourceDb, db, tagName, partial, dstOnly) ); + f = stopAfter(abortDBBackup(sourceDb, db, tagName, partial, dstOnly)); break; case DB_PAUSE: - f = stopAfter( changeDBBackupResumed(sourceDb, db, true) ); + f = stopAfter(changeDBBackupResumed(sourceDb, db, true)); break; case DB_RESUME: - f = stopAfter( changeDBBackupResumed(sourceDb, db, false) ); + f = stopAfter(changeDBBackupResumed(sourceDb, db, false)); break; case DB_UNDEFINED: default: @@ -4050,35 +4197,29 @@ int main(int argc, char* argv[]) { runNetwork(); - if(f.isValid() && f.isReady() && !f.isError() && !f.get().present()) { + if (f.isValid() && f.isReady() && !f.isError() && !f.get().present()) { status = FDB_EXIT_ERROR; } - if(fstatus.isValid() && fstatus.isReady() && !fstatus.isError() && fstatus.get().present()) { + if (fstatus.isValid() && fstatus.isReady() && !fstatus.isError() && fstatus.get().present()) { status = fstatus.get().get(); } - #ifdef ALLOC_INSTRUMENTATION +#ifdef ALLOC_INSTRUMENTATION { - cout << "Page Counts: " - << FastAllocator<16>::pageCount << " " - << FastAllocator<32>::pageCount << " " - << FastAllocator<64>::pageCount << " " - << FastAllocator<128>::pageCount << " " - << FastAllocator<256>::pageCount << " " - << FastAllocator<512>::pageCount << " " - << FastAllocator<1024>::pageCount << " " - << FastAllocator<2048>::pageCount << " " - << FastAllocator<4096>::pageCount << " " - << FastAllocator<8192>::pageCount << " " - << FastAllocator<16384>::pageCount << endl; + cout << "Page Counts: " << FastAllocator<16>::pageCount << " " << FastAllocator<32>::pageCount << " " + << FastAllocator<64>::pageCount << " " << FastAllocator<128>::pageCount << " " + << FastAllocator<256>::pageCount << " " << FastAllocator<512>::pageCount << " " + << FastAllocator<1024>::pageCount << " " << FastAllocator<2048>::pageCount << " " + << FastAllocator<4096>::pageCount << " " << FastAllocator<8192>::pageCount << " " + << FastAllocator<16384>::pageCount << endl; - vector< std::pair > typeNames; - for( auto i = allocInstr.begin(); i != allocInstr.end(); ++i ) { + vector> typeNames; + for (auto i = allocInstr.begin(); i != allocInstr.end(); ++i) { std::string s; #ifdef __linux__ - char *demangled = abi::__cxa_demangle(i->first, NULL, NULL, NULL); + char* demangled = abi::__cxa_demangle(i->first, NULL, NULL, NULL); if (demangled) { s = demangled; if (StringRef(s).startsWith(LiteralStringRef("(anonymous namespace)::"))) @@ -4096,19 +4237,24 @@ int main(int argc, char* argv[]) { s = s.substr(LiteralStringRef("struct ").size()); #endif - typeNames.push_back( std::make_pair(s, i->first) ); + typeNames.push_back(std::make_pair(s, i->first)); } std::sort(typeNames.begin(), typeNames.end()); - for(int i=0; i { - std::string prompt; - ThreadReturnPromise> result; + struct Read : TypedAction { + std::string prompt; + ThreadReturnPromise> result; - virtual double getTimeEstimate() { return 0.0; } - explicit Read(std::string const& prompt) : prompt(prompt) {} - }; + virtual double getTimeEstimate() { return 0.0; } + explicit Read(std::string const& prompt) : prompt(prompt) {} + }; - void action(Read& r) { - try { - r.result.send( read(r.prompt) ); - } catch (Error& e) { - r.result.sendError(e); - } catch (...) { - r.result.sendError(unknown_error()); - } - } + void action(Read& r) { + try { + r.result.send(read(r.prompt)); + } catch (Error& e) { + r.result.sendError(e); + } catch (...) { + r.result.sendError(unknown_error()); + } + } private: - Optional read(std::string const& prompt) { - #if HAVE_LINENOISE - errno = 0; - char* line = linenoise(prompt.c_str()); - if (line) { - std::string s(line); - free(line); - return s; - } else { - if (errno == EAGAIN) // Ctrl-C - return std::string(); - return Optional(); - } - #else - std::string line; - std::fputs( prompt.c_str(), stdout ); - if (!std::getline( std::cin, line ).eof()) { - return line; - } else - return Optional(); - #endif - } + Optional read(std::string const& prompt) { +#if HAVE_LINENOISE + errno = 0; + char* line = linenoise(prompt.c_str()); + if (line) { + std::string s(line); + free(line); + return s; + } else { + if (errno == EAGAIN) // Ctrl-C + return std::string(); + return Optional(); + } +#else + std::string line; + std::fputs(prompt.c_str(), stdout); + if (!std::getline(std::cin, line).eof()) { + return line; + } else + return Optional(); +#endif + } }; -LineNoise::LineNoise( - std::function< void(std::string const&, std::vector&) > _completion_callback, - std::function< Hint(std::string const&) > _hint_callback, - int maxHistoryLines, - bool multiline ) - : threadPool( createGenericThreadPool() ) -{ - reader = new LineNoiseReader(); +LineNoise::LineNoise(std::function&)> _completion_callback, + std::function _hint_callback, + int maxHistoryLines, + bool multiline) + : threadPool(createGenericThreadPool()) { + reader = new LineNoiseReader(); - #if HAVE_LINENOISE - // It should be OK to call these functions from this thread, since read() can't be called yet - // The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely invoke the callbacks we've been given +#if HAVE_LINENOISE + // It should be OK to call these functions from this thread, since read() can't be called yet + // The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely + // invoke the callbacks we've been given - // linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables - static std::function< void(std::string const&, std::vector&) > completion_callback; - static std::function< Hint(std::string const&) > hint_callback; - completion_callback = _completion_callback; - hint_callback = _hint_callback; + // linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables + static std::function&)> completion_callback; + static std::function hint_callback; + completion_callback = _completion_callback; + hint_callback = _hint_callback; - linenoiseHistorySetMaxLen( maxHistoryLines ); - linenoiseSetMultiLine( multiline ); - linenoiseSetCompletionCallback( [](const char* line, linenoiseCompletions* lc) { - // This code will run in the thread pool - std::vector completions; - onMainThread( [line, &completions]() -> Future { - completion_callback(line, completions); - return Void(); - }).getBlocking(); - for( auto const& c : completions ) - linenoiseAddCompletion( lc, c.c_str() ); - }); - linenoiseSetHintsCallback( [](const char* line, int* color, int*bold) -> char* { - Hint h = onMainThread( [line]() -> Future { - return hint_callback(line); - }).getBlocking(); - if (!h.valid) return NULL; - *color = h.color; - *bold = h.bold; - return strdup( h.text.c_str() ); - }); - linenoiseSetFreeHintsCallback( free ); - #endif + linenoiseHistorySetMaxLen(maxHistoryLines); + linenoiseSetMultiLine(multiline); + linenoiseSetCompletionCallback([](const char* line, linenoiseCompletions* lc) { + // This code will run in the thread pool + std::vector completions; + onMainThread([line, &completions]() -> Future { + completion_callback(line, completions); + return Void(); + }).getBlocking(); + for (auto const& c : completions) + linenoiseAddCompletion(lc, c.c_str()); + }); + linenoiseSetHintsCallback([](const char* line, int* color, int* bold) -> char* { + Hint h = onMainThread([line]() -> Future { return hint_callback(line); }).getBlocking(); + if (!h.valid) + return NULL; + *color = h.color; + *bold = h.bold; + return strdup(h.text.c_str()); + }); + linenoiseSetFreeHintsCallback(free); +#endif - threadPool->addThread(reader); + threadPool->addThread(reader); } LineNoise::~LineNoise() { - threadPool.clear(); + threadPool.clear(); } -Future> LineNoise::read( std::string const& prompt ) { - auto r = new LineNoiseReader::Read(prompt); - auto f = r->result.getFuture(); - threadPool->post(r); - return f; +Future> LineNoise::read(std::string const& prompt) { + auto r = new LineNoiseReader::Read(prompt); + auto f = r->result.getFuture(); + threadPool->post(r); + return f; } ACTOR Future waitKeyboardInterrupt(boost::asio::io_service* ios) { - state boost::asio::signal_set signals(*ios, SIGINT); - Promise result; - signals.async_wait([result](const boost::system::error_code& error, int signal_number) { - if (error) { - result.sendError(io_error()); - } else { - result.send(Void()); - } - }); + state boost::asio::signal_set signals(*ios, SIGINT); + Promise result; + signals.async_wait([result](const boost::system::error_code& error, int signal_number) { + if (error) { + result.sendError(io_error()); + } else { + result.send(Void()); + } + }); - wait(result.getFuture()); - return Void(); + wait(result.getFuture()); + return Void(); } Future LineNoise::onKeyboardInterrupt() { - boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService); - if (!ios) return Never(); - return waitKeyboardInterrupt(ios); + boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService); + if (!ios) + return Never(); + return waitKeyboardInterrupt(ios); } -void LineNoise::historyAdd( std::string const& line ) { - #if HAVE_LINENOISE - linenoiseHistoryAdd( line.c_str() ); - #endif +void LineNoise::historyAdd(std::string const& line) { +#if HAVE_LINENOISE + linenoiseHistoryAdd(line.c_str()); +#endif } -void LineNoise::historyLoad( std::string const& filename ) { - #if HAVE_LINENOISE - if(linenoiseHistoryLoad(filename.c_str()) != 0) { - throw io_error(); - } - #endif +void LineNoise::historyLoad(std::string const& filename) { +#if HAVE_LINENOISE + if (linenoiseHistoryLoad(filename.c_str()) != 0) { + throw io_error(); + } +#endif } -void LineNoise::historySave( std::string const& filename ) { - #if HAVE_LINENOISE - if(linenoiseHistorySave(filename.c_str()) != 0) { - throw io_error(); - } - #endif +void LineNoise::historySave(std::string const& filename) { +#if HAVE_LINENOISE + if (linenoiseHistorySave(filename.c_str()) != 0) { + throw io_error(); + } +#endif } diff --git a/fdbcli/FlowLineNoise.h b/fdbcli/FlowLineNoise.h index b81ce88939..87f88c69de 100644 --- a/fdbcli/FlowLineNoise.h +++ b/fdbcli/FlowLineNoise.h @@ -26,39 +26,37 @@ #include struct LineNoise : NonCopyable { - // Wraps the linenoise library so that it can be called from asynchronous Flow code - // Only create one of these at a time; the linenoise library only supports one history - // - // The current implementation does not support calling read concurrently with any other - // function (or itself). + // Wraps the linenoise library so that it can be called from asynchronous Flow code + // Only create one of these at a time; the linenoise library only supports one history + // + // The current implementation does not support calling read concurrently with any other + // function (or itself). - struct Hint { - std::string text; - int color; - bool bold; - bool valid; - Hint() : text(), color(), bold(), valid() {} - Hint( std::string const& text, int color, bool bold ) : text(text), color(color), bold(bold), valid(true) {} - }; + struct Hint { + std::string text; + int color; + bool bold; + bool valid; + Hint() : text(), color(), bold(), valid() {} + Hint(std::string const& text, int color, bool bold) : text(text), color(color), bold(bold), valid(true) {} + }; - LineNoise( - std::function< void(std::string const&, std::vector&) > completion_callback, - std::function< Hint(std::string const&) > hint_callback, - int maxHistoryLines, - bool multiline - ); - ~LineNoise(); + LineNoise(std::function&)> completion_callback, + std::function hint_callback, + int maxHistoryLines, + bool multiline); + ~LineNoise(); - Future< Optional > read( std::string const& prompt ); // Returns "nothing" on EOF - void historyAdd( std::string const& line ); + Future> read(std::string const& prompt); // Returns "nothing" on EOF + void historyAdd(std::string const& line); - void historyLoad( std::string const& filename ); - void historySave( std::string const& filename ); + void historyLoad(std::string const& filename); + void historySave(std::string const& filename); - static Future onKeyboardInterrupt(); // Returns when Ctrl-C is next pressed (i.e. SIGINT) + static Future onKeyboardInterrupt(); // Returns when Ctrl-C is next pressed (i.e. SIGINT) - Reference threadPool; - struct LineNoiseReader* reader; + Reference threadPool; + struct LineNoiseReader* reader; }; #endif \ No newline at end of file diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index a568ba0f6f..c2a0a2e2e7 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -52,7 +52,7 @@ #include "fdbclient/versions.h" -#include "flow/actorcompiler.h" // This must be the last #include. +#include "flow/actorcompiler.h" // This must be the last #include. extern const char* getSourceVersion(); @@ -98,7 +98,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP }, TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS }; + SO_END_OF_OPTIONS }; void printAtCol(const char* text, int col) { const char* iter = text; @@ -107,12 +107,15 @@ void printAtCol(const char* text, int col) { do { iter++; - if (*iter == '\n' || *iter == ' ' || *iter == '\0') space = iter; + if (*iter == '\n' || *iter == ' ' || *iter == '\0') + space = iter; if (*iter == '\n' || *iter == '\0' || (iter - start == col)) { - if (!space) space = iter; + if (!space) + space = iter; printf("%.*s\n", (int)(space - start), start); start = space; - if (*start == ' ' || *start == '\n') start++; + if (*start == ' ' || *start == '\n') + start++; space = NULL; } } while (*iter); @@ -125,12 +128,15 @@ std::string lineWrap(const char* text, int col) { std::string out = ""; do { iter++; - if (*iter == '\n' || *iter == ' ' || *iter == '\0') space = iter; + if (*iter == '\n' || *iter == ' ' || *iter == '\0') + space = iter; if (*iter == '\n' || *iter == '\0' || (iter - start == col)) { - if (!space) space = iter; + if (!space) + space = iter; out += format("%.*s\n", (int)(space - start), start); start = space; - if (*start == ' '/* || *start == '\n'*/) start++; + if (*start == ' ' /* || *start == '\n'*/) + start++; space = NULL; } } while (*iter); @@ -139,78 +145,83 @@ std::string lineWrap(const char* text, int col) { class FdbOptions { public: - //Prints an error and throws invalid_option or invalid_option_value if the option could not be set - void setOption(Reference tr, StringRef optionStr, bool enabled, Optional arg, bool intrans) { + // Prints an error and throws invalid_option or invalid_option_value if the option could not be set + void setOption(Reference tr, + StringRef optionStr, + bool enabled, + Optional arg, + bool intrans) { auto transactionItr = transactionOptions.legalOptions.find(optionStr.toString()); - if(transactionItr != transactionOptions.legalOptions.end()) + if (transactionItr != transactionOptions.legalOptions.end()) setTransactionOption(tr, transactionItr->second, enabled, arg, intrans); else { - fprintf(stderr, "ERROR: invalid option '%s'. Try `help options' for a list of available options.\n", optionStr.toString().c_str()); + fprintf(stderr, + "ERROR: invalid option '%s'. Try `help options' for a list of available options.\n", + optionStr.toString().c_str()); throw invalid_option(); } } - //Applies all enabled transaction options to the given transaction + // Applies all enabled transaction options to the given transaction void apply(Reference tr) { - for(auto itr = transactionOptions.options.begin(); itr != transactionOptions.options.end(); ++itr) + for (auto itr = transactionOptions.options.begin(); itr != transactionOptions.options.end(); ++itr) tr->setOption(itr->first, itr->second.castTo()); } - //Returns true if any options have been set - bool hasAnyOptionsEnabled() { - return !transactionOptions.options.empty(); - } + // Returns true if any options have been set + bool hasAnyOptionsEnabled() { return !transactionOptions.options.empty(); } - //Prints a list of enabled options, along with their parameters (if any) + // Prints a list of enabled options, along with their parameters (if any) void print() { bool found = false; found = found || transactionOptions.print(); - if(!found) + if (!found) printf("There are no options enabled\n"); } - //Returns a vector of the names of all documented options - std::vector getValidOptions() { - return transactionOptions.getValidOptions(); - } + // Returns a vector of the names of all documented options + std::vector getValidOptions() { return transactionOptions.getValidOptions(); } - //Prints the help string obtained by invoking `help options' - void printHelpString() { - transactionOptions.printHelpString(); - } + // Prints the help string obtained by invoking `help options' + void printHelpString() { transactionOptions.printHelpString(); } private: - //Sets a transaction option. If intrans == true, then this option is also applied to the passed in transaction. - void setTransactionOption(Reference tr, FDBTransactionOptions::Option option, bool enabled, Optional arg, bool intrans) { - if(enabled && arg.present() != FDBTransactionOptions::optionInfo.getMustExist(option).hasParameter) { + // Sets a transaction option. If intrans == true, then this option is also applied to the passed in transaction. + void setTransactionOption(Reference tr, + FDBTransactionOptions::Option option, + bool enabled, + Optional arg, + bool intrans) { + if (enabled && arg.present() != FDBTransactionOptions::optionInfo.getMustExist(option).hasParameter) { fprintf(stderr, "ERROR: option %s a parameter\n", arg.present() ? "did not expect" : "expected"); throw invalid_option_value(); } - if(intrans) + if (intrans) tr->setOption(option, arg); transactionOptions.setOption(option, enabled, arg.castTo()); } - //A group of enabled options (of type T::Option) as well as a legal options map from string to T::Option + // A group of enabled options (of type T::Option) as well as a legal options map from string to T::Option template struct OptionGroup { std::map>> options; std::map legalOptions; - OptionGroup() { } - OptionGroup(OptionGroup &base) : options(base.options.begin(), base.options.end()), legalOptions(base.legalOptions) { } + OptionGroup() {} + OptionGroup(OptionGroup& base) + : options(base.options.begin(), base.options.end()), legalOptions(base.legalOptions) {} - //Enable or disable an option. Returns true if option value changed + // Enable or disable an option. Returns true if option value changed bool setOption(typename T::Option option, bool enabled, Optional arg) { auto optionItr = options.find(option); - if(enabled && (optionItr == options.end() || Optional>(optionItr->second).castTo< StringRef >() != arg)) { + if (enabled && (optionItr == options.end() || + Optional>(optionItr->second).castTo() != arg)) { options[option] = arg.castTo>(); return true; - } - else if(!enabled && optionItr != options.end()) { + } else if (!enabled && optionItr != options.end()) { options.erase(optionItr); return true; } @@ -218,14 +229,14 @@ private: return false; } - //Prints a list of all enabled options in this group + // Prints a list of all enabled options in this group bool print() { bool found = false; - for(auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) { + for (auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) { auto optionItr = options.find(itr->second); - if(optionItr != options.end()) { - if(optionItr->second.present()) + if (optionItr != options.end()) { + if (optionItr->second.present()) printf("%s: `%s'\n", itr->first.c_str(), formatStringRef(optionItr->second.get()).c_str()); else printf("%s\n", itr->first.c_str()); @@ -237,7 +248,7 @@ private: return found; } - //Returns true if the specified option is documented + // Returns true if the specified option is documented bool isDocumented(typename T::Option option) { FDBOptionInfo info = T::optionInfo.getMustExist(option); @@ -245,25 +256,25 @@ private: return !info.comment.empty() && info.comment.substr(0, deprecatedStr.size()) != deprecatedStr; } - //Returns a vector of the names of all documented options + // Returns a vector of the names of all documented options std::vector getValidOptions() { std::vector ret; for (auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) - if(isDocumented(itr->second)) + if (isDocumented(itr->second)) ret.push_back(itr->first); return ret; } - //Prints a help string for each option in this group. Any options with no comment - //are excluded from this help string. Lines are wrapped to 80 characters. + // Prints a help string for each option in this group. Any options with no comment + // are excluded from this help string. Lines are wrapped to 80 characters. void printHelpString() { - for(auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) { - if(isDocumented(itr->second)) { + for (auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) { + if (isDocumented(itr->second)) { FDBOptionInfo info = T::optionInfo.getMustExist(itr->second); std::string helpStr = info.name + " - " + info.comment; - if(info.hasParameter) + if (info.hasParameter) helpStr += " " + info.parameterComment; helpStr += "\n"; @@ -277,15 +288,15 @@ private: public: FdbOptions() { - for(auto itr = FDBTransactionOptions::optionInfo.begin(); itr != FDBTransactionOptions::optionInfo.end(); ++itr) + for (auto itr = FDBTransactionOptions::optionInfo.begin(); itr != FDBTransactionOptions::optionInfo.end(); + ++itr) transactionOptions.legalOptions[itr->second.name] = itr->first; } - FdbOptions(FdbOptions &base) : transactionOptions(base.transactionOptions) { } + FdbOptions(FdbOptions& base) : transactionOptions(base.transactionOptions) {} }; -static std::string formatStringRef(StringRef item, bool fullEscaping = false) -{ +static std::string formatStringRef(StringRef item, bool fullEscaping = false) { std::string ret; for (int i = 0; i < item.size(); i++) { @@ -304,17 +315,14 @@ static std::string formatStringRef(StringRef item, bool fullEscaping = false) return ret; } -static bool tokencmp(StringRef token, const char *command) -{ +static bool tokencmp(StringRef token, const char* command) { if (token.size() != strlen(command)) return false; return !memcmp(token.begin(), command, token.size()); } - -static std::vector> parseLine(std::string& line, bool& err, bool& partial) -{ +static std::vector> parseLine(std::string& line, bool& err, bool& partial) { err = false; partial = false; @@ -329,69 +337,69 @@ static std::vector> parseLine(std::string& line, bool& er while (i <= line.length()) { switch (line[i]) { - case ';': - if (!quoted) { - if (i > offset || (forcetoken && i == offset)) - buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); - ret.push_back(std::move(buf)); - offset = i = line.find_first_not_of(' ', i+1); - forcetoken = false; - } else - i++; - break; + case ';': + if (!quoted) { + if (i > offset || (forcetoken && i == offset)) + buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); + ret.push_back(std::move(buf)); + offset = i = line.find_first_not_of(' ', i + 1); + forcetoken = false; + } else + i++; + break; + case '"': + quoted = !quoted; + line.erase(i, 1); + forcetoken = true; + break; + case ' ': + if (!quoted) { + if (i > offset || (forcetoken && i == offset)) + buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); + offset = i = line.find_first_not_of(' ', i); + forcetoken = false; + } else + i++; + break; + case '\\': + if (i + 2 > line.length()) { + err = true; + ret.push_back(std::move(buf)); + return ret; + } + switch (line[i + 1]) { + char ent, save; case '"': - quoted = !quoted; - line.erase(i, 1); - forcetoken = true; - break; - case ' ': - if (!quoted) { - if (i > offset || (forcetoken && i == offset)) - buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); - offset = i = line.find_first_not_of(' ', i); - forcetoken = false; - } else - i++; - break; case '\\': - if (i + 2 > line.length()) { + case ' ': + case ';': + line.erase(i, 1); + break; + case 'x': + if (i + 4 > line.length()) { err = true; ret.push_back(std::move(buf)); return ret; } - switch (line[i+1]) { - char ent, save; - case '"': - case '\\': - case ' ': - case ';': - line.erase(i, 1); - break; - case 'x': - if (i + 4 > line.length()) { - err = true; - ret.push_back(std::move(buf)); - return ret; - } - char *pEnd; - save = line[i + 4]; - line[i + 4] = 0; - ent = char(strtoul(line.data() + i + 2, &pEnd, 16)); - if (*pEnd) { - err = true; - ret.push_back(std::move(buf)); - return ret; - } - line[i + 4] = save; - line.replace(i, 4, 1, ent); - break; - default: - err = true; - ret.push_back(std::move(buf)); - return ret; + char* pEnd; + save = line[i + 4]; + line[i + 4] = 0; + ent = char(strtoul(line.data() + i + 2, &pEnd, 16)); + if (*pEnd) { + err = true; + ret.push_back(std::move(buf)); + return ret; } + line[i + 4] = save; + line.replace(i, 4, 1, ent); + break; default: - i++; + err = true; + ret.push_back(std::move(buf)); + return ret; + } + default: + i++; } } @@ -409,12 +417,14 @@ static std::vector> parseLine(std::string& line, bool& er static void printProgramUsage(const char* name) { printf("FoundationDB CLI " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n" - "usage: %s [OPTIONS]\n" - "\n", name); + "usage: %s [OPTIONS]\n" + "\n", + name); printf(" -C CONNFILE The path of a file containing the connection string for the\n" - " FoundationDB cluster. The default is first the value of the\n" - " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" - " then `%s'.\n", platform::getDefaultClusterFilePath().c_str()); + " FoundationDB cluster. The default is first the value of the\n" + " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" + " then `%s'.\n", + platform::getDefaultClusterFilePath().c_str()); printf(" --log Enables trace file logging for the CLI session.\n" " --log-dir PATH Specifes the output directory for trace files. If\n" " unspecified, defaults to the current directory. Has\n" @@ -431,13 +441,12 @@ static void printProgramUsage(const char* name) { #endif " --knob_KNOBNAME KNOBVALUE\n" " Changes a knob option. KNOBNAME should be lowercase.\n" - " --debug-tls Prints the TLS configuration and certificate chain, then exits.\n" - " Useful in reporting and diagnosing TLS issues.\n" + " --debug-tls Prints the TLS configuration and certificate chain, then exits.\n" + " Useful in reporting and diagnosing TLS issues.\n" " -v, --version Print FoundationDB CLI version information and exit.\n" " -h, --help Display this help and exit.\n"); } - struct CommandHelp { std::string usage; std::string short_desc; @@ -453,145 +462,200 @@ std::set hiddenCommands; #define ESCAPINGKV "\n\nFor information on escaping keys and values, type `help escaping'." void initHelp() { - helpMap["begin"] = CommandHelp( - "begin", - "begin a new transaction", - "By default, the fdbcli operates in autocommit mode. All operations are performed in their own transaction, and are automatically committed for you. By explicitly beginning a transaction, successive operations are all performed as part of a single transaction.\n\nTo commit the transaction, use the commit command. To discard the transaction, use the reset command."); - helpMap["commit"] = CommandHelp( - "commit", - "commit the current transaction", - "Any sets or clears executed after the start of the current transaction will be committed to the database. On success, the committed version number is displayed. If commit fails, the error is displayed and the transaction must be retried."); + helpMap["begin"] = + CommandHelp("begin", + "begin a new transaction", + "By default, the fdbcli operates in autocommit mode. All operations are performed in their own " + "transaction, and are automatically committed for you. By explicitly beginning a transaction, " + "successive operations are all performed as part of a single transaction.\n\nTo commit the " + "transaction, use the commit command. To discard the transaction, use the reset command."); + helpMap["commit"] = CommandHelp("commit", + "commit the current transaction", + "Any sets or clears executed after the start of the current transaction will be " + "committed to the database. On success, the committed version number is displayed. " + "If commit fails, the error is displayed and the transaction must be retried."); helpMap["clear"] = CommandHelp( - "clear ", - "clear a key from the database", - "Clear succeeds even if the specified key is not present, but may fail because of conflicts." ESCAPINGK); + "clear ", + "clear a key from the database", + "Clear succeeds even if the specified key is not present, but may fail because of conflicts." ESCAPINGK); helpMap["clearrange"] = CommandHelp( - "clearrange ", - "clear a range of keys from the database", - "All keys between BEGINKEY (inclusive) and ENDKEY (exclusive) are cleared from the database. This command will succeed even if the specified range is empty, but may fail because of conflicts." ESCAPINGK); + "clearrange ", + "clear a range of keys from the database", + "All keys between BEGINKEY (inclusive) and ENDKEY (exclusive) are cleared from the database. This command will " + "succeed even if the specified range is empty, but may fail because of conflicts." ESCAPINGK); helpMap["configure"] = CommandHelp( - "configure [new] |logs=|resolvers=>*", - "change the database configuration", - "The `new' option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When used, both a redundancy mode and a storage engine must be specified.\n\nRedundancy mode:\n single - one copy of the data. Not fault tolerant.\n double - two copies of data (survive one failure).\n triple - three copies of data (survive two failures).\n three_data_hall - See the Admin Guide.\n three_datacenter - See the Admin Guide.\n\nStorage engine:\n ssd - B-Tree storage engine optimized for solid state disks.\n memory - Durable in-memory storage engine for small datasets.\n\nproxies=: Sets the desired number of proxies in the cluster. Must be at least 1, or set to -1 which restores the number of proxies to the default value.\n\nlogs=: Sets the desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of logs to the default value.\n\nresolvers=: Sets the desired number of resolvers in the cluster. Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\nSee the FoundationDB Administration Guide for more information."); + "configure [new] " + "|" + "logs=|resolvers=>*", + "change the database configuration", + "The `new' option, if present, initializes a new database with the given configuration rather than changing " + "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be " + "specified.\n\nRedundancy mode:\n single - one copy of the data. Not fault tolerant.\n double - two copies " + "of data (survive one failure).\n triple - three copies of data (survive two failures).\n three_data_hall - " + "See the Admin Guide.\n three_datacenter - See the Admin Guide.\n\nStorage engine:\n ssd - B-Tree storage " + "engine optimized for solid state disks.\n memory - Durable in-memory storage engine for small " + "datasets.\n\nproxies=: Sets the desired number of proxies in the cluster. Must be at least 1, or set " + "to -1 which restores the number of proxies to the default value.\n\nlogs=: Sets the desired number of " + "log servers in the cluster. Must be at least 1, or set to -1 which restores the number of logs to the default " + "value.\n\nresolvers=: Sets the desired number of resolvers in the cluster. Must be at least 1, or " + "set to -1 which restores the number of resolvers to the default value.\n\nSee the FoundationDB Administration " + "Guide for more information."); helpMap["fileconfigure"] = CommandHelp( - "fileconfigure [new] ", - "change the database configuration from a file", - "The `new' option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. Load a JSON document from the provided file, and change the database configuration to match the contents of the JSON document. The format should be the same as the value of the \"configuration\" entry in status JSON without \"excluded_servers\" or \"coordinators_count\"."); + "fileconfigure [new] ", + "change the database configuration from a file", + "The `new' option, if present, initializes a new database with the given configuration rather than changing " + "the configuration of an existing one. Load a JSON document from the provided file, and change the database " + "configuration to match the contents of the JSON document. The format should be the same as the value of the " + "\"configuration\" entry in status JSON without \"excluded_servers\" or \"coordinators_count\"."); helpMap["coordinators"] = CommandHelp( - "coordinators auto|
+ [description=new_cluster_description]", - "change cluster coordinators or description", - "If 'auto' is specified, coordinator addresses will be choosen automatically to support the configured redundancy level. (If the current set of coordinators are healthy and already support the redundancy level, nothing will be changed.)\n\nOtherwise, sets the coordinators to the list of IP:port pairs specified by
+. An fdbserver process must be running on each of the specified addresses.\n\ne.g. coordinators 10.0.0.1:4000 10.0.0.2:4000 10.0.0.3:4000\n\nIf 'description=desc' is specified then the description field in the cluster\nfile is changed to desc, which must match [A-Za-z0-9_]+."); - helpMap["exclude"] = - CommandHelp("exclude [FORCE] [failed] [no_wait] ", "exclude servers from the database", - "If no addresses are specified, lists the set of excluded servers.\n\nFor each IP address or " - "IP:port pair in , adds the address to the set of excluded servers then waits until all " - "database state has been safely moved away from the specified servers. If 'no_wait' is set, the " - "command returns \nimmediately without checking if the exclusions have completed successfully.\n" - "If 'FORCE' is set, the command does not perform safety checks before excluding.\n" - "If 'failed' is set, the transaction log queue is dropped pre-emptively before waiting\n" - "for data movement to finish and the server cannot be included again."); - helpMap["include"] = CommandHelp( - "include all|", - "permit previously-excluded servers to rejoin the database", - "If `all' is specified, the excluded servers list is cleared.\n\nFor each IP address or IP:port pair in , removes any matching exclusions from the excluded servers list. (A specified IP will match all IP:* exclusion entries)"); - helpMap["setclass"] = CommandHelp( - "setclass [
]", - "change the class of a process", - "If no address and class are specified, lists the classes of all servers.\n\nSetting the class to `default' resets the process class to the class specified on the command line. The available classes are `unset', `storage', `transaction', `resolution', `proxy', `master', `test', `unset', `stateless', `log', `router', `cluster_controller', `fast_restore', `data_distributor', `coordinator', `ratekeeper', `storage_cache', `backup', and `default'."); - helpMap["status"] = CommandHelp( - "status [minimal|details|json]", - "get the status of a FoundationDB cluster", - "If the cluster is down, this command will print a diagnostic which may be useful in figuring out what is wrong. If the cluster is running, this command will print cluster statistics.\n\nSpecifying `minimal' will provide a minimal description of the status of your database.\n\nSpecifying `details' will provide load information for individual workers.\n\nSpecifying `json' will provide status information in a machine readable JSON format."); + "coordinators auto|
+ [description=new_cluster_description]", + "change cluster coordinators or description", + "If 'auto' is specified, coordinator addresses will be choosen automatically to support the configured " + "redundancy level. (If the current set of coordinators are healthy and already support the redundancy level, " + "nothing will be changed.)\n\nOtherwise, sets the coordinators to the list of IP:port pairs specified by " + "
+. An fdbserver process must be running on each of the specified addresses.\n\ne.g. coordinators " + "10.0.0.1:4000 10.0.0.2:4000 10.0.0.3:4000\n\nIf 'description=desc' is specified then the description field in " + "the cluster\nfile is changed to desc, which must match [A-Za-z0-9_]+."); + helpMap["exclude"] = CommandHelp( + "exclude [FORCE] [failed] [no_wait] ", + "exclude servers from the database", + "If no addresses are specified, lists the set of excluded servers.\n\nFor each IP address or " + "IP:port pair in , adds the address to the set of excluded servers then waits until all " + "database state has been safely moved away from the specified servers. If 'no_wait' is set, the " + "command returns \nimmediately without checking if the exclusions have completed successfully.\n" + "If 'FORCE' is set, the command does not perform safety checks before excluding.\n" + "If 'failed' is set, the transaction log queue is dropped pre-emptively before waiting\n" + "for data movement to finish and the server cannot be included again."); + helpMap["include"] = + CommandHelp("include all|", + "permit previously-excluded servers to rejoin the database", + "If `all' is specified, the excluded servers list is cleared.\n\nFor each IP address or IP:port " + "pair in , removes any matching exclusions from the excluded servers list. (A " + "specified IP will match all IP:* exclusion entries)"); + helpMap["setclass"] = + CommandHelp("setclass [
]", + "change the class of a process", + "If no address and class are specified, lists the classes of all servers.\n\nSetting the class to " + "`default' resets the process class to the class specified on the command line. The available " + "classes are `unset', `storage', `transaction', `resolution', `proxy', `master', `test', `unset', " + "`stateless', `log', `router', `cluster_controller', `fast_restore', `data_distributor', " + "`coordinator', `ratekeeper', `storage_cache', `backup', and `default'."); + helpMap["status"] = + CommandHelp("status [minimal|details|json]", + "get the status of a FoundationDB cluster", + "If the cluster is down, this command will print a diagnostic which may be useful in figuring out " + "what is wrong. If the cluster is running, this command will print cluster " + "statistics.\n\nSpecifying `minimal' will provide a minimal description of the status of your " + "database.\n\nSpecifying `details' will provide load information for individual " + "workers.\n\nSpecifying `json' will provide status information in a machine readable JSON format."); helpMap["exit"] = CommandHelp("exit", "exit the CLI", ""); helpMap["quit"] = CommandHelp(); helpMap["waitconnected"] = CommandHelp(); helpMap["waitopen"] = CommandHelp(); - helpMap["sleep"] = CommandHelp( - "sleep ", - "sleep for a period of time", - ""); - helpMap["get"] = CommandHelp( - "get ", - "fetch the value for a given key", - "Displays the value of KEY in the database, or `not found' if KEY is not present." ESCAPINGK); - helpMap["getrange"] = CommandHelp( - "getrange [ENDKEY] [LIMIT]", - "fetch key/value pairs in a range of keys", - "Displays up to LIMIT keys and values for keys between BEGINKEY (inclusive) and ENDKEY (exclusive). If ENDKEY is omitted, then the range will include all keys starting with BEGINKEY. LIMIT defaults to 25 if omitted." ESCAPINGK); + helpMap["sleep"] = CommandHelp("sleep ", "sleep for a period of time", ""); + helpMap["get"] = + CommandHelp("get ", + "fetch the value for a given key", + "Displays the value of KEY in the database, or `not found' if KEY is not present." ESCAPINGK); + helpMap["getrange"] = CommandHelp("getrange [ENDKEY] [LIMIT]", + "fetch key/value pairs in a range of keys", + "Displays up to LIMIT keys and values for keys between BEGINKEY (inclusive) and " + "ENDKEY (exclusive). If ENDKEY is omitted, then the range will include all keys " + "starting with BEGINKEY. LIMIT defaults to 25 if omitted." ESCAPINGK); helpMap["getrangekeys"] = CommandHelp( - "getrangekeys [ENDKEY] [LIMIT]", - "fetch keys in a range of keys", - "Displays up to LIMIT keys for keys between BEGINKEY (inclusive) and ENDKEY (exclusive). If ENDKEY is omitted, then the range will include all keys starting with BEGINKEY. LIMIT defaults to 25 if omitted." ESCAPINGK); + "getrangekeys [ENDKEY] [LIMIT]", + "fetch keys in a range of keys", + "Displays up to LIMIT keys for keys between BEGINKEY (inclusive) and ENDKEY (exclusive). If ENDKEY is omitted, " + "then the range will include all keys starting with BEGINKEY. LIMIT defaults to 25 if omitted." ESCAPINGK); helpMap["getversion"] = - CommandHelp("getversion", "Fetch the current read version", + CommandHelp("getversion", + "Fetch the current read version", "Displays the current read version of the database or currently running transaction."); helpMap["advanceversion"] = CommandHelp( - "advanceversion ", "Force the cluster to recover at the specified version", + "advanceversion ", + "Force the cluster to recover at the specified version", "Forces the cluster to recover at the specified version. If the specified version is larger than the current " "version of the cluster, the cluster version is advanced " "to the specified version via a forced recovery."); - helpMap["reset"] = CommandHelp( - "reset", - "reset the current transaction", - "Any sets or clears executed after the start of the active transaction will be discarded."); - helpMap["rollback"] = CommandHelp( - "rollback", - "rolls back the current transaction", - "The active transaction will be discarded, including any sets or clears executed since the transaction was started."); - helpMap["set"] = CommandHelp( - "set ", - "set a value for a given key", - "If KEY is not already present in the database, it will be created." ESCAPINGKV); + helpMap["reset"] = + CommandHelp("reset", + "reset the current transaction", + "Any sets or clears executed after the start of the active transaction will be discarded."); + helpMap["rollback"] = CommandHelp("rollback", + "rolls back the current transaction", + "The active transaction will be discarded, including any sets or clears executed " + "since the transaction was started."); + helpMap["set"] = CommandHelp("set ", + "set a value for a given key", + "If KEY is not already present in the database, it will be created." ESCAPINGKV); helpMap["option"] = CommandHelp( - "option