From bb081bcfc20749ca7d88a0aedd1fff96a70bb47b Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 26 Apr 2023 11:08:51 +0800 Subject: [PATCH] refact: rename package for cli & dist & node & rocksdb & test modules TODO: some files need update manully Change-Id: If83118e2dd123a7a88f9c5f7b33fbc7cf1b55de7 --- hg-store-cli/deploy-snapshot.sh | 17 ++ hg-store-cli/pom.xml | 19 +- .../baidu/hugegraph/store/cli/AppConfig.java | 40 --- .../hugegraph/store/cli/util/HgMetricX.java | 48 ---- .../apache/hugegraph/store/cli/AppConfig.java | 58 +++++ .../store/cli/StoreConsoleApplication.java | 61 +++-- .../store/cli/loader/HgThread2DB.java | 231 +++++++++++------- .../store/cli/scan/GrpcShardScanner.java | 27 +- .../store/cli/scan/HgStoreCommitter.java | 38 ++- .../store/cli/scan/HgStoreScanner.java | 70 ++++-- .../hugegraph/store/cli/util/HgCliUtil.java | 50 ++-- .../hugegraph/store/cli/util/HgMetricX.java | 61 +++++ .../main/resources/hg-store-client.properties | 18 +- hg-store-cli/src/main/resources/log4j2.xml | 23 +- .../baidu/hugegraph/store/HgStoreEngine.java | 2 +- .../business/AbstractSelectIterator.java | 2 +- .../store/business/BusinessHandler.java | 2 +- .../store/business/BusinessHandlerImpl.java | 16 +- .../store/business/DefaultDataMover.java | 4 +- .../store/business/FilterIterator.java | 4 +- .../store/business/GraphStoreIterator.java | 4 +- .../store/business/InnerKeyFilter.java | 4 +- .../business/MultiPartitionIterator.java | 2 +- .../store/business/SelectIterator.java | 4 +- .../store/meta/DeletedFileManager.java | 2 +- .../store/meta/base/DBSessionBuilder.java | 2 +- .../store/meta/base/GlobalMetaStore.java | 4 +- .../store/meta/base/MetaStoreBase.java | 6 +- .../store/meta/base/PartitionMetaStore.java | 2 +- .../store/metric/SystemMetricService.java | 4 +- .../store/options/RaftRocksdbOptions.java | 2 +- .../store/util/PartitionMetaStoreWrapper.java | 2 +- .../hugegraph/store/HgBusinessImplTest.java | 6 +- .../baidu/hugegraph/store/UnitTestBase.java | 4 +- .../store/meta/GraphIDManagerTest.java | 2 +- hg-store-dist/pom.xml | 17 ++ .../assembly/descriptor/server-assembly.xml | 17 ++ .../static/bin/restart-hugegraph-store.sh | 17 ++ .../static/bin/start-hugegraph-store.sh | 17 ++ .../static/bin/stop-hugegraph-store.sh | 17 ++ hg-store-dist/src/assembly/static/bin/util.sh | 17 ++ .../src/assembly/static/conf/log4j2.xml | 35 ++- hg-store-node/pom.xml | 19 +- hg-store-node/src/assembly/static/bin/app.sh | 17 ++ .../src/assembly/static/bin/hg-get-release.sh | 17 ++ .../src/assembly/static/bin/hg-update.sh | 17 ++ .../hugegraph/store/node/AppShutdownHook.java | 42 ---- .../controller/HgStoreMetricsController.java | 49 ---- .../store/node/entry/RestResult.java | 14 -- .../store/node/grpc/EmptyIterator.java | 39 --- .../store/node/grpc/GRpcServerConfig.java | 29 --- .../store/node/grpc/HgStoreNodeState.java | 40 --- .../store/node/grpc/QueryCondition.java | 22 -- .../store/node/grpc/ScanOneShotResponse.java | 78 ------ .../hugegraph/store/node/grpc/ScanQuery.java | 87 ------- .../node/listener/ContextClosedListener.java | 40 --- .../store/node/metrics/DriveMetrics.java | 39 --- .../store/node/metrics/GRpcExMetrics.java | 73 ------ .../store/node/metrics/MetricsConfig.java | 30 --- .../store/node/metrics/ProcfsMetrics.java | 49 ---- .../baidu/hugegraph/store/node/util/Err.java | 23 -- .../store/node/util/HgStoreConst.java | 21 -- .../hugegraph/store/node/util/Result.java | 22 -- .../hugegraph/store/node/util/TkEntry.java | 48 ---- .../hugegraph/store/node/AppConfig.java | 23 +- .../hugegraph/store/node/AppShutdownHook.java | 59 +++++ .../store/node/StoreNodeApplication.java | 38 ++- .../controller/HgStoreMetricsController.java | 67 +++++ .../controller/HgStoreStatusController.java | 30 ++- .../node/controller/HgTestController.java | 54 ++-- .../store/node/controller/IndexAPI.java | 64 +++-- .../store/node/controller/PartitionAPI.java | 90 ++++--- .../store/node/entry/RestResult.java | 31 +++ .../store/node/grpc/BatchGrpcClosure.java | 82 ++++--- .../store/node/grpc/BatchScanIterator.java | 46 +++- .../store/node/grpc/EmptyIterator.java | 56 +++++ .../store/node/grpc/FusingScanIterator.java | 24 +- .../store/node/grpc/GRpcServerConfig.java | 46 ++++ .../store/node/grpc/GrpcClosure.java | 31 ++- .../store/node/grpc/HgStoreNodeService.java | 111 +++++---- .../store/node/grpc/HgStoreNodeState.java | 57 +++++ .../store/node/grpc/HgStoreSessionImpl.java | 190 ++++++++------ .../store/node/grpc/HgStoreStateService.java | 19 +- .../store/node/grpc/HgStoreStateSubject.java | 45 +++- .../store/node/grpc/HgStoreStreamImpl.java | 30 ++- .../store/node/grpc/HgStoreWrapperEx.java | 45 +++- .../store/node/grpc/ParallelScanIterator.java | 69 ++++-- .../store/node/grpc/QueryCondition.java | 39 +++ .../node/grpc/ScanBatchOneShotResponse.java | 40 ++- .../store/node/grpc/ScanBatchResponse.java | 48 ++-- .../store/node/grpc/ScanBatchResponse3.java | 105 +++++--- .../node/grpc/ScanBatchResponseFactory.java | 7 +- .../store/node/grpc/ScanOneShotResponse.java | 95 +++++++ .../hugegraph/store/node/grpc/ScanQuery.java | 104 ++++++++ .../store/node/grpc/ScanQueryProducer.java | 48 +++- .../store/node/grpc/ScanStreamResponse.java | 66 +++-- .../hugegraph/store/node/grpc/ScanUtil.java | 144 +++++++---- .../store/node/grpc/scan/GraphStoreImpl.java | 25 +- .../node/grpc/scan/ScanResponseObserver.java | 44 ++-- .../node/listener/ContextClosedListener.java | 57 +++++ .../node/listener/PdConfigureListener.java | 75 ++++-- .../store/node/metrics/DriveMetrics.java | 58 +++++ .../store/node/metrics/GRpcExMetrics.java | 94 +++++++ .../store/node/metrics/JRaftMetrics.java | 138 +++++++---- .../store/node/metrics/MetricsConfig.java | 48 ++++ .../store/node/metrics/MetricsUtil.java | 11 +- .../store/node/metrics/ProcfsEntry.java | 8 +- .../store/node/metrics/ProcfsMetrics.java | 67 +++++ .../store/node/metrics/ProcfsReader.java | 15 +- .../store/node/metrics/ProcfsSmaps.java | 8 +- .../store/node/metrics/RocksDBMetrics.java | 173 ++++++++----- .../node/metrics/RocksDBMetricsConst.java | 27 +- .../store/node/metrics/StoreMetrics.java | 41 +++- .../store/node/metrics/SystemMetrics.java | 31 +-- .../store/node/model/HgNodeStatus.java | 25 +- .../hugegraph/store/node/util/Base58.java | 29 ++- .../apache/hugegraph/store/node/util/Err.java | 40 +++ .../hugegraph/store/node/util/HgAssert.java | 36 ++- .../store/node/util/HgBufferProxy.java | 36 ++- .../hugegraph/store/node/util/HgChannel.java | 22 +- .../store/node/util/HgExecutorUtil.java | 25 +- .../hugegraph/store/node/util/HgGrpc.java | 33 ++- .../store/node/util/HgRegexUtil.java | 23 +- .../store/node/util/HgStoreConst.java | 39 +++ .../store/node/util/HgStoreNodeUtil.java | 23 +- .../store/node/util/PropertyUtil.java | 27 +- .../hugegraph/store/node/util/Result.java | 45 ++++ .../hugegraph/store/node/util/TkEntry.java | 65 +++++ .../src/main/resources/log4j2-dev.xml | 37 ++- .../store/boot/StoreNodeServer00.java | 32 --- .../store/boot/StoreNodeServer01.java | 35 --- .../store/boot/StoreNodeServer03.java | 35 --- .../store/boot/StoreNodeServer04.java | 23 -- .../store/boot/StoreNodeServer05.java | 23 -- .../store/boot/StoreNodeServer06.java | 23 -- .../store/node/HgStoreNodeServiceTest.java | 55 ----- .../store/node/metrics/JraftMetricsTest.java | 44 ---- .../store/boot/StoreNodeServer00.java | 57 +++++ .../store/boot/StoreNodeServer01.java | 59 +++++ .../store/boot/StoreNodeServer02.java | 38 ++- .../store/boot/StoreNodeServer03.java | 57 +++++ .../store/boot/StoreNodeServer04.java | 41 ++++ .../store/boot/StoreNodeServer05.java | 41 ++++ .../store/boot/StoreNodeServer06.java | 41 ++++ .../store/node/HgStoreNodeServiceTest.java | 73 ++++++ .../store/node/metrics/JraftMetricsTest.java | 59 +++++ .../src/test/resources/log4j2-dev.xml | 37 ++- hg-store-rocksdb/pom.xml | 20 +- .../rocksdb/access/DBSnapshotMeta.java | 39 --- .../rocksdb/access/ScanIterator.java | 32 --- .../rocksdb/access/util/Asserts.java | 26 -- .../rocksdb/access/DBSnapshotMeta.java | 56 +++++ .../rocksdb/access/DBStoreException.java | 19 +- .../rocksdb/access/RocksDBFactory.java | 19 +- .../rocksdb/access/RocksDBOptions.java | 19 +- .../rocksdb/access/RocksDBScanIterator.java | 21 +- .../rocksdb/access/RocksDBSession.java | 21 +- .../rocksdb/access/ScanIterator.java | 49 ++++ .../rocksdb/access/SessionOperator.java | 19 +- .../rocksdb/access/SessionOperatorImpl.java | 24 +- .../rocksdb/access/util/Asserts.java | 43 ++++ .../hugegraph/rocksdb/access/util/CRC64.java | 19 +- .../rocksdb/access/util/ZipUtils.java | 23 +- .../rocksdb/access/RocksDBFactoryTest.java | 21 +- .../rocksdb/access/RocksDBSessionTest.java | 32 ++- .../hugegraph/rocksdb/access/RocksdbDump.java | 19 +- .../rocksdb/access/SnapshotManagerTest.java | 29 ++- .../src/test/resources/hugegraph-2.properties | 16 ++ .../src/test/resources/hugegraph.properties | 16 ++ .../src/test/resources/log4j2.xml | 41 +++- hg-store-test/pom.xml | 17 ++ .../src/main/java/client/BaseClientTest.java | 17 ++ .../main/java/client/ChangeShardNumTest.java | 17 ++ .../src/main/java/client/ClientSuiteTest.java | 17 ++ .../src/main/java/client/ClientTest.java | 17 ++ .../src/main/java/client/HgAssertTest.java | 17 ++ .../src/main/java/client/HgKvStoreTest.java | 17 ++ .../src/main/java/client/HgPairTest.java | 17 ++ .../client/HgSessionManagerRaftPDTest.java | 17 ++ .../main/java/client/HgStoreClientTest.java | 17 ++ .../java/client/HgStoreNodeStateTest.java | 17 ++ .../client/graph/GraphStoreClientTest.java | 17 ++ .../src/main/java/common/BaseCommonTest.java | 17 ++ .../java/common/ByteBufferAllocatorTest.java | 17 ++ .../src/main/java/common/CommonSuiteTest.java | 17 ++ .../main/java/common/KVByteBufferTest.java | 17 ++ .../src/main/java/core/BaseCoreTest.java | 17 ++ .../src/main/java/core/CoreSuiteTest.java | 17 ++ .../src/main/java/core/CoreTest.java | 17 ++ .../main/java/core/HgBusinessImplTest.java | 23 +- .../src/main/java/core/HgCmdClientTest.java | 17 ++ .../src/main/java/core/MetricServiceTest.java | 17 ++ .../main/java/core/StoreEngineTestBase.java | 17 ++ .../core/raft/HgStoreStateMachineTest.java | 17 ++ .../main/java/core/raft/MockStateMachine.java | 17 ++ .../java/core/raft/RaftOperationTest.java | 17 ++ .../main/java/core/raft/RaftUtilsTest.java | 17 ++ .../core/snapshot/HgSnapshotHandlerTest.java | 17 ++ .../java/core/store/HgStoreEngineTest.java | 17 ++ .../java/core/store/PartitionEngineTest.java | 17 ++ .../PartitionInstructionProcessorTest.java | 17 ++ .../store/meta/MetadataKeyHelperTest.java | 17 ++ .../core/store/meta/PartitionManagerTest.java | 17 ++ .../java/core/store/meta/TaskManagerTest.java | 17 ++ .../store/meta/asynctask/CleanTaskTest.java | 17 ++ .../core/store/util/MiscUtilClassTest.java | 17 ++ .../util/PartitionMetaStoreWrapperTest.java | 17 ++ .../java/core/store/util/ZipUtilsTest.java | 17 ++ .../main/java/raftcore/BytesCarrierTest.java | 17 ++ .../src/main/java/raftcore/RaftSuiteTest.java | 17 ++ .../raftcore/ZeroByteStringHelperTest.java | 17 ++ .../main/java/rocksdb/BaseRocksDbTest.java | 23 +- .../main/java/rocksdb/RocksDBFactoryTest.java | 23 +- .../main/java/rocksdb/RocksDbSuiteTest.java | 17 ++ .../src/main/java/service/BaseServerTest.java | 17 ++ .../main/java/service/ServerSuiteTest.java | 17 ++ .../src/main/java/service/SessionTest.java | 17 ++ .../main/java/util/CopyOnWriteCacheTest.java | 17 ++ .../src/main/java/util/HgStoreTestUtil.java | 17 ++ .../src/main/java/util/UnitTestBase.java | 21 +- .../src/main/java/util/UnsafeUtilTest.java | 17 ++ 221 files changed, 5268 insertions(+), 2238 deletions(-) delete mode 100644 hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/AppConfig.java delete mode 100644 hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/util/HgMetricX.java create mode 100644 hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/AppConfig.java rename hg-store-cli/src/main/java/{com/baidu => org/apache}/hugegraph/store/cli/StoreConsoleApplication.java (64%) rename hg-store-cli/src/main/java/{com/baidu => org/apache}/hugegraph/store/cli/loader/HgThread2DB.java (70%) rename hg-store-cli/src/main/java/{com/baidu => org/apache}/hugegraph/store/cli/scan/GrpcShardScanner.java (86%) rename hg-store-cli/src/main/java/{com/baidu => org/apache}/hugegraph/store/cli/scan/HgStoreCommitter.java (51%) rename hg-store-cli/src/main/java/{com/baidu => org/apache}/hugegraph/store/cli/scan/HgStoreScanner.java (72%) rename hg-store-cli/src/main/java/{com/baidu => org/apache}/hugegraph/store/cli/util/HgCliUtil.java (76%) create mode 100644 hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgMetricX.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/AppShutdownHook.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/HgStoreMetricsController.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/entry/RestResult.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/EmptyIterator.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/GRpcServerConfig.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreNodeState.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/QueryCondition.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanOneShotResponse.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanQuery.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/listener/ContextClosedListener.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/DriveMetrics.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/GRpcExMetrics.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/MetricsConfig.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/ProcfsMetrics.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/Err.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgStoreConst.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/Result.java delete mode 100644 hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/TkEntry.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/AppConfig.java (86%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppShutdownHook.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/StoreNodeApplication.java (56%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgStoreMetricsController.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/controller/HgStoreStatusController.java (64%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/controller/HgTestController.java (71%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/controller/IndexAPI.java (61%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/controller/PartitionAPI.java (73%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/entry/RestResult.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/BatchGrpcClosure.java (70%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/BatchScanIterator.java (68%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/EmptyIterator.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/FusingScanIterator.java (69%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GRpcServerConfig.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/GrpcClosure.java (55%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/HgStoreNodeService.java (74%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeState.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/HgStoreSessionImpl.java (74%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/HgStoreStateService.java (59%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/HgStoreStateSubject.java (52%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/HgStoreStreamImpl.java (76%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/HgStoreWrapperEx.java (68%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/ParallelScanIterator.java (82%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/QueryCondition.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java (57%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/ScanBatchResponse.java (88%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/ScanBatchResponse3.java (73%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/ScanBatchResponseFactory.java (89%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanOneShotResponse.java create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanQuery.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/ScanQueryProducer.java (82%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/ScanStreamResponse.java (78%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/ScanUtil.java (56%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/scan/GraphStoreImpl.java (65%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/grpc/scan/ScanResponseObserver.java (84%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/ContextClosedListener.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/listener/PdConfigureListener.java (81%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/DriveMetrics.java create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/GRpcExMetrics.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/metrics/JRaftMetrics.java (63%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/MetricsConfig.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/metrics/MetricsUtil.java (86%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/metrics/ProcfsEntry.java (93%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsMetrics.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/metrics/ProcfsReader.java (90%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/metrics/ProcfsSmaps.java (94%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/metrics/RocksDBMetrics.java (59%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/metrics/RocksDBMetricsConst.java (84%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/metrics/StoreMetrics.java (62%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/metrics/SystemMetrics.java (87%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/model/HgNodeStatus.java (52%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/util/Base58.java (77%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Err.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/util/HgAssert.java (68%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/util/HgBufferProxy.java (72%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/util/HgChannel.java (78%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/util/HgExecutorUtil.java (70%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/util/HgGrpc.java (71%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/util/HgRegexUtil.java (65%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgStoreConst.java rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/util/HgStoreNodeUtil.java (62%) rename hg-store-node/src/main/java/{com/baidu => org/apache}/hugegraph/store/node/util/PropertyUtil.java (66%) create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Result.java create mode 100644 hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/TkEntry.java delete mode 100644 hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer00.java delete mode 100644 hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer01.java delete mode 100644 hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer03.java delete mode 100644 hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer04.java delete mode 100644 hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer05.java delete mode 100644 hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer06.java delete mode 100644 hg-store-node/src/test/java/com/baidu/hugegraph/store/node/HgStoreNodeServiceTest.java delete mode 100644 hg-store-node/src/test/java/com/baidu/hugegraph/store/node/metrics/JraftMetricsTest.java create mode 100644 hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer00.java create mode 100644 hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer01.java rename hg-store-node/src/test/java/{com/baidu => org/apache}/hugegraph/store/boot/StoreNodeServer02.java (50%) create mode 100644 hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer03.java create mode 100644 hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer04.java create mode 100644 hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer05.java create mode 100644 hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer06.java create mode 100644 hg-store-node/src/test/java/org/apache/hugegraph/store/node/HgStoreNodeServiceTest.java create mode 100644 hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/JraftMetricsTest.java delete mode 100644 hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/DBSnapshotMeta.java delete mode 100644 hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/ScanIterator.java delete mode 100644 hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/util/Asserts.java create mode 100644 hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/DBSnapshotMeta.java rename hg-store-rocksdb/src/main/java/{com/baidu => org/apache}/hugegraph/rocksdb/access/DBStoreException.java (59%) rename hg-store-rocksdb/src/main/java/{com/baidu => org/apache}/hugegraph/rocksdb/access/RocksDBFactory.java (92%) rename hg-store-rocksdb/src/main/java/{com/baidu => org/apache}/hugegraph/rocksdb/access/RocksDBOptions.java (95%) rename hg-store-rocksdb/src/main/java/{com/baidu => org/apache}/hugegraph/rocksdb/access/RocksDBScanIterator.java (88%) rename hg-store-rocksdb/src/main/java/{com/baidu => org/apache}/hugegraph/rocksdb/access/RocksDBSession.java (97%) create mode 100644 hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/ScanIterator.java rename hg-store-rocksdb/src/main/java/{com/baidu => org/apache}/hugegraph/rocksdb/access/SessionOperator.java (66%) rename hg-store-rocksdb/src/main/java/{com/baidu => org/apache}/hugegraph/rocksdb/access/SessionOperatorImpl.java (92%) create mode 100644 hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/util/Asserts.java rename hg-store-rocksdb/src/main/java/{com/baidu => org/apache}/hugegraph/rocksdb/access/util/CRC64.java (89%) rename hg-store-rocksdb/src/main/java/{com/baidu => org/apache}/hugegraph/rocksdb/access/util/ZipUtils.java (83%) rename hg-store-rocksdb/src/test/java/{com/baidu => org/apache}/hugegraph/rocksdb/access/RocksDBFactoryTest.java (72%) rename hg-store-rocksdb/src/test/java/{com/baidu => org/apache}/hugegraph/rocksdb/access/RocksDBSessionTest.java (93%) rename hg-store-rocksdb/src/test/java/{com/baidu => org/apache}/hugegraph/rocksdb/access/RocksdbDump.java (66%) rename hg-store-rocksdb/src/test/java/{com/baidu => org/apache}/hugegraph/rocksdb/access/SnapshotManagerTest.java (94%) diff --git a/hg-store-cli/deploy-snapshot.sh b/hg-store-cli/deploy-snapshot.sh index c17b7dc34..ce2277626 100644 --- a/hg-store-cli/deploy-snapshot.sh +++ b/hg-store-cli/deploy-snapshot.sh @@ -1,4 +1,21 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + readonly REPO_URL=http://10.14.139.8:8081/artifactory/star-snapshot mvn --settings ../settings.xml -Dmaven.test.skip=true -DaltDeploymentRepository=star-local::default::${REPO_URL} clean deploy diff --git a/hg-store-cli/pom.xml b/hg-store-cli/pom.xml index 62640a112..ed2d472b3 100644 --- a/hg-store-cli/pom.xml +++ b/hg-store-cli/pom.xml @@ -1,4 +1,21 @@ + + @@ -51,7 +68,7 @@ - com.baidu.hugegraph.store.cli.StoreConsoleApplication + org.apache.hugegraph.store.cli.StoreConsoleApplication diff --git a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/AppConfig.java b/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/AppConfig.java deleted file mode 100644 index cf795c8fe..000000000 --- a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/AppConfig.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.baidu.hugegraph.store.cli; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -/** - * @author lynn.bond@hotmail.com on 2022/2/15 - */ -@Data -@Component -public class AppConfig { - - @Value("${pd.address}") - private String pdAddress; - - @Value("${net.kv.scanner.page.size}") - private int scannerPageSize; - - @Value("${scanner.graph}") - private String scannerGraph; - - @Value("${scanner.table}") - private String scannerTable; - - @Value("${scanner.max}") - private int scannerMax; - - @Value("${scanner.mod}") - private int scannerModNumber; - - @Value("${committer.graph}") - private String committerGraph; - - @Value("${committer.table}") - private String committerTable; - - @Value("${committer.amount}") - private int committerAmount; -} diff --git a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/util/HgMetricX.java b/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/util/HgMetricX.java deleted file mode 100644 index 6e610bed6..000000000 --- a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/util/HgMetricX.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.baidu.hugegraph.store.cli.util; - -import java.util.concurrent.atomic.AtomicLong; - -/** - * @author lynn.bond@hotmail.com on 2022/1/29 - */ -public class HgMetricX { - private long start; - private long end; - - private long waitStart = System.currentTimeMillis(); - private long waitTotal; - - public static HgMetricX ofStart() { - return new HgMetricX(System.currentTimeMillis()); - } - - private HgMetricX(long start) { - this.start = start; - } - - ; - - public long start() { - return this.start = System.currentTimeMillis(); - } - - public long end() { - return this.end = System.currentTimeMillis(); - } - - public long past() { - return this.end - this.start; - } - - public long getWaitTotal() { - return this.waitTotal; - } - - public void startWait() { - this.waitStart = System.currentTimeMillis(); - } - - public void appendWait() { - this.waitTotal += System.currentTimeMillis() - waitStart; - } -} diff --git a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/AppConfig.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/AppConfig.java new file mode 100644 index 000000000..4d728fe95 --- /dev/null +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/AppConfig.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cli; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import lombok.Data; + +/** + * @author lynn.bond@hotmail.com on 2022/2/15 + */ +@Data +@Component +public class AppConfig { + + @Value("${pd.address}") + private String pdAddress; + + @Value("${net.kv.scanner.page.size}") + private int scannerPageSize; + + @Value("${scanner.graph}") + private String scannerGraph; + + @Value("${scanner.table}") + private String scannerTable; + + @Value("${scanner.max}") + private int scannerMax; + + @Value("${scanner.mod}") + private int scannerModNumber; + + @Value("${committer.graph}") + private String committerGraph; + + @Value("${committer.table}") + private String committerTable; + + @Value("${committer.amount}") + private int committerAmount; +} diff --git a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/StoreConsoleApplication.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/StoreConsoleApplication.java similarity index 64% rename from hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/StoreConsoleApplication.java rename to hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/StoreConsoleApplication.java index db6fc9e6d..362f9ff6b 100644 --- a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/StoreConsoleApplication.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/StoreConsoleApplication.java @@ -1,21 +1,38 @@ -package com.baidu.hugegraph.store.cli; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.pd.client.PDClient; -import com.baidu.hugegraph.pd.client.PDConfig; -import com.baidu.hugegraph.pd.common.PDException; -import com.baidu.hugegraph.store.HgStoreClient; -import com.baidu.hugegraph.store.cli.loader.HgThread2DB; -import com.baidu.hugegraph.store.cli.scan.GrpcShardScanner; -import com.baidu.hugegraph.store.cli.scan.HgStoreCommitter; -import com.baidu.hugegraph.store.cli.scan.HgStoreScanner; +package org.apache.hugegraph.store.cli; -import lombok.extern.slf4j.Slf4j; +import java.io.IOException; + +import org.apache.hugegraph.store.cli.loader.HgThread2DB; +import org.apache.hugegraph.store.cli.scan.GrpcShardScanner; +import org.apache.hugegraph.store.cli.scan.HgStoreCommitter; +import org.apache.hugegraph.store.cli.scan.HgStoreScanner; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import java.io.IOException; +import com.baidu.hugegraph.pd.client.PDConfig; +import com.baidu.hugegraph.pd.common.PDException; +import com.baidu.hugegraph.store.HgStoreClient; + +import lombok.extern.slf4j.Slf4j; /** @@ -36,11 +53,11 @@ public class StoreConsoleApplication implements CommandLineRunner { @Override public void run(String... args) throws IOException, InterruptedException, PDException { - if (args.length <= 0){ + if (args.length <= 0) { // HgThread2DB hB = new HgThread2DB("localhost:8686"); // hB.startMultiprocessQuery("12", "10"); System.out.println("参数类型 cmd[-load, -query, -scan]"); - }else { + } else { switch (args[0]) { case "-load": HgThread2DB hgThread2DB = new HgThread2DB(args[1]); @@ -48,9 +65,9 @@ public class StoreConsoleApplication implements CommandLineRunner { hgThread2DB.setGraphName(args[3]); } try { - if (args[2].equals("order")){ + if (args[2].equals("order")) { hgThread2DB.testOrder(args[4]); - }else { + } else { hgThread2DB.startMultiprocessInsert(args[2]); } } catch (IOException e) { @@ -66,9 +83,9 @@ public class StoreConsoleApplication implements CommandLineRunner { } break; case "-scan": - if ( args.length < 4){ + if (args.length < 4) { System.out.println("参数类型 -scan pd graphName tableName"); - }else { + } else { doScan(args[1], args[2], args[3]); } break; @@ -86,17 +103,17 @@ public class StoreConsoleApplication implements CommandLineRunner { } } - private void doCommit(){ - HgStoreCommitter committer=HgStoreCommitter.of(appConfig.getCommitterGraph()); - committer.put(appConfig.getScannerTable(),appConfig.getCommitterAmount()); + private void doCommit() { + HgStoreCommitter committer = HgStoreCommitter.of(appConfig.getCommitterGraph()); + committer.put(appConfig.getScannerTable(), appConfig.getCommitterAmount()); } private void doScan(String pd, String graphName, String tableName) throws PDException { HgStoreClient storeClient = HgStoreClient.create(PDConfig.of(pd) - .setEnableCache(true)); + .setEnableCache(true)); HgStoreScanner storeScanner = HgStoreScanner.of(storeClient, graphName); storeScanner.scanTable2(tableName); - // storeScanner.scanHash(); + // storeScanner.scanHash(); } } diff --git a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/loader/HgThread2DB.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/loader/HgThread2DB.java similarity index 70% rename from hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/loader/HgThread2DB.java rename to hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/loader/HgThread2DB.java index c374f3e02..0b75c308b 100644 --- a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/loader/HgThread2DB.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/loader/HgThread2DB.java @@ -1,29 +1,33 @@ -package com.baidu.hugegraph.store.cli.loader; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.pd.client.PDClient; -import com.baidu.hugegraph.pd.client.PDConfig; -import com.baidu.hugegraph.store.HgStoreClient; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgStoreSession; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgScanQuery; -import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator; -import com.baidu.hugegraph.store.client.util.MetricX; -import com.baidu.hugegraph.store.cli.util.HgCliUtil; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.cli.loader; -import java.io.IOException; +import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStreamReader; -import java.io.BufferedReader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.Iterator; import java.util.List; - import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.LinkedBlockingQueue; @@ -32,6 +36,21 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import org.apache.hugegraph.store.cli.util.HgCliUtil; + +import com.baidu.hugegraph.pd.client.PDClient; +import com.baidu.hugegraph.pd.client.PDConfig; +import com.baidu.hugegraph.store.HgKvEntry; +import com.baidu.hugegraph.store.HgKvIterator; +import com.baidu.hugegraph.store.HgOwnerKey; +import com.baidu.hugegraph.store.HgScanQuery; +import com.baidu.hugegraph.store.HgStoreClient; +import com.baidu.hugegraph.store.HgStoreSession; +import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator; +import com.baidu.hugegraph.store.client.util.MetricX; + +import lombok.extern.slf4j.Slf4j; + /** * 使用pd，支持raft @@ -42,35 +61,37 @@ public class HgThread2DB { private static PDClient pdClient; public String graphName = "hugegraphtest"; /*正在进行和在排队的任务的总数*/ - private static AtomicInteger taskTotal = new AtomicInteger(0); - private static AtomicInteger queryTaskTotal = new AtomicInteger(0); + private static final AtomicInteger taskTotal = new AtomicInteger(0); + private static final AtomicInteger queryTaskTotal = new AtomicInteger(0); private static ThreadPoolExecutor threadPool = null; private static ThreadPoolExecutor queryThreadPool = null; - private static AtomicLong insertDataCount = new AtomicLong(); - private static AtomicLong queryCount = new AtomicLong(); - private static AtomicLong totalQueryCount = new AtomicLong(); - volatile long startTime = System.currentTimeMillis(); + private static final AtomicLong insertDataCount = new AtomicLong(); + private static final AtomicLong queryCount = new AtomicLong(); + private static final AtomicLong totalQueryCount = new AtomicLong(); + volatile long startTime = System.currentTimeMillis(); private static int limitScanBatchCount = 100; private static ArrayBlockingQueue listQueue = null; - private static AtomicLong longId = new AtomicLong(); + private static final AtomicLong longId = new AtomicLong(); - private static CountDownLatch countDownLatch = null; - private HgStoreClient storeClient; + private static final CountDownLatch countDownLatch = null; + private final HgStoreClient storeClient; public HgThread2DB(String pdAddr) { int threadCount = Runtime.getRuntime().availableProcessors(); listQueue = new ArrayBlockingQueue>(100000000); queryThreadPool = new ThreadPoolExecutor(500, 1000, - 200, TimeUnit.SECONDS, new ArrayBlockingQueue( 1000)); - threadPool = new ThreadPoolExecutor(threadCount*2, threadCount * 3, - 200, TimeUnit.SECONDS, new ArrayBlockingQueue(threadCount + 100)); + 200, TimeUnit.SECONDS, + new ArrayBlockingQueue(1000)); + threadPool = new ThreadPoolExecutor(threadCount * 2, threadCount * 3, + 200, TimeUnit.SECONDS, + new ArrayBlockingQueue(threadCount + 100)); storeClient = HgStoreClient.create(PDConfig.of(pdAddr) - .setEnableCache(true)); + .setEnableCache(true)); pdClient = storeClient.getPdClient(); } - public void setGraphName(String graphName){ + public void setGraphName(String graphName) { this.graphName = graphName; log.info("setGraphName {}", graphName); } @@ -89,13 +110,14 @@ public class HgThread2DB { session.put(tableName, hgKey, value); }); - if ( insertDataCount.get() > 10000000){ - synchronized (insertDataCount){ + if (insertDataCount.get() > 10000000) { + synchronized (insertDataCount) { long count = insertDataCount.get(); insertDataCount.set(0); - if (count > 10000000){ - log.info("count : " + count + " qps : " + count * 1000 / (System.currentTimeMillis() - startTime) - +" threadCount : " + taskTotal); + if (count > 10000000) { + log.info("count : " + count + " qps : " + + count * 1000 / (System.currentTimeMillis() - startTime) + + " threadCount : " + taskTotal); startTime = System.currentTimeMillis(); } } @@ -120,20 +142,21 @@ public class HgThread2DB { for (int y = 0; y < maxlist; y++) { insertDataCount.getAndIncrement(); - String strLine = "" + getLong() + getLong() + getLong() + getLong(); + String strLine = getLong() + getLong() + getLong() + getLong(); // log.info("========{}",strLine); HgOwnerKey hgKey = HgCliUtil.toOwnerKey(strLine, strLine); byte[] value = HgCliUtil.toBytes(strLine); session.put(tableName, hgKey, value); } - if ( insertDataCount.get() > 10000000){ - synchronized (insertDataCount){ + if (insertDataCount.get() > 10000000) { + synchronized (insertDataCount) { long count = insertDataCount.get(); insertDataCount.set(0); - if (count > 10000000){ - log.info("count : " + count + " qps : " + count * 1000 / (System.currentTimeMillis() - startTime) - +" threadCount : " + taskTotal); + if (count > 10000000) { + log.info("count : " + count + " qps : " + + count * 1000 / (System.currentTimeMillis() - startTime) + + " threadCount : " + taskTotal); startTime = System.currentTimeMillis(); } } @@ -154,12 +177,13 @@ public class HgThread2DB { HgStoreSession session = storeClient.openSession(graphName); session.beginTx(); int loop = Integer.parseInt(input); - if (loop == 0){ + if (loop == 0) { loop = 2000; } for (int i = 0; i < loop; i++) { long startTime = System.currentTimeMillis(); - HgOwnerKey hgOwnerKey = HgCliUtil.toOwnerKey(startTime + "owner:" + i, startTime + "k:" + i); + HgOwnerKey hgOwnerKey = + HgCliUtil.toOwnerKey(startTime + "owner:" + i, startTime + "k:" + i); session.put(tableName, hgOwnerKey, HgCliUtil.toBytes(i)); } @@ -189,6 +213,7 @@ public class HgThread2DB { /** * 多线程读取文件入库 + * * @throws IOException * @throws InterruptedException */ @@ -200,7 +225,8 @@ public class HgThread2DB { long dataCount = 0; if (readfile.exists()) { // 读取文件 - InputStreamReader isr = new InputStreamReader(new FileInputStream(readfile), "UTF-8"); + InputStreamReader isr = new InputStreamReader(new FileInputStream(readfile), + StandardCharsets.UTF_8); BufferedReader reader = new BufferedReader(isr); String strLine = null; @@ -227,7 +253,7 @@ public class HgThread2DB { e.printStackTrace(); } taskTotal.decrementAndGet(); - synchronized (taskTotal){ + synchronized (taskTotal) { taskTotal.notifyAll(); } } @@ -235,8 +261,8 @@ public class HgThread2DB { taskTotal.getAndIncrement(); threadPool.execute(task); - while ( taskTotal.get() > 100){ - synchronized (taskTotal){ + while (taskTotal.get() > 100) { + synchronized (taskTotal) { taskTotal.wait(); } } @@ -251,7 +277,7 @@ public class HgThread2DB { isr.close(); reader.close(); // 把剩余的入库 - if (!keys.isEmpty()){ + if (!keys.isEmpty()) { List finalKeys1 = keys; Runnable task = new Runnable() { public void run() { @@ -261,7 +287,7 @@ public class HgThread2DB { e.printStackTrace(); } taskTotal.decrementAndGet(); - synchronized (taskTotal){ + synchronized (taskTotal) { taskTotal.notifyAll(); } } @@ -269,11 +295,11 @@ public class HgThread2DB { threadPool.execute(task); taskTotal.getAndIncrement(); } - while ( taskTotal.get() > 0){ - synchronized (taskTotal){ + while (taskTotal.get() > 0) { + synchronized (taskTotal) { try { taskTotal.wait(1000); - if ( taskTotal.get() > 0){ + if (taskTotal.get() > 0) { System.out.println("wait thread exit " + taskTotal.get()); } } catch (InterruptedException e) { @@ -284,7 +310,7 @@ public class HgThread2DB { threadPool.shutdown(); - }else { + } else { System.out.println("样本文件不存在：" + filepath); } metrics.end(); @@ -297,6 +323,7 @@ public class HgThread2DB { /** * 多线程读取文件入库 + * * @throws IOException * @throws InterruptedException */ @@ -342,11 +369,11 @@ public class HgThread2DB { } } - while ( taskTotal.get() > 0){ - synchronized (taskTotal){ + while (taskTotal.get() > 0) { + synchronized (taskTotal) { try { taskTotal.wait(1000); - if ( taskTotal.get() > 0){ + if (taskTotal.get() > 0) { System.out.println("wait thread exit " + taskTotal.get()); } } catch (InterruptedException e) { @@ -383,7 +410,7 @@ public class HgThread2DB { while (!listQueue.isEmpty()) { log.info(" ====== start scanBatch2 count:{} list:{}=============", - queryThreadPool.getActiveCount(), listQueue.size()); + queryThreadPool.getActiveCount(), listQueue.size()); List keys = (List) listQueue.take(); List newQueryList = new ArrayList<>(); @@ -413,9 +440,11 @@ public class HgThread2DB { queryCount.set(0); if (count > 1000000) { log.info("count : " + count + " qps : " + count * 1000 / - (System.currentTimeMillis() - startTime) - + " threadCount : " + queryThreadPool.getActiveCount() + " queueSize:" - + listQueue.size()); + (System.currentTimeMillis() - + startTime) + + " threadCount : " + + queryThreadPool.getActiveCount() + " queueSize:" + + listQueue.size()); startTime = System.currentTimeMillis(); } } @@ -448,12 +477,14 @@ public class HgThread2DB { /** * 多线程查询 - * @param point 起始查询点，后续根据这个点查询到的value做为下一次的查询条件进行迭代 + * + * @param point 起始查询点，后续根据这个点查询到的value做为下一次的查询条件进行迭代 * @param scanCount 允许启动的线程数量 * @throws IOException * @throws InterruptedException */ - public void startMultiprocessQuery(String point, String scanCount) throws IOException, InterruptedException { + public void startMultiprocessQuery(String point, String scanCount) throws IOException, + InterruptedException { log.info("--- start startMultiprocessQuery---"); startTime = System.currentTimeMillis(); MetricX metrics = MetricX.ofStart(); @@ -466,69 +497,79 @@ public class HgThread2DB { final long[] start = {System.currentTimeMillis()}; LinkedBlockingQueue[] queue = new LinkedBlockingQueue[limitScanBatchCount]; - for(int i =0; i strKey = Arrays.asList(new String[]{"20727483", "50329304", "26199460", "1177521", "27960125", - "30440025", "15833920", "15015183", "33153097", "21250581"}); - strKey.forEach(key->{ + List strKey = Arrays.asList( + "20727483", "50329304", "26199460", "1177521", "27960125", + "30440025", "15833920", "15015183", "33153097", "21250581"); + strKey.forEach(key -> { log.info("newkey:{}", key); HgOwnerKey hgKey = HgCliUtil.toOwnerKey(key, key); queue[0].add(hgKey); }); - for(int i =0; i> iterators = session.scanBatch2( - HgScanQuery.prefixIteratorOf(HgCliUtil.TABLE_NAME, new Iterator() { - HgOwnerKey current = null; + HgScanQuery.prefixIteratorOf(HgCliUtil.TABLE_NAME, + new Iterator() { + HgOwnerKey current = null; - @Override - public boolean hasNext() { - while (current == null) { - try { - current = (HgOwnerKey) queue[finalI].poll(1, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // - } - } - if (current == null){ - log.info("===== current is null =========="); - } - return current != null; - } + @Override + public boolean hasNext() { + while (current == null) { + try { + current = + (HgOwnerKey) queue[finalI].poll( + 1, + TimeUnit.SECONDS); + } catch ( + InterruptedException e) { + // + } + } + if (current == null) { + log.info( + "===== current is " + + "null =========="); + } + return current != null; + } - @Override - public HgOwnerKey next() { - return current; - } - }) + @Override + public HgOwnerKey next() { + return current; + } + }) ); - new Thread(()->{ + new Thread(() -> { while (iterators.hasNext()) { HgKvIterator iterator = iterators.next(); long c = 0; while (iterator.hasNext()) { String newPoint = HgCliUtil.toStr(iterator.next().value()); HgOwnerKey newHgKey = HgCliUtil.toOwnerKey(newPoint, newPoint); - if ( queue[(int) (c % limitScanBatchCount)].size() < 1000000) { + if (queue[(int) (c % limitScanBatchCount)].size() < 1000000) { queue[(int) (c % limitScanBatchCount)].add(newHgKey); } c++; } - if ( counter[0].addAndGet(c) > 1000000){ + if (counter[0].addAndGet(c) > 1000000) { synchronized (counter) { - if ( counter[0].get() > 10000000) { - log.info("count {}, qps {}", counter[0].get(), counter[0].get() * 1000 / (System.currentTimeMillis() - start[0])); + if (counter[0].get() > 10000000) { + log.info("count {}, qps {}", counter[0].get(), + counter[0].get() * 1000 / + (System.currentTimeMillis() - start[0])); start[0] = System.currentTimeMillis(); counter[0].set(0); } } } } - }, "client query thread:" + i ).start(); + }, "client query thread:" + i).start(); log.info("===== read thread exit =========="); } latch.await(); @@ -537,7 +578,7 @@ public class HgThread2DB { metrics.end(); log.info("*************************************************"); log.info(" 主进程执行时间:" + metrics.past() / 1000 + "秒; 查询：" + totalQueryCount.get() - + "次，qps:" + totalQueryCount.get() * 1000 / metrics.past()); + + "次，qps:" + totalQueryCount.get() * 1000 / metrics.past()); log.info("*************************************************"); System.out.println("-----主进程执行结束---------"); } diff --git a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/scan/GrpcShardScanner.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/GrpcShardScanner.java similarity index 86% rename from hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/scan/GrpcShardScanner.java rename to hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/GrpcShardScanner.java index 456d59bac..d29c0b506 100644 --- a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/scan/GrpcShardScanner.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/GrpcShardScanner.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.cli.scan; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cli.scan; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; @@ -26,9 +43,9 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class GrpcShardScanner { - private volatile boolean closed = false; - private AtomicInteger sum = new AtomicInteger(); - private ConcurrentHashMap> + private final boolean closed = false; + private final AtomicInteger sum = new AtomicInteger(); + private final ConcurrentHashMap> observers = new ConcurrentHashMap<>(); public void getData() { @@ -137,8 +154,6 @@ public class GrpcShardScanner { observer.onNext(builder.build()); } catch (Exception e) { e.printStackTrace(); - } finally { - } } diff --git a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/scan/HgStoreCommitter.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreCommitter.java similarity index 51% rename from hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/scan/HgStoreCommitter.java rename to hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreCommitter.java index 2563681df..0189aa752 100644 --- a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/scan/HgStoreCommitter.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreCommitter.java @@ -1,12 +1,29 @@ -package com.baidu.hugegraph.store.cli.scan; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cli.scan; + +import org.apache.hugegraph.store.cli.util.HgCliUtil; import com.baidu.hugegraph.store.HgOwnerKey; import com.baidu.hugegraph.store.HgSessionManager; import com.baidu.hugegraph.store.HgStoreSession; import com.baidu.hugegraph.store.client.HgStoreNodeManager; -import static com.baidu.hugegraph.store.cli.util.HgCliUtil.*; - /** * @author lynn.bond@hotmail.com on 2022/2/28 */ @@ -33,7 +50,7 @@ public class HgStoreCommitter { return HgSessionManager.getInstance().openSession(graphName); } - public void put(String tableName,int amount) { + public void put(String tableName, int amount) { //*************** Put Benchmark **************//* String keyPrefix = "PUT-BENCHMARK"; HgStoreSession session = getStoreSession(); @@ -44,16 +61,19 @@ public class HgStoreCommitter { long start = System.currentTimeMillis(); for (int i = 0; i < amount; i++) { - HgOwnerKey key = toOwnerKey(keyPrefix + "-" + padLeftZeros(String.valueOf(i), length)); - byte[] value = toBytes(keyPrefix + "-V-" + i); + HgOwnerKey key = HgCliUtil.toOwnerKey( + keyPrefix + "-" + HgCliUtil.padLeftZeros(String.valueOf(i), length)); + byte[] value = HgCliUtil.toBytes(keyPrefix + "-V-" + i); session.put(tableName, key, value); if ((i + 1) % 100_000 == 0) { - println("---------- " + (i + 1) + " --------"); - println("Preparing took: " + (System.currentTimeMillis() - start) + " ms."); + HgCliUtil.println("---------- " + (i + 1) + " --------"); + HgCliUtil.println( + "Preparing took: " + (System.currentTimeMillis() - start) + " ms."); session.commit(); - println("Committing took: " + (System.currentTimeMillis() - start) + " ms."); + HgCliUtil.println( + "Committing took: " + (System.currentTimeMillis() - start) + " ms."); start = System.currentTimeMillis(); session.beginTx(); } diff --git a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/scan/HgStoreScanner.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreScanner.java similarity index 72% rename from hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/scan/HgStoreScanner.java rename to hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreScanner.java index 5fe2433a5..d80de3696 100644 --- a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/scan/HgStoreScanner.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreScanner.java @@ -1,18 +1,43 @@ -package com.baidu.hugegraph.store.cli.scan; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cli.scan; + +import java.util.Arrays; +import java.util.List; + +import org.apache.hugegraph.store.cli.util.HgCliUtil; +import org.apache.hugegraph.store.cli.util.HgMetricX; import com.baidu.hugegraph.pd.client.PDClient; import com.baidu.hugegraph.pd.common.PDException; import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.*; -import com.baidu.hugegraph.store.cli.util.HgCliUtil; -import com.baidu.hugegraph.store.cli.util.HgMetricX; +import com.baidu.hugegraph.store.HgKvEntry; +import com.baidu.hugegraph.store.HgKvIterator; +import com.baidu.hugegraph.store.HgKvStore; +import com.baidu.hugegraph.store.HgScanQuery; +import com.baidu.hugegraph.store.HgSessionManager; +import com.baidu.hugegraph.store.HgStoreClient; +import com.baidu.hugegraph.store.HgStoreSession; import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator; import com.baidu.hugegraph.store.client.util.HgStoreClientConfig; import com.baidu.hugegraph.store.client.util.MetricX; -import lombok.extern.slf4j.Slf4j; -import java.util.Arrays; -import java.util.List; +import lombok.extern.slf4j.Slf4j; /** * @author lynn.bond@hotmail.com on 2022/2/14 @@ -20,7 +45,7 @@ import java.util.List; @Slf4j public class HgStoreScanner { - private HgStoreClient storeClient; + private final HgStoreClient storeClient; private final String graphName; private long modNumber = 1_000_000; private int max = 10_000_000; @@ -69,7 +94,8 @@ public class HgStoreScanner { HgMetricX hgMetricX = HgMetricX.ofStart(); HgStoreSession session = getStoreSession(); int count = 0; - KvCloseableIterator> iterator = session.scanBatch2(HgScanQuery.tableOf(tableName)); + KvCloseableIterator> iterator = + session.scanBatch2(HgScanQuery.tableOf(tableName)); long start = System.currentTimeMillis(); while (iterator.hasNext()) { @@ -80,7 +106,9 @@ public class HgStoreScanner { iterator2.next(); if (count % (modNumber) == 0) { log.info("Scanning keys: " + count + " time is " + modNumber * 1000 - / (System.currentTimeMillis() - start)); + / + (System.currentTimeMillis() - + start)); start = System.currentTimeMillis(); } if (count == max) { @@ -112,13 +140,18 @@ public class HgStoreScanner { String graphName = "/DEFAULT/graphs/hugegraph1/"; HgStoreSession session = getStoreSession(graphName); int count = 0; - String query = "{\"conditions\":[{\"cls\":\"S\",\"el\":{\"key\":\"ID\",\"relation\":\"SCAN\",\"value\"" + - ":{\"start\":\"61180\",\"end\":\"63365\",\"length\":0}}}],\"optimizedType\":\"NONE\",\"ids\":[]," + + String query = + "{\"conditions\":[{\"cls\":\"S\",\"el\":{\"key\":\"ID\",\"relation\":\"SCAN\"," + + "\"value\"" + + ":{\"start\":\"61180\",\"end\":\"63365\",\"length\":0}}}]," + + "\"optimizedType\":\"NONE\",\"ids\":[]," + "\"mustSortByInput\":true,\"resultType\":\"EDGE\",\"offset\":0," + "\"actualOffset\":0,\"actualStoreOffset\":" + - "0,\"limit\":9223372036854775807,\"capacity\":-1,\"showHidden\":false,\"showDeleting\":false," + + "0,\"limit\":9223372036854775807,\"capacity\":-1,\"showHidden\":false," + + "\"showDeleting\":false," + "\"showExpired\":false,\"olap\":false,\"withProperties\":false,\"olapPks\":[]}"; - //HgKvIterator iterator = session.scanIterator(tableName,0,715827883, HgKvStore.SCAN_ANY,null); + //HgKvIterator iterator = session.scanIterator(tableName,0,715827883, + // HgKvStore.SCAN_ANY,null); //HgKvIterator iterator = session.scanIterator(tableName,61180,63365, 348, null); //HgKvIterator iterator = session.scanIterator(tableName,0,65535, 348, null); @@ -154,7 +187,8 @@ public class HgStoreScanner { public static final byte[] EMPTY_BYTES = new byte[0]; public void scanTable2(String tableName) throws PDException { - // java -jar hg-store-cli-3.6.0-SNAPSHOT.jar -scan 10.45.30.212:8989 "DEFAULT/case_112/g" g+ie + // java -jar hg-store-cli-3.6.0-SNAPSHOT.jar -scan 10.45.30.212:8989 "DEFAULT/case_112/g" + // g+ie PDClient pdClient = storeClient.getPdClient(); List partitions = pdClient.getPartitions(0, graphName); HgStoreSession session = storeClient.openSession(graphName); @@ -164,8 +198,10 @@ public class HgStoreScanner { for (Metapb.Partition partition : partitions) { while (true) { try (HgKvIterator iterator = session.scanIterator(tableName, - (int) (partition.getStartKey()), (int) (partition.getEndKey()), - HgKvStore.SCAN_HASHCODE, EMPTY_BYTES)) { + (int) (partition.getStartKey()), + (int) (partition.getEndKey()), + HgKvStore.SCAN_HASHCODE, + EMPTY_BYTES)) { if (position != null) { iterator.seek(position); } diff --git a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/util/HgCliUtil.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgCliUtil.java similarity index 76% rename from hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/util/HgCliUtil.java rename to hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgCliUtil.java index 9358f807c..38e2256ec 100644 --- a/hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/util/HgCliUtil.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgCliUtil.java @@ -1,15 +1,36 @@ -package com.baidu.hugegraph.store.cli.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cli.util; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; import com.baidu.hugegraph.store.HgKvEntry; import com.baidu.hugegraph.store.HgOwnerKey; import com.baidu.hugegraph.store.HgStoreSession; import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.*; -import java.util.function.Function; - /** * @author lynn.bond@hotmail.com on 2022/2/14 @@ -23,27 +44,28 @@ public class HgCliUtil { return batchPut(session, keyPrefix, 100); } - public static Map batchPut(HgStoreSession session, String keyPrefix, int loop) { + public static Map batchPut(HgStoreSession session, String keyPrefix, + int loop) { return batchPut(session, TABLE_NAME, keyPrefix, loop); } public static Map batchPut(HgStoreSession session, String tableName , String keyPrefix, int loop) { - return batchPut(session, tableName, keyPrefix, loop,1, key -> toOwnerKey(key)); + return batchPut(session, tableName, keyPrefix, loop, 1, key -> toOwnerKey(key)); } public static Map batchPut(HgStoreSession session, String tableName - , String keyPrefix, int loop,int start) { - return batchPut(session, tableName, keyPrefix, loop,start, key -> toOwnerKey(key)); + , String keyPrefix, int loop, int start) { + return batchPut(session, tableName, keyPrefix, loop, start, key -> toOwnerKey(key)); } public static Map batchPut(HgStoreSession session, String tableName - , String keyPrefix, int loop, Function f){ - return batchPut(session,tableName,keyPrefix,loop,1,f); + , String keyPrefix, int loop, Function f) { + return batchPut(session, tableName, keyPrefix, loop, 1, f); } public static Map batchPut(HgStoreSession session, String tableName - , String keyPrefix, int loop, int start,Function f) { + , String keyPrefix, int loop, int start, Function f) { Map res = new LinkedHashMap<>(); @@ -59,7 +81,7 @@ public class HgCliUtil { session.put(tableName, key, value); if ((i + 1) % 10000 == 0) { - println("commit: " + (i+1)); + println("commit: " + (i + 1)); session.commit(); session.beginTx(); } diff --git a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgMetricX.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgMetricX.java new file mode 100644 index 000000000..c4b989a55 --- /dev/null +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgMetricX.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cli.util; + +/** + * @author lynn.bond@hotmail.com on 2022/1/29 + */ +public class HgMetricX { + private long start; + private long end; + + private long waitStart = System.currentTimeMillis(); + private long waitTotal; + + public static HgMetricX ofStart() { + return new HgMetricX(System.currentTimeMillis()); + } + + private HgMetricX(long start) { + this.start = start; + } + + public long start() { + return this.start = System.currentTimeMillis(); + } + + public long end() { + return this.end = System.currentTimeMillis(); + } + + public long past() { + return this.end - this.start; + } + + public long getWaitTotal() { + return this.waitTotal; + } + + public void startWait() { + this.waitStart = System.currentTimeMillis(); + } + + public void appendWait() { + this.waitTotal += System.currentTimeMillis() - waitStart; + } +} diff --git a/hg-store-cli/src/main/resources/hg-store-client.properties b/hg-store-cli/src/main/resources/hg-store-client.properties index 61b464dc2..f72fbbb82 100644 --- a/hg-store-cli/src/main/resources/hg-store-client.properties +++ b/hg-store-cli/src/main/resources/hg-store-client.properties @@ -1,7 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# #grpc.timeout.seconds=10 #grpc.max.inbound.message.size= #grpc.max.outbound.message.size= net.kv.scanner.buffer.size=10000 -net.kv.scanner.page.size = 20000 +net.kv.scanner.page.size=20000 #Unit:second net.kv.scanner.have.next.timeout=1000 \ No newline at end of file diff --git a/hg-store-cli/src/main/resources/log4j2.xml b/hg-store-cli/src/main/resources/log4j2.xml index 1f393358f..050766fb3 100644 --- a/hg-store-cli/src/main/resources/log4j2.xml +++ b/hg-store-cli/src/main/resources/log4j2.xml @@ -1,5 +1,22 @@ + + @@ -28,7 +45,7 @@ - + @@ -36,8 +53,8 @@ - - + + diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/HgStoreEngine.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/HgStoreEngine.java index 83507bdf6..88df8a622 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/HgStoreEngine.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/HgStoreEngine.java @@ -11,7 +11,7 @@ import com.alipay.sofa.jraft.rpc.RpcServer; import com.alipay.sofa.jraft.util.Endpoint; import com.baidu.hugegraph.pd.common.PDException; import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; import com.baidu.hugegraph.store.business.DataMover; import com.baidu.hugegraph.store.business.BusinessHandler; import com.baidu.hugegraph.store.business.BusinessHandlerImpl; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/AbstractSelectIterator.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/AbstractSelectIterator.java index 2041b7563..4f15af3f5 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/AbstractSelectIterator.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/AbstractSelectIterator.java @@ -6,7 +6,7 @@ import com.baidu.hugegraph.backend.serializer.AbstractSerializer; import com.baidu.hugegraph.backend.serializer.BinarySerializer; import com.baidu.hugegraph.backend.store.BackendEntry; import com.baidu.hugegraph.iterator.CIter; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.structure.HugeElement; import com.baidu.hugegraph.util.Bytes; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandler.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandler.java index f4872d5a8..68dbe1603 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandler.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandler.java @@ -13,7 +13,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.baidu.hugegraph.pd.grpc.pulse.CleanType; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.store.grpc.Graphpb; import com.baidu.hugegraph.store.grpc.common.Key; import com.baidu.hugegraph.store.grpc.common.OpType; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandlerImpl.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandlerImpl.java index b53d81ad2..767f29e13 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandlerImpl.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandlerImpl.java @@ -27,13 +27,13 @@ import com.alipay.sofa.jraft.util.Utils; import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.config.OptionSpace; import com.baidu.hugegraph.pd.grpc.pulse.CleanType; -import com.baidu.hugegraph.rocksdb.access.DBStoreException; -import com.baidu.hugegraph.rocksdb.access.RocksDBFactory; -import com.baidu.hugegraph.rocksdb.access.RocksDBFactory.RocksdbChangedListener; -import com.baidu.hugegraph.rocksdb.access.RocksDBOptions; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; -import com.baidu.hugegraph.rocksdb.access.SessionOperator; +import org.apache.hugegraph.rocksdb.access.DBStoreException; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.RocksdbChangedListener; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.SessionOperator; import com.baidu.hugegraph.store.HgStoreEngine; import com.baidu.hugegraph.store.cmd.CleanDataRequest; import com.baidu.hugegraph.store.grpc.Graphpb.ScanPartitionRequest; @@ -89,7 +89,7 @@ public class BusinessHandlerImpl implements BusinessHandler { public static HugeConfig initRocksdb(Map rocksdbConfig, RocksdbChangedListener listener) { // 注册rocksdb配置 OptionSpace.register("rocksdb", - "com.baidu.hugegraph.rocksdb.access.RocksDBOptions"); + "org.apache.hugegraph.rocksdb.access.RocksDBOptions"); RocksDBOptions.instance(); HugeConfig hConfig = new HugeConfig(rocksdbConfig); factory.setHugeConfig(hConfig); diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/DefaultDataMover.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/DefaultDataMover.java index e5b69a87f..afc8d2ba7 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/DefaultDataMover.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/DefaultDataMover.java @@ -3,8 +3,8 @@ package com.baidu.hugegraph.store.business; import com.alipay.sofa.jraft.Status; import com.baidu.hugegraph.pd.grpc.Metapb; import com.baidu.hugegraph.pd.grpc.pulse.CleanType; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.store.cmd.BatchPutRequest; import com.baidu.hugegraph.store.cmd.BatchPutResponse; import com.baidu.hugegraph.store.cmd.CleanDataRequest; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/FilterIterator.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/FilterIterator.java index 9d91d096c..62aed5159 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/FilterIterator.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/FilterIterator.java @@ -5,8 +5,8 @@ import org.apache.commons.lang3.ArrayUtils; import com.baidu.hugegraph.backend.query.ConditionQuery; import com.baidu.hugegraph.backend.serializer.BinaryBackendEntry; import com.baidu.hugegraph.backend.store.BackendEntry; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; +import org.apache.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.structure.HugeElement; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/GraphStoreIterator.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/GraphStoreIterator.java index 602d639ed..83bb44cda 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/GraphStoreIterator.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/GraphStoreIterator.java @@ -20,8 +20,8 @@ import org.codehaus.groovy.jsr223.GroovyScriptEngineImpl; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.serializer.BinaryBackendEntry; import com.baidu.hugegraph.backend.store.BackendEntry; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; +import org.apache.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.schema.EdgeLabel; import com.baidu.hugegraph.schema.PropertyKey; import com.baidu.hugegraph.schema.VertexLabel; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/InnerKeyFilter.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/InnerKeyFilter.java index 609ec8c44..6a116fbe0 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/InnerKeyFilter.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/InnerKeyFilter.java @@ -1,7 +1,7 @@ package com.baidu.hugegraph.store.business; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; +import org.apache.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.store.term.Bits; import java.util.Arrays; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/MultiPartitionIterator.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/MultiPartitionIterator.java index c3c57e841..c979589f2 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/MultiPartitionIterator.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/MultiPartitionIterator.java @@ -1,6 +1,6 @@ package com.baidu.hugegraph.store.business; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.ScanIterator; import java.nio.ByteBuffer; import java.util.*; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/SelectIterator.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/SelectIterator.java index 9ebda1552..f0feb9744 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/SelectIterator.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/SelectIterator.java @@ -6,8 +6,8 @@ import java.util.Set; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.serializer.BytesBuffer; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; +import org.apache.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.type.define.DataType; import com.baidu.hugegraph.type.define.SerialEnum; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/DeletedFileManager.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/DeletedFileManager.java index f4491d9cc..75033d53b 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/DeletedFileManager.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/DeletedFileManager.java @@ -1,6 +1,6 @@ package com.baidu.hugegraph.store.meta; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; import com.baidu.hugegraph.store.meta.base.GlobalMetaStore; import com.baidu.hugegraph.store.options.MetadataOptions; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/DBSessionBuilder.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/DBSessionBuilder.java index 7358a2b3b..8d9edd452 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/DBSessionBuilder.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/DBSessionBuilder.java @@ -1,6 +1,6 @@ package com.baidu.hugegraph.store.meta.base; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; import com.baidu.hugegraph.store.util.HgStoreException; public interface DBSessionBuilder { diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/GlobalMetaStore.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/GlobalMetaStore.java index fc399e97b..bf067e562 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/GlobalMetaStore.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/GlobalMetaStore.java @@ -1,7 +1,7 @@ package com.baidu.hugegraph.store.meta.base; -import com.baidu.hugegraph.rocksdb.access.RocksDBFactory; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; import com.baidu.hugegraph.store.options.MetadataOptions; import java.util.Arrays; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/MetaStoreBase.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/MetaStoreBase.java index f1485c812..d5cac283c 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/MetaStoreBase.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/MetaStoreBase.java @@ -1,8 +1,8 @@ package com.baidu.hugegraph.store.meta.base; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; -import com.baidu.hugegraph.rocksdb.access.SessionOperator; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.SessionOperator; import com.baidu.hugegraph.store.util.HgStoreException; import com.baidu.hugegraph.store.util.Asserts; import com.google.protobuf.GeneratedMessageV3; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/PartitionMetaStore.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/PartitionMetaStore.java index 25f23c79e..a05698436 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/PartitionMetaStore.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/PartitionMetaStore.java @@ -1,6 +1,6 @@ package com.baidu.hugegraph.store.meta.base; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; /** * 元数据存储在分区的default cf中 diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/metric/SystemMetricService.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/metric/SystemMetricService.java index 1cae0bd36..043c6781a 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/metric/SystemMetricService.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/metric/SystemMetricService.java @@ -1,7 +1,7 @@ package com.baidu.hugegraph.store.metric; -import com.baidu.hugegraph.rocksdb.access.RocksDBFactory; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; import com.baidu.hugegraph.store.HgStoreEngine; import com.sun.management.OperatingSystemMXBean; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/RaftRocksdbOptions.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/RaftRocksdbOptions.java index 83f24cbf1..09fee92af 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/RaftRocksdbOptions.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/RaftRocksdbOptions.java @@ -3,7 +3,7 @@ package com.baidu.hugegraph.store.options; import com.alipay.sofa.jraft.storage.impl.RocksDBLogStorage; import com.alipay.sofa.jraft.util.StorageOptionsFactory; import com.baidu.hugegraph.config.HugeConfig; -import com.baidu.hugegraph.rocksdb.access.RocksDBOptions; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; import com.baidu.hugegraph.store.business.BusinessHandlerImpl; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/PartitionMetaStoreWrapper.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/PartitionMetaStoreWrapper.java index d67729d3a..29f21b6af 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/PartitionMetaStoreWrapper.java +++ b/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/PartitionMetaStoreWrapper.java @@ -1,6 +1,6 @@ package com.baidu.hugegraph.store.util; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; import com.baidu.hugegraph.store.HgStoreEngine; import com.baidu.hugegraph.store.meta.base.MetaStoreBase; diff --git a/hg-store-core/src/test/java/com/baidu/hugegraph/store/HgBusinessImplTest.java b/hg-store-core/src/test/java/com/baidu/hugegraph/store/HgBusinessImplTest.java index 38a599b43..9437959df 100644 --- a/hg-store-core/src/test/java/com/baidu/hugegraph/store/HgBusinessImplTest.java +++ b/hg-store-core/src/test/java/com/baidu/hugegraph/store/HgBusinessImplTest.java @@ -1,8 +1,8 @@ package com.baidu.hugegraph.store; -import com.baidu.hugegraph.rocksdb.access.RocksDBFactory; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.store.business.BusinessHandler; import com.baidu.hugegraph.store.business.BusinessHandlerImpl; import com.baidu.hugegraph.store.options.HgStoreEngineOptions; diff --git a/hg-store-core/src/test/java/com/baidu/hugegraph/store/UnitTestBase.java b/hg-store-core/src/test/java/com/baidu/hugegraph/store/UnitTestBase.java index 2ddede37c..4f0c99c3d 100644 --- a/hg-store-core/src/test/java/com/baidu/hugegraph/store/UnitTestBase.java +++ b/hg-store-core/src/test/java/com/baidu/hugegraph/store/UnitTestBase.java @@ -1,7 +1,7 @@ package com.baidu.hugegraph.store; -import com.baidu.hugegraph.rocksdb.access.RocksDBFactory; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; import com.baidu.hugegraph.store.business.BusinessHandler; import com.baidu.hugegraph.store.business.BusinessHandlerImpl; import com.baidu.hugegraph.store.meta.PartitionManager; diff --git a/hg-store-core/src/test/java/com/baidu/hugegraph/store/meta/GraphIDManagerTest.java b/hg-store-core/src/test/java/com/baidu/hugegraph/store/meta/GraphIDManagerTest.java index dfce2572b..82adf21e2 100644 --- a/hg-store-core/src/test/java/com/baidu/hugegraph/store/meta/GraphIDManagerTest.java +++ b/hg-store-core/src/test/java/com/baidu/hugegraph/store/meta/GraphIDManagerTest.java @@ -1,7 +1,7 @@ package com.baidu.hugegraph.store.meta; import com.baidu.hugegraph.pd.common.PDException; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; import com.baidu.hugegraph.store.UnitTestBase; import com.baidu.hugegraph.store.meta.base.DBSessionBuilder; import org.junit.Assert; diff --git a/hg-store-dist/pom.xml b/hg-store-dist/pom.xml index 866bb2278..647a253f4 100644 --- a/hg-store-dist/pom.xml +++ b/hg-store-dist/pom.xml @@ -1,4 +1,21 @@ + + diff --git a/hg-store-dist/src/assembly/descriptor/server-assembly.xml b/hg-store-dist/src/assembly/descriptor/server-assembly.xml index 61e37c02c..a5380934a 100644 --- a/hg-store-dist/src/assembly/descriptor/server-assembly.xml +++ b/hg-store-dist/src/assembly/descriptor/server-assembly.xml @@ -1,3 +1,20 @@ + + distribution false diff --git a/hg-store-dist/src/assembly/static/bin/restart-hugegraph-store.sh b/hg-store-dist/src/assembly/static/bin/restart-hugegraph-store.sh index 5a8582acb..085ddad43 100644 --- a/hg-store-dist/src/assembly/static/bin/restart-hugegraph-store.sh +++ b/hg-store-dist/src/assembly/static/bin/restart-hugegraph-store.sh @@ -1,5 +1,22 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + function abs_path() { SOURCE="${BASH_SOURCE[0]}" while [ -h "$SOURCE" ]; do diff --git a/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index ce5dea063..956f901af 100644 --- a/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -1,5 +1,22 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + function abs_path() { SOURCE="${BASH_SOURCE[0]}" while [ -h "$SOURCE" ]; do diff --git a/hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh b/hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh index e2039b5ff..e830b437d 100644 --- a/hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh +++ b/hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh @@ -1,5 +1,22 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + abs_path() { SOURCE="${BASH_SOURCE[0]}" while [ -h "$SOURCE" ]; do diff --git a/hg-store-dist/src/assembly/static/bin/util.sh b/hg-store-dist/src/assembly/static/bin/util.sh index 188d4bb54..ef09ed515 100644 --- a/hg-store-dist/src/assembly/static/bin/util.sh +++ b/hg-store-dist/src/assembly/static/bin/util.sh @@ -1,5 +1,22 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + function command_available() { local cmd=$1 if [ `command -v $cmd >/dev/null 2>&1` ]; then diff --git a/hg-store-dist/src/assembly/static/conf/log4j2.xml b/hg-store-dist/src/assembly/static/conf/log4j2.xml index 55710ab2f..b280a6d7b 100644 --- a/hg-store-dist/src/assembly/static/conf/log4j2.xml +++ b/hg-store-dist/src/assembly/static/conf/log4j2.xml @@ -1,5 +1,22 @@ + + @@ -27,7 +44,7 @@ - + @@ -35,8 +52,8 @@ - - + + @@ -54,7 +71,7 @@ - + @@ -62,8 +79,8 @@ - - + + @@ -82,7 +99,7 @@ - + @@ -90,8 +107,8 @@ - - + + diff --git a/hg-store-node/pom.xml b/hg-store-node/pom.xml index e382890e3..f9b486529 100644 --- a/hg-store-node/pom.xml +++ b/hg-store-node/pom.xml @@ -1,4 +1,21 @@ + + @@ -140,7 +157,7 @@ - com.baidu.hugegraph.store.node.StoreNodeApplication + org.apache.hugegraph.store.node.StoreNodeApplication diff --git a/hg-store-node/src/assembly/static/bin/app.sh b/hg-store-node/src/assembly/static/bin/app.sh index 6c8c8fbec..dab64ed99 100755 --- a/hg-store-node/src/assembly/static/bin/app.sh +++ b/hg-store-node/src/assembly/static/bin/app.sh @@ -1,4 +1,21 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + #ver 0.1.0 liyan75 on 2021/10/08 #readonly CUR_SHELL=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) readonly CUR_SHELL_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) diff --git a/hg-store-node/src/assembly/static/bin/hg-get-release.sh b/hg-store-node/src/assembly/static/bin/hg-get-release.sh index 86c64f501..fc6051590 100644 --- a/hg-store-node/src/assembly/static/bin/hg-get-release.sh +++ b/hg-store-node/src/assembly/static/bin/hg-get-release.sh @@ -1,4 +1,21 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + #ver 0.1.0 liyan75 on 2021/12/17 readonly REPO_SNAPSHOT_PATH=http://10.14.139.8:8082/artifactory/star readonly REPO_FILE_PATH=http://10.14.139.8:8082/artifactory/star-file diff --git a/hg-store-node/src/assembly/static/bin/hg-update.sh b/hg-store-node/src/assembly/static/bin/hg-update.sh index 195f0dd07..fa8abd787 100644 --- a/hg-store-node/src/assembly/static/bin/hg-update.sh +++ b/hg-store-node/src/assembly/static/bin/hg-update.sh @@ -1,4 +1,21 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + #ver 0.1.0 liyan75 on 2021/12/17 readonly REPO_SNAPSHOT_PATH=http://10.14.139.8:8082/artifactory/star-snapshot readonly REPO_FILE_PATH=http://10.14.139.8:8082/artifactory/star-file diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/AppShutdownHook.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/AppShutdownHook.java deleted file mode 100644 index c1d70439d..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/AppShutdownHook.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.baidu.hugegraph.store.node; - -import com.baidu.hugegraph.rocksdb.access.RocksDBFactory; - -/** - * @author lynn.bond@hotmail.com copy from web - */ -public class AppShutdownHook extends Thread { - - private Thread mainThread; - private boolean shutDownSignalReceived; - - public AppShutdownHook(Thread mainThread) { - super(); - this.mainThread = mainThread; - this.shutDownSignalReceived = false; - Runtime.getRuntime().addShutdownHook(this); - } - - @Override - public void run() { - System.out.println("Shut down signal received."); - this.shutDownSignalReceived = true; - mainThread.interrupt(); - - doSomethingForShutdown(); - - try { - mainThread.join(); //当收到停止信号时，等待mainThread的执行完成 - } catch (InterruptedException e) { - } - System.out.println("Shut down complete."); - } - - public boolean shouldShutDown() { - return shutDownSignalReceived; - } - - private void doSomethingForShutdown() { - RocksDBFactory.getInstance().releaseAllGraphDB(); - } -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/HgStoreMetricsController.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/HgStoreMetricsController.java deleted file mode 100644 index 7be853579..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/HgStoreMetricsController.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.baidu.hugegraph.store.node.controller; - -import com.alipay.sofa.jraft.core.NodeMetrics; -import com.baidu.hugegraph.store.node.grpc.HgStoreNodeService; -import com.baidu.hugegraph.store.node.metrics.DriveMetrics; -import com.baidu.hugegraph.store.node.metrics.SystemMetrics; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RestController; - -import java.util.HashMap; -import java.util.Map; - -/** - * @author lynn.bond@hotmail.com on 2021/11/23 - */ -@RestController -@RequestMapping(value = "/metrics",method = RequestMethod.GET) -public class HgStoreMetricsController { - @Autowired - HgStoreNodeService nodeService; - - private SystemMetrics systemMetrics = new SystemMetrics(); - private DriveMetrics driveMetrics = new DriveMetrics(); - - @GetMapping - public Map index(){ - return new HashMap<>(); - } - - @GetMapping("system") - public Map> system(){ - return this.systemMetrics.metrics(); - } - - @GetMapping("drive") - public Map> drive(){ - return this.driveMetrics.metrics(); - } - - @GetMapping("raft") - public Map getRaftMetrics(){ - return nodeService.getNodeMetrics(); - } - - -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/entry/RestResult.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/entry/RestResult.java deleted file mode 100644 index 6eb566481..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/entry/RestResult.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.baidu.hugegraph.store.node.entry; - -import java.io.Serializable; - -import lombok.Data; - -@Data -public class RestResult implements Serializable { - public static final String OK = "OK"; - public static final String ERR = "ERR"; - String state; - String message; - Serializable data; -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/EmptyIterator.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/EmptyIterator.java deleted file mode 100644 index 2100776a2..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/EmptyIterator.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.baidu.hugegraph.store.node.grpc; - -import com.baidu.hugegraph.rocksdb.access.ScanIterator; - -/** - * @author lynn.bond@hotmail.com on 2021/11/29 - */ -final class EmptyIterator implements ScanIterator { - - @Override - public boolean hasNext() { - return false; - } - - @Override - public boolean isValid() { - return false; - } - - @Override - public T next() { - return null; - } - - @Override - public long count() { - return 0; - } - - @Override - public byte[] position() { - return new byte[0]; - } - - @Override - public void close() { - - } -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/GRpcServerConfig.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/GRpcServerConfig.java deleted file mode 100644 index ffff50f74..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/GRpcServerConfig.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.baidu.hugegraph.store.node.grpc; - -import org.lognet.springboot.grpc.GRpcServerBuilderConfigurer; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import com.baidu.hugegraph.store.node.AppConfig; -import com.baidu.hugegraph.store.node.util.HgExecutorUtil; - -import io.grpc.ServerBuilder; - -/** - * @author lynn.bond@hotmail.com on 2022/3/4 - */ -@Component -public class GRpcServerConfig extends GRpcServerBuilderConfigurer { - public final static String EXECUTOR_NAME = "hg-grpc"; - @Autowired - private AppConfig appConfig; - - @Override - public void configure(ServerBuilder serverBuilder) { - AppConfig.ThreadPoolGrpc grpc = appConfig.getThreadPoolGrpc(); - serverBuilder.executor(HgExecutorUtil.createExecutor(EXECUTOR_NAME, grpc.getCore(), grpc.getMax(), - grpc.getQueue()) - ); - } - -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreNodeState.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreNodeState.java deleted file mode 100644 index 9d996b7a9..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreNodeState.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.baidu.hugegraph.store.node.grpc; - -import com.baidu.hugegraph.store.grpc.state.NodeStateType; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * @author lynn.bond@hotmail.com created on 2021/11/3 - */ -@ThreadSafe -public final class HgStoreNodeState { - - private static NodeStateType curState = NodeStateType.STARTING; - - public static NodeStateType getState() { - return curState; - } - - private static void setState(NodeStateType state) { - curState = state; - change(); - } - - private static void change() { - HgStoreStateSubject.notifyAll(curState); - } - - public static void goOnline() { - setState(NodeStateType.ONLINE); - } - - public static void goStarting() { - setState(NodeStateType.STARTING); - } - - public static void goStopping() { - setState(NodeStateType.STOPPING); - } - -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/QueryCondition.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/QueryCondition.java deleted file mode 100644 index 0f436f585..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/QueryCondition.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.baidu.hugegraph.store.node.grpc; - -/** - * @author lynn.bond@hotmail.com on 2023/2/8 - */ -public interface QueryCondition { - byte[] getStart(); - - byte[] getEnd(); - - byte[] getPrefix(); - - int getKeyCode(); - - int getScanType(); - - byte[] getQuery(); - - byte[] getPosition(); - - int getSerialNo(); -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanOneShotResponse.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanOneShotResponse.java deleted file mode 100644 index fa7c91ae7..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanOneShotResponse.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.baidu.hugegraph.store.node.grpc; - -import static com.baidu.hugegraph.store.node.grpc.ScanUtil.getIterator; -import static com.baidu.hugegraph.store.node.grpc.ScanUtil.toSq; - -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; -import com.baidu.hugegraph.store.grpc.common.Kv; -import com.baidu.hugegraph.store.grpc.stream.KvPageRes; -import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq; -import com.baidu.hugegraph.store.node.util.HgGrpc; -import com.baidu.hugegraph.store.node.util.HgStoreNodeUtil; -import com.google.protobuf.ByteString; - -import io.grpc.Status; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; - - -/** - * @author lynn.bond@hotmail.com created on 2022/02/17 - * @version 3.6.0 - */ -@Slf4j -public class ScanOneShotResponse{ - - /** - * Handle one-shot scan - * - * @param request - * @param responseObserver - */ - public static void scanOneShot(ScanStreamReq request, StreamObserver responseObserver, - HgStoreWrapperEx wrapper) { - KvPageRes.Builder resBuilder = KvPageRes.newBuilder(); - Kv.Builder kvBuilder = Kv.newBuilder(); - ScanIterator iterator = getIterator(toSq(request), wrapper); - - long limit = request.getLimit(); - - if (limit <= 0) { - responseObserver.onError(HgGrpc.toErr("limit<=0, please to invoke stream scan.")); - return; - } - - int count = 0; - - try { - while (iterator.hasNext()) { - - if (++count > limit) { - break; - } - - RocksDBSession.BackendColumn col = iterator.next(); - - resBuilder.addData(kvBuilder - .setKey(ByteString.copyFrom(col.name)) - .setValue(ByteString.copyFrom(col.value)) - .setCode(HgStoreNodeUtil.toInt(iterator.position())) //position == partition-id. - ); - - } - - responseObserver.onNext(resBuilder.build()); - responseObserver.onCompleted(); - - } catch (Throwable t) { - String msg = "an exception occurred during data scanning"; - responseObserver.onError(HgGrpc.toErr(Status.INTERNAL, msg, t)); - }finally { - iterator.close(); - } - - } - - -} \ No newline at end of file diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanQuery.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanQuery.java deleted file mode 100644 index 385f591c4..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanQuery.java +++ /dev/null @@ -1,87 +0,0 @@ -package com.baidu.hugegraph.store.node.grpc; - -import com.baidu.hugegraph.store.grpc.common.ScanMethod; - -import java.util.Arrays; - -/** - * @author lynn.bond@hotmail.com on 2022/2/28 - */ -class ScanQuery implements QueryCondition{ - String graph; - String table; - ScanMethod method; - - byte[] start; - byte[] end; - byte[] prefix; - int keyCode; - int scanType; - byte[] query; - byte[] position; - int serialNo; - - @Override - public byte[] getStart() { - return this.start; - } - - @Override - public byte[] getEnd() { - return this.end; - } - - @Override - public byte[] getPrefix() { - return this.prefix; - } - - @Override - public int getKeyCode() { - return this.keyCode; - } - - @Override - public int getScanType() { - return this.scanType; - } - - @Override - public byte[] getQuery() { - return this.query; - } - - @Override - public byte[] getPosition() { - return this.position; - } - - @Override - public int getSerialNo() { - return this.serialNo; - } - - static ScanQuery of() { - return new ScanQuery(); - } - - private ScanQuery() { - } - - @Override - public String toString() { - return "ScanQuery{" + - "graph='" + graph + '\'' + - ", table='" + table + '\'' + - ", method=" + method + - ", start=" + Arrays.toString(start) + - ", end=" + Arrays.toString(end) + - ", prefix=" + Arrays.toString(prefix) + - ", partition=" + keyCode + - ", scanType=" + scanType + - ", serialNo=" + serialNo + - ", query=" + Arrays.toString(query) + - ", position=" + Arrays.toString(position) + - '}'; - } -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/listener/ContextClosedListener.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/listener/ContextClosedListener.java deleted file mode 100644 index adf699351..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/listener/ContextClosedListener.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.baidu.hugegraph.store.node.listener; - -import java.util.concurrent.ThreadPoolExecutor; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationListener; -import org.springframework.context.event.ContextClosedEvent; - -import com.baidu.hugegraph.store.node.grpc.HgStoreStreamImpl; - -import lombok.extern.slf4j.Slf4j; - -/** - * @author zhangyingjie - * @date 2023/2/17 - **/ -@Slf4j -public class ContextClosedListener implements ApplicationListener { - - @Autowired - HgStoreStreamImpl storeStream; - @Override - public void onApplicationEvent(ContextClosedEvent event) { - try { - log.info("closing scan threads...."); - ThreadPoolExecutor executor = storeStream.getRealExecutor(); - if (executor != null) { - try { - executor.shutdownNow(); - } catch (Exception e) { - - } - } - } catch (Exception e) { - - } finally { - log.info("closed scan threads"); - } - } -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/DriveMetrics.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/DriveMetrics.java deleted file mode 100644 index 9d490f8c4..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/DriveMetrics.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.baidu.hugegraph.store.node.metrics; - -import java.io.File; -import java.util.*; -import java.util.stream.Collectors; - -/** - * @author lynn.bond@hotmail.com on 2021/11/23 - */ -@Deprecated -public class DriveMetrics { - - private static long MIB = 1024 * 1024; - // TODO: add a cache - public Map> metrics() { - File[] rootDrive = File.listRoots(); - - if (rootDrive == null) { - return new LinkedHashMap(0); - } - - Map> metrics = new HashMap<>(); - - for (File d : rootDrive) { - Map buf = new HashMap<>(); - buf.put("total_space", d.getTotalSpace() / MIB); - buf.put("free_space", d.getFreeSpace() / MIB); - buf.put("usable_space", d.getUsableSpace() / MIB); - buf.put("size_unit", "MB"); - - metrics.put(d.getPath().replace("\\",""), buf); - - } - - return metrics; - - } - -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/GRpcExMetrics.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/GRpcExMetrics.java deleted file mode 100644 index 6ad4a0651..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/GRpcExMetrics.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.baidu.hugegraph.store.node.metrics; - -import com.baidu.hugegraph.store.node.grpc.GRpcServerConfig; -import com.baidu.hugegraph.store.node.util.HgExecutorUtil; -import io.micrometer.core.instrument.Gauge; -import io.micrometer.core.instrument.MeterRegistry; - -import java.util.concurrent.ThreadPoolExecutor; - -/** - * @author lynn.bond@hotmail.com on 2022/3/8 - */ -public class GRpcExMetrics { - public final static String PREFIX = "grpc"; - private static MeterRegistry registry; - private final static ExecutorWrapper wrapper = new ExecutorWrapper(); - - private GRpcExMetrics() {} - - public synchronized static void init(MeterRegistry meterRegistry) { - if (registry == null) { - registry = meterRegistry; - registerMeters(); - } - } - - private static void registerMeters() { - registerExecutor(); - - } - - private static void registerExecutor(){ - - Gauge.builder(PREFIX + ".executor.pool.size",wrapper,(e)->e.getPoolSize()) - .description("The current number of threads in the pool.") - .register(registry); - - Gauge.builder(PREFIX + ".executor.core.pool.size",wrapper,(e)->e.getCorePoolSize()) - .description("The largest number of threads that have ever simultaneously been in the pool.") - .register(registry); - - Gauge.builder(PREFIX + ".executor.active.count",wrapper,(e)->e.getActiveCount()) - .description("The approximate number of threads that are actively executing tasks.") - .register(registry); - } - - private static class ExecutorWrapper{ - ThreadPoolExecutor pool; - - void init(){ - if(this.pool==null){ - pool=HgExecutorUtil.getThreadPoolExecutor(GRpcServerConfig.EXECUTOR_NAME); - } - } - - double getPoolSize(){ - init(); - return this.pool==null?0d:this.pool.getPoolSize(); - } - - int getCorePoolSize(){ - init(); - return this.pool==null?0:this.pool.getCorePoolSize(); - } - - int getActiveCount(){ - init(); - return this.pool==null?0:this.pool.getActiveCount(); - } - - } - -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/MetricsConfig.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/MetricsConfig.java deleted file mode 100644 index 5bb07de95..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/MetricsConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.baidu.hugegraph.store.node.metrics; - -import io.micrometer.core.instrument.MeterRegistry; -import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** - * @author lynn.bond@hotmail.com on 2021/11/24 - */ -@Configuration -public class MetricsConfig { - - @Bean - public MeterRegistryCustomizer metricsCommonTags() { - return (registry) -> registry.config().commonTags("hg", "store"); - } - - @Bean - public MeterRegistryCustomizer registerMeters() { - return (registry) -> { - StoreMetrics.init(registry); - RocksDBMetrics.init(registry); - JRaftMetrics.init(registry); - ProcfsMetrics.init(registry); - GRpcExMetrics.init(registry); - }; - } - -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/ProcfsMetrics.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/ProcfsMetrics.java deleted file mode 100644 index 3f6b5e12d..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/ProcfsMetrics.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.baidu.hugegraph.store.node.metrics; - - -import io.micrometer.core.instrument.Gauge; -import io.micrometer.core.instrument.MeterRegistry; - -/** - * @author lynn.bond@hotmail.com on 2022/3/1 - * @version 0.1.0 - */ -public class ProcfsMetrics { - - private static MeterRegistry registry; - public final static String PREFIX = "process_memory"; - - private final static ProcfsSmaps smaps = new ProcfsSmaps();; - private ProcfsMetrics() { - - } - - public synchronized static void init(MeterRegistry meterRegistry) { - if (registry == null) { - registry = meterRegistry; - registerMeters(); - } - } - - private static void registerMeters() { - registerProcessGauge(); - } - - private static void registerProcessGauge() { - Gauge.builder(PREFIX + ".rss.bytes",()-> smaps.get(ProcfsSmaps.KEY.RSS)) - .register(registry); - - Gauge.builder(PREFIX + ".pss.bytes",()-> smaps.get(ProcfsSmaps.KEY.PSS)) - .register(registry); - - Gauge.builder(PREFIX + ".vss.bytes",()-> smaps.get(ProcfsSmaps.KEY.VSS)) - .register(registry); - - Gauge.builder(PREFIX + ".swap.bytes",()-> smaps.get(ProcfsSmaps.KEY.SWAP)) - .register(registry); - - Gauge.builder(PREFIX + ".swappss.bytes",()-> smaps.get(ProcfsSmaps.KEY.SWAPPSS)) - .register(registry); - } - -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/Err.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/Err.java deleted file mode 100644 index a9995007c..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/Err.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.baidu.hugegraph.store.node.util; - -/** - * @author lynn.bond@hotmail.com - */ - class Err { - private String msg; - - public static Err of(String msg){ - return new Err(msg); - } - - private Err(String msg){ - this.msg=msg; - } - - @Override - public String toString() { - return "Err{" + - "msg='" + msg + '\'' + - '}'; - } -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgStoreConst.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgStoreConst.java deleted file mode 100644 index 23d18d415..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgStoreConst.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.baidu.hugegraph.store.node.util; - -import java.util.Collections; -import java.util.List; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/22 - */ -public final class HgStoreConst { - - public final static int SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS=300; - - public final static byte[] EMPTY_BYTES=new byte[0]; - - public static final List EMPTY_LIST = Collections.EMPTY_LIST; - - public final static int SCAN_ALL_PARTITIONS_ID=-1; // means scan all partitions. - - private HgStoreConst(){} - -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/Result.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/Result.java deleted file mode 100644 index 6f27fff99..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/Result.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.baidu.hugegraph.store.node.util; - -/** - * @author lynn.bond@hotmail.com - */ -public class Result{ - private Err err; - private T t; - public static Result of(){ - return new Result(); - } - private Result (){} - public T get(){ - return t; - } - public void set(T t){ - this.t=t; - } - public void err(String msg){ - this.err=Err.of(msg); - } -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/TkEntry.java b/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/TkEntry.java deleted file mode 100644 index 67cc0f06b..000000000 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/TkEntry.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.baidu.hugegraph.store.node.util; - -import java.util.Arrays; -import java.util.Objects; - -/** - * Table Key pair. - */ -public class TkEntry { - private String table; - private byte[] key; - - public TkEntry(String table, byte[] key) { - this.table = table; - this.key = key; - } - - public String getTable() { - return table; - } - - public byte[] getKey() { - return key; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof TkEntry)) return false; - TkEntry tk = (TkEntry) o; - return Objects.equals(table, tk.table) && Arrays.equals(key, tk.key); - } - - @Override - public int hashCode() { - int result = Objects.hash(table); - result = 31 * result + Arrays.hashCode(key); - return result; - } - - @Override - public String toString() { - return "Tk{" + - "table='" + table + '\'' + - ", key=" + Arrays.toString(key) + - '}'; - } -} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/AppConfig.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java similarity index 86% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/AppConfig.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java index fa6b59675..85957f653 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/AppConfig.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.node; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node; import java.util.HashMap; import java.util.Map; @@ -158,14 +175,14 @@ public class AppConfig { @Configuration @ConfigurationProperties(prefix = "app") public class LabelConfig { - private Map label = new HashMap<>(); + private final Map label = new HashMap<>(); } @Data @Configuration @ConfigurationProperties(prefix = "") public class RocksdbConfig { - private Map rocksdb = new HashMap<>(); + private final Map rocksdb = new HashMap<>(); } @PostConstruct diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppShutdownHook.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppShutdownHook.java new file mode 100644 index 000000000..d31fba3b9 --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppShutdownHook.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; + +/** + * @author lynn.bond@hotmail.com copy from web + */ +public class AppShutdownHook extends Thread { + + private final Thread mainThread; + private boolean shutDownSignalReceived; + + public AppShutdownHook(Thread mainThread) { + super(); + this.mainThread = mainThread; + this.shutDownSignalReceived = false; + Runtime.getRuntime().addShutdownHook(this); + } + + @Override + public void run() { + System.out.println("Shut down signal received."); + this.shutDownSignalReceived = true; + mainThread.interrupt(); + + doSomethingForShutdown(); + + try { + mainThread.join(); //当收到停止信号时，等待mainThread的执行完成 + } catch (InterruptedException ignored) { + } + System.out.println("Shut down complete."); + } + + public boolean shouldShutDown() { + return shutDownSignalReceived; + } + + private void doSomethingForShutdown() { + RocksDBFactory.getInstance().releaseAllGraphDB(); + } +} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/StoreNodeApplication.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java similarity index 56% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/StoreNodeApplication.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java index 420d76f50..1f16c5ac1 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/StoreNodeApplication.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java @@ -1,12 +1,30 @@ -package com.baidu.hugegraph.store.node; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.alipay.remoting.util.StringUtils; -import com.baidu.hugegraph.store.node.listener.ContextClosedListener; -import com.baidu.hugegraph.store.node.listener.PdConfigureListener; +package org.apache.hugegraph.store.node; + +import org.apache.hugegraph.store.node.listener.ContextClosedListener; +import org.apache.hugegraph.store.node.listener.PdConfigureListener; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.ConfigurableApplicationContext; +import com.alipay.remoting.util.StringUtils; + /** * */ @@ -27,10 +45,14 @@ public class StoreNodeApplication { System.setProperty("logging.path", "logs"); } System.setProperty("com.alipay.remoting.client.log.level", "WARN"); - if (System.getProperty("bolt.channel_write_buf_low_water_mark") == null) - System.setProperty("bolt.channel_write_buf_low_water_mark", Integer.toString(4 * 1024 * 1024)); - if (System.getProperty("bolt.channel_write_buf_high_water_mark") == null) - System.setProperty("bolt.channel_write_buf_high_water_mark", Integer.toString(8 * 1024 * 1024)); + if (System.getProperty("bolt.channel_write_buf_low_water_mark") == null) { + System.setProperty("bolt.channel_write_buf_low_water_mark", + Integer.toString(4 * 1024 * 1024)); + } + if (System.getProperty("bolt.channel_write_buf_high_water_mark") == null) { + System.setProperty("bolt.channel_write_buf_high_water_mark", + Integer.toString(8 * 1024 * 1024)); + } SpringApplication application = new SpringApplication(StoreNodeApplication.class); PdConfigureListener listener = new PdConfigureListener(); ContextClosedListener closedListener = new ContextClosedListener(); diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgStoreMetricsController.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgStoreMetricsController.java new file mode 100644 index 000000000..42c746f67 --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgStoreMetricsController.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.controller; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hugegraph.store.node.grpc.HgStoreNodeService; +import org.apache.hugegraph.store.node.metrics.DriveMetrics; +import org.apache.hugegraph.store.node.metrics.SystemMetrics; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.alipay.sofa.jraft.core.NodeMetrics; + +/** + * @author lynn.bond@hotmail.com on 2021/11/23 + */ +@RestController +@RequestMapping(value = "/metrics", method = RequestMethod.GET) +public class HgStoreMetricsController { + @Autowired + HgStoreNodeService nodeService; + + private final SystemMetrics systemMetrics = new SystemMetrics(); + private final DriveMetrics driveMetrics = new DriveMetrics(); + + @GetMapping + public Map index() { + return new HashMap<>(); + } + + @GetMapping("system") + public Map> system() { + return this.systemMetrics.metrics(); + } + + @GetMapping("drive") + public Map> drive() { + return this.driveMetrics.metrics(); + } + + @GetMapping("raft") + public Map getRaftMetrics() { + return nodeService.getNodeMetrics(); + } + + +} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/HgStoreStatusController.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgStoreStatusController.java similarity index 64% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/HgStoreStatusController.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgStoreStatusController.java index 522f9f69b..49854e5d9 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/HgStoreStatusController.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgStoreStatusController.java @@ -1,7 +1,28 @@ -package com.baidu.hugegraph.store.node.controller; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.controller; import java.io.Serializable; +import org.apache.hugegraph.store.node.entry.RestResult; +import org.apache.hugegraph.store.node.grpc.HgStoreNodeState; +import org.apache.hugegraph.store.node.grpc.HgStoreStreamImpl; +import org.apache.hugegraph.store.node.model.HgNodeStatus; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.GetMapping; @@ -11,10 +32,6 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import com.baidu.hugegraph.store.grpc.state.ScanState; -import com.baidu.hugegraph.store.node.entry.RestResult; -import com.baidu.hugegraph.store.node.grpc.HgStoreNodeState; -import com.baidu.hugegraph.store.node.grpc.HgStoreStreamImpl; -import com.baidu.hugegraph.store.node.model.HgNodeStatus; import com.google.protobuf.util.JsonFormat; /** @@ -27,7 +44,8 @@ public class HgStoreStatusController { HgStoreStreamImpl streamImpl; @GetMapping("/-/echo") - public HgNodeStatus greeting(@RequestParam(value = "name", defaultValue = "World") String name) { + public HgNodeStatus greeting( + @RequestParam(value = "name", defaultValue = "World") String name) { return new HgNodeStatus(0, name + " is ok."); } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/HgTestController.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgTestController.java similarity index 71% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/HgTestController.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgTestController.java index e1cc3197d..af4c55f62 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/HgTestController.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgTestController.java @@ -1,11 +1,26 @@ -package com.baidu.hugegraph.store.node.controller; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.PartitionEngine; -import com.baidu.hugegraph.store.node.grpc.HgStoreNodeService; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.meta.Store; -import com.baidu.hugegraph.store.cmd.HgCmdProcessor; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.node.controller; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hugegraph.store.node.grpc.HgStoreNodeService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.GetMapping; @@ -13,8 +28,11 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import java.util.ArrayList; -import java.util.List; +import com.baidu.hugegraph.store.PartitionEngine; +import com.baidu.hugegraph.store.meta.Partition; +import com.baidu.hugegraph.store.meta.Store; + +import lombok.extern.slf4j.Slf4j; /** * 仅用于测试 @@ -32,10 +50,10 @@ public class HgTestController { Store store = null; PartitionEngine engine = nodeService.getStoreEngine().getPartitionEngine(0); - for(Partition partition : engine.getPartitions().values()){ + for (Partition partition : engine.getPartitions().values()) { store = nodeService.getStoreEngine().getHgCmdClient() - .getStoreInfo(engine.getLeader().toString()); - }; + .getStoreInfo(engine.getLeader().toString()); + } return store; } @@ -50,14 +68,15 @@ public class HgTestController { public String deleteRaftNode(@PathVariable(value = "groupId") int groupId) { List graphs = new ArrayList<>(); PartitionEngine engine = nodeService.getStoreEngine().getPartitionEngine(groupId); - if ( engine != null ){ - engine.getPartitions().forEach((k, v)->{ + if (engine != null) { + engine.getPartitions().forEach((k, v) -> { graphs.add(v.getGraphName()); }); nodeService.getStoreEngine().destroyPartitionEngine(groupId, graphs); return "OK"; - }else + } else { return "未找到分区"; + } } @@ -79,9 +98,10 @@ public class HgTestController { nodeService.getStoreEngine().getBusinessHandler().closeAll(); return "close all!"; } + @GetMapping(value = "/snapshot", produces = MediaType.APPLICATION_JSON_VALUE) public String doSnapshot() { - nodeService.getStoreEngine().getPartitionEngines().forEach((k,v)->{ + nodeService.getStoreEngine().getPartitionEngines().forEach((k, v) -> { v.snapshot(); }); return "snapshot OK!"; @@ -89,7 +109,7 @@ public class HgTestController { @GetMapping(value = "/compact", produces = MediaType.APPLICATION_JSON_VALUE) public String dbCompaction() { - nodeService.getStoreEngine().getPartitionEngines().forEach((k,v)->{ + nodeService.getStoreEngine().getPartitionEngines().forEach((k, v) -> { nodeService.getStoreEngine().getBusinessHandler().dbCompaction("", k); }); return "snapshot OK!"; diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/IndexAPI.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/IndexAPI.java similarity index 61% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/IndexAPI.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/IndexAPI.java index 018b02e0b..1d0ff0bd8 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/IndexAPI.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/IndexAPI.java @@ -1,23 +1,42 @@ -package com.baidu.hugegraph.store.node.controller; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.alipay.sofa.jraft.entity.PeerId; -import com.alipay.sofa.jraft.util.Endpoint; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.metric.HgStoreMetric; -import com.baidu.hugegraph.store.node.grpc.HgStoreNodeService; -import com.baidu.hugegraph.store.meta.Partition; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +package org.apache.hugegraph.store.node.controller; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.hugegraph.store.node.grpc.HgStoreNodeService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.util.Endpoint; +import com.baidu.hugegraph.pd.grpc.Metapb; +import com.baidu.hugegraph.store.meta.Partition; +import com.baidu.hugegraph.store.metric.HgStoreMetric; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + @RestController @Slf4j @RequestMapping("/") @@ -34,12 +53,13 @@ public class IndexAPI { } @Data - class StoreInfo{ + class StoreInfo { private int leaderCount; private int partitionCount; } + @Data - public class Raft{ + public class Raft { private int groupId; private String role; private String conf; @@ -47,19 +67,19 @@ public class IndexAPI { private long logIndex; private List peers; private List learners; - private List partitions = new ArrayList<>(); + private final List partitions = new ArrayList<>(); } @Data public class PartitionInfo { - private int id; // region id - private String graphName; + private final int id; // region id + private final String graphName; // Region key range [startKey, endKey) - private long startKey; - private long endKey; + private final long startKey; + private final long endKey; private HgStoreMetric.Partition metric; - private String version; - private Metapb.PartitionState workState; + private final String version; + private final Metapb.PartitionState workState; private String leader; diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/PartitionAPI.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/PartitionAPI.java similarity index 73% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/PartitionAPI.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/PartitionAPI.java index 9458be12f..5588e8609 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/PartitionAPI.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/PartitionAPI.java @@ -1,28 +1,55 @@ -package com.baidu.hugegraph.store.node.controller; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.controller; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.node.AppConfig; +import org.apache.hugegraph.store.node.grpc.HgStoreNodeService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.util.Endpoint; import com.baidu.hugegraph.pd.common.PDException; import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.store.HgStoreEngine; import com.baidu.hugegraph.store.PartitionEngine; import com.baidu.hugegraph.store.business.BusinessHandler; -import com.baidu.hugegraph.store.metric.HgStoreMetric; import com.baidu.hugegraph.store.business.InnerKeyCreator; -import com.baidu.hugegraph.store.node.AppConfig; -import com.baidu.hugegraph.store.node.grpc.HgStoreNodeService; import com.baidu.hugegraph.store.meta.Partition; +import com.baidu.hugegraph.store.metric.HgStoreMetric; import com.baidu.hugegraph.util.Bytes; import com.taobao.arthas.agent.attach.ArthasAgent; + import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.*; - -import java.util.*; @RestController @Slf4j @@ -35,12 +62,13 @@ public class PartitionAPI { AppConfig appConfig; @GetMapping(value = "/partitions", produces = "application/json") - public Map getPartitions(@RequestParam(required= false, defaultValue = "") String flags) { + public Map getPartitions( + @RequestParam(required = false, defaultValue = "") String flags) { boolean accurate = false; - if(!flags.isEmpty()) { + if (!flags.isEmpty()) { List flagList = Arrays.asList(flags.split(",")); - if(flagList.contains("accurate")) { + if (flagList.contains("accurate")) { accurate = true; } } @@ -65,13 +93,14 @@ public class PartitionAPI { raft.setTerm(engine.getLeaderTerm()); raft.setLogIndex(engine.getCommittedIndex()); raft.setPartitionCount(engine.getPartitions().size()); - for(Map.Entry partitionEntry : engine.getPartitions().entrySet()) { + for (Map.Entry partitionEntry : engine.getPartitions().entrySet()) { String graphName = partitionEntry.getKey(); Partition pt = partitionEntry.getValue(); PartitionInfo partition = new PartitionInfo(pt); // 此处为了打开所有的图，metric只返回已打开的图 businessHandler.getLatestSequenceNumber(graphName, pt.getId()); - partition.setMetric(businessHandler.getPartitionMetric(graphName, pt.getId(), accurate)); + partition.setMetric( + businessHandler.getPartitionMetric(graphName, pt.getId(), accurate)); partition.setLeader(pt.isLeader() == engine.isLeader() ? "OK" : "Error"); raft.getPartitions().add(partition); } @@ -110,11 +139,13 @@ public class PartitionAPI { return raft; //return okMap("partition", rafts); } + /** * 打印分区的所有key */ @GetMapping(value = "/partition/dump/{id}", produces = MediaType.APPLICATION_JSON_VALUE) - public Map dumpPartition(@PathVariable(value = "id") int id) throws PDException { + public Map dumpPartition(@PathVariable(value = "id") int id) throws + PDException { HgStoreEngine storeEngine = nodeService.getStoreEngine(); BusinessHandler handler = storeEngine.getBusinessHandler(); InnerKeyCreator innerKeyCreator = new InnerKeyCreator(handler); @@ -142,18 +173,20 @@ public class PartitionAPI { * 打印分区的所有key */ @GetMapping(value = "/partition/clean/{id}", produces = MediaType.APPLICATION_JSON_VALUE) - public Map cleanPartition(@PathVariable(value = "id") int id) throws PDException { + public Map cleanPartition(@PathVariable(value = "id") int id) throws + PDException { HgStoreEngine storeEngine = nodeService.getStoreEngine(); BusinessHandler handler = storeEngine.getBusinessHandler(); storeEngine.getPartitionEngine(id).getPartitions().forEach((graph, partition) -> { - handler.cleanPartition(graph, id); + handler.cleanPartition(graph, id); }); return okMap("ok", null); } @GetMapping(value = "/arthasstart", produces = "application/json") - public Map arthasstart(@RequestParam(required= false, defaultValue = "") String flags) { + public Map arthasstart( + @RequestParam(required = false, defaultValue = "") String flags) { HashMap configMap = new HashMap(); configMap.put("arthas.telnetPort", appConfig.getArthasConfig().getTelnetPort()); configMap.put("arthas.httpPort", appConfig.getArthasConfig().getHttpPort()); @@ -165,8 +198,9 @@ public class PartitionAPI { ret.add("Arthas 启动成功"); return okMap("arthasstart", ret); } + @Data - public class Raft{ + public class Raft { private int groupId; private String role; private String conf; @@ -176,19 +210,19 @@ public class PartitionAPI { private List peers; private List learners; private int partitionCount; - private List partitions = new ArrayList<>(); + private final List partitions = new ArrayList<>(); } @Data public class PartitionInfo { - private int id; // region id - private String graphName; + private final int id; // region id + private final String graphName; // Region key range [startKey, endKey) - private long startKey; - private long endKey; + private final long startKey; + private final long endKey; private HgStoreMetric.Partition metric; - private String version; - private Metapb.PartitionState workState; + private final String version; + private final Metapb.PartitionState workState; private String leader; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/entry/RestResult.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/entry/RestResult.java new file mode 100644 index 000000000..5d65cd271 --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/entry/RestResult.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.entry; + +import java.io.Serializable; + +import lombok.Data; + +@Data +public class RestResult implements Serializable { + public static final String OK = "OK"; + public static final String ERR = "ERR"; + String state; + String message; + Serializable data; +} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/BatchGrpcClosure.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchGrpcClosure.java similarity index 70% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/BatchGrpcClosure.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchGrpcClosure.java index a639c98db..978baa3db 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/BatchGrpcClosure.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchGrpcClosure.java @@ -1,22 +1,26 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; /** * @author lynn.bond@hotmail.com on 2022/1/27 */ -import com.alipay.sofa.jraft.Status; -import com.baidu.hugegraph.store.grpc.common.ResCode; -import com.baidu.hugegraph.store.grpc.common.ResStatus; -import com.baidu.hugegraph.store.grpc.session.FeedbackRes; -import com.baidu.hugegraph.store.grpc.session.PartitionFaultResponse; -import com.baidu.hugegraph.store.grpc.session.PartitionFaultType; -import com.baidu.hugegraph.store.grpc.session.PartitionLeader; -import com.baidu.hugegraph.store.raft.RaftClosure; -import com.baidu.hugegraph.store.util.HgRaftError; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; - import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -28,6 +32,21 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; +import org.apache.commons.collections.CollectionUtils; + +import com.alipay.sofa.jraft.Status; +import com.baidu.hugegraph.store.grpc.common.ResCode; +import com.baidu.hugegraph.store.grpc.common.ResStatus; +import com.baidu.hugegraph.store.grpc.session.FeedbackRes; +import com.baidu.hugegraph.store.grpc.session.PartitionFaultResponse; +import com.baidu.hugegraph.store.grpc.session.PartitionFaultType; +import com.baidu.hugegraph.store.grpc.session.PartitionLeader; +import com.baidu.hugegraph.store.raft.RaftClosure; +import com.baidu.hugegraph.store.util.HgRaftError; + +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; + /** * 批量处理的grpc回调封装类 * @@ -80,6 +99,7 @@ class BatchGrpcClosure { /** * 不使用计数器latch + * * @return */ public RaftClosure newClosureNoLatch() { @@ -102,11 +122,12 @@ class BatchGrpcClosure { if (leaderMap.size() > 0) { PartitionFaultResponse.Builder partitionFault = - PartitionFaultResponse.newBuilder().setFaultType(PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LEADER); + PartitionFaultResponse.newBuilder().setFaultType( + PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LEADER); leaderMap.forEach((k, v) -> { partitionFault.addPartitionLeaders(PartitionLeader.newBuilder() - .setPartitionId(k) - .setLeaderId(v).build()); + .setPartitionId(k) + .setLeaderId(v).build()); }); errorResponse = partitionFault.build(); } else { @@ -122,7 +143,7 @@ class BatchGrpcClosure { faultType = PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LOCAL; break; default: - log.error("Unmatchable errorStatus: "+errorStatus); + log.error("Unmatchable errorStatus: " + errorStatus); } errorResponse = PartitionFaultResponse.newBuilder().setFaultType(faultType).build(); } @@ -151,15 +172,19 @@ class BatchGrpcClosure { observer.onNext(ok.apply(results)); } else { observer.onNext((V) FeedbackRes.newBuilder() - .setStatus(ResStatus.newBuilder().setCode(ResCode.RES_CODE_FAIL).setMsg(getErrorMsg())) - .setPartitionFaultResponse(this.getErrorResponse()) - .build()); + .setStatus(ResStatus.newBuilder() + .setCode(ResCode.RES_CODE_FAIL) + .setMsg(getErrorMsg())) + .setPartitionFaultResponse(this.getErrorResponse()) + .build()); } } catch (InterruptedException e) { log.error("waitFinish exception: ", e); observer.onNext((V) FeedbackRes.newBuilder() - .setStatus(ResStatus.newBuilder().setCode(ResCode.RES_CODE_FAIL) - .setMsg(e.getLocalizedMessage()).build()).build()); + .setStatus(ResStatus.newBuilder() + .setCode(ResCode.RES_CODE_FAIL) + .setMsg(e.getLocalizedMessage()) + .build()).build()); } observer.onCompleted(); } @@ -172,18 +197,19 @@ class BatchGrpcClosure { AtomicReference res = new AtomicReference<>(results.get(0)); results.forEach(e -> { try { - if (e.getStatus().getCode() != ResCode.RES_CODE_OK) + if (e.getStatus().getCode() != ResCode.RES_CODE_OK) { res.set(e); - }catch (Exception ex){ + } + } catch (Exception ex) { log.error("{}", ex); } }); return res.get(); } else { return FeedbackRes.newBuilder() - .setStatus(ResStatus.newBuilder() - .setCode(ResCode.RES_CODE_OK).build()) - .build(); + .setStatus(ResStatus.newBuilder() + .setCode(ResCode.RES_CODE_OK).build()) + .build(); } } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/BatchScanIterator.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchScanIterator.java similarity index 68% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/BatchScanIterator.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchScanIterator.java index 2c2084b9e..3fed037ac 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/BatchScanIterator.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchScanIterator.java @@ -1,16 +1,34 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.pd.common.KVPair; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; -import com.baidu.hugegraph.store.node.util.HgAssert; -import com.baidu.hugegraph.store.node.util.HgStoreConst; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.node.grpc; -import javax.annotation.concurrent.NotThreadSafe; import java.util.NoSuchElementException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; +import javax.annotation.concurrent.NotThreadSafe; + +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.node.util.HgAssert; +import org.apache.hugegraph.store.node.util.HgStoreConst; + +import com.baidu.hugegraph.pd.common.KVPair; + /** * @author lynn.bond@hotmail.com on 2022/2/28 * @version 0.3.0 added limit support @@ -24,9 +42,11 @@ public final class BatchScanIterator implements ScanIterator { private boolean hasNext = false; private long curCount; private long curLimit; - private AtomicBoolean closed=new AtomicBoolean(); - public static BatchScanIterator of(Supplier> iteratorSupplier, - Supplier limitSupplier) { + private final AtomicBoolean closed = new AtomicBoolean(); + + public static BatchScanIterator of( + Supplier> iteratorSupplier, + Supplier limitSupplier) { HgAssert.isArgumentNotNull(iteratorSupplier, "iteratorSupplier"); HgAssert.isArgumentNotNull(limitSupplier, "limitSupplier"); return new BatchScanIterator(iteratorSupplier, limitSupplier); @@ -39,9 +59,9 @@ public final class BatchScanIterator implements ScanIterator { } private ScanIterator getIterator() { - ScanIterator buf = null; + ScanIterator buf; int count = 0; - this.curCount = 0l; + this.curCount = 0L; do { buf = this.batchSupplier.get().getValue(); @@ -114,7 +134,7 @@ public final class BatchScanIterator implements ScanIterator { @Override public void close() { - if(this.closed.getAndSet(true)==false){ + if (!this.closed.getAndSet(true)) { if (this.iterator != null) { this.iterator.close(); } diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/EmptyIterator.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/EmptyIterator.java new file mode 100644 index 000000000..a66ab251d --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/EmptyIterator.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +import org.apache.hugegraph.rocksdb.access.ScanIterator; + +/** + * @author lynn.bond@hotmail.com on 2021/11/29 + */ +final class EmptyIterator implements ScanIterator { + + @Override + public boolean hasNext() { + return false; + } + + @Override + public boolean isValid() { + return false; + } + + @Override + public T next() { + return null; + } + + @Override + public long count() { + return 0; + } + + @Override + public byte[] position() { + return new byte[0]; + } + + @Override + public void close() { + + } +} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/FusingScanIterator.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/FusingScanIterator.java similarity index 69% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/FusingScanIterator.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/FusingScanIterator.java index b8f19c4d2..5e3e28d99 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/FusingScanIterator.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/FusingScanIterator.java @@ -1,10 +1,27 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.rocksdb.access.ScanIterator; +package org.apache.hugegraph.store.node.grpc; import java.util.NoSuchElementException; import java.util.function.Supplier; +import org.apache.hugegraph.rocksdb.access.ScanIterator; + /** * This is a wrapper of the ScanIterator that provides a mechanism * to set a threshold value in order to abort the iterating operation. @@ -19,7 +36,8 @@ final class FusingScanIterator implements ScanIterator { private ScanIterator iterator; private byte[] position = EMPTY_BYTES; - public static FusingScanIterator maxOf(long maxThreshold, Supplier iteratorSupplier) { + public static FusingScanIterator maxOf(long maxThreshold, + Supplier iteratorSupplier) { FusingScanIterator res = new FusingScanIterator(); res.max = maxThreshold; res.supplier = iteratorSupplier; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GRpcServerConfig.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GRpcServerConfig.java new file mode 100644 index 000000000..93549daa0 --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GRpcServerConfig.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +import org.apache.hugegraph.store.node.AppConfig; +import org.apache.hugegraph.store.node.util.HgExecutorUtil; +import org.lognet.springboot.grpc.GRpcServerBuilderConfigurer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import io.grpc.ServerBuilder; + +/** + * @author lynn.bond@hotmail.com on 2022/3/4 + */ +@Component +public class GRpcServerConfig extends GRpcServerBuilderConfigurer { + public final static String EXECUTOR_NAME = "hg-grpc"; + @Autowired + private AppConfig appConfig; + + @Override + public void configure(ServerBuilder serverBuilder) { + AppConfig.ThreadPoolGrpc grpc = appConfig.getThreadPoolGrpc(); + serverBuilder.executor( + HgExecutorUtil.createExecutor(EXECUTOR_NAME, grpc.getCore(), grpc.getMax(), + grpc.getQueue()) + ); + } + +} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/GrpcClosure.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcClosure.java similarity index 55% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/GrpcClosure.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcClosure.java index ac0b2d15d..c3ba1bf3e 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/GrpcClosure.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcClosure.java @@ -1,20 +1,38 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.grpc.session.FeedbackRes; -import com.baidu.hugegraph.store.raft.RaftClosure; -import io.grpc.stub.StreamObserver; +package org.apache.hugegraph.store.node.grpc; import java.util.HashMap; import java.util.List; import java.util.Map; +import com.baidu.hugegraph.store.grpc.session.FeedbackRes; +import com.baidu.hugegraph.store.raft.RaftClosure; + +import io.grpc.stub.StreamObserver; + /** * @author lynn.bond@hotmail.com on 2022/1/27 */ abstract class GrpcClosure implements RaftClosure { private V result; - private Map leaderMap = new HashMap<>(); + private final Map leaderMap = new HashMap<>(); public V getResult() { return result; @@ -38,8 +56,9 @@ abstract class GrpcClosure implements RaftClosure { */ public static void setResult(RaftClosure raftClosure, V result) { GrpcClosure closure = (GrpcClosure) raftClosure; - if (closure != null) + if (closure != null) { closure.setResult(result); + } } public static RaftClosure newRaftClosure(StreamObserver observer) { diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreNodeService.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java similarity index 74% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreNodeService.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java index 9d3b33de0..9b5544be9 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreNodeService.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java @@ -1,61 +1,84 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +import static com.baidu.hugegraph.store.grpc.common.GraphMethod.GRAPH_METHOD_DELETE; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.apache.hugegraph.store.node.AppConfig; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.core.NodeMetrics; import com.baidu.hugegraph.store.HgStoreEngine; import com.baidu.hugegraph.store.business.DefaultDataMover; -import com.baidu.hugegraph.store.util.HgStoreException; import com.baidu.hugegraph.store.grpc.session.*; -import com.baidu.hugegraph.store.node.AppConfig; import com.baidu.hugegraph.store.options.HgStoreEngineOptions; import com.baidu.hugegraph.store.options.RaftRocksdbOptions; import com.baidu.hugegraph.store.raft.RaftClosure; import com.baidu.hugegraph.store.raft.RaftOperation; import com.baidu.hugegraph.store.raft.RaftTaskHandler; import com.baidu.hugegraph.store.util.HgRaftError; +import com.baidu.hugegraph.store.util.HgStoreException; import com.google.protobuf.CodedInputStream; import com.google.protobuf.CodedOutputStream; + import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import static com.baidu.hugegraph.store.grpc.common.GraphMethod.GRAPH_METHOD_DELETE; /** * @projectName: raft task executor */ @Slf4j @Service -public class HgStoreNodeService implements RaftTaskHandler{ - public static final byte BATCH_OP = 0x12; - public static final byte TABLE_OP = 0x13; - public static final byte GRAPH_OP = 0x14; - public static final byte CLEAN_OP = 0x15; +public class HgStoreNodeService implements RaftTaskHandler { + public static final byte BATCH_OP = 0x12; + public static final byte TABLE_OP = 0x13; + public static final byte GRAPH_OP = 0x14; + public static final byte CLEAN_OP = 0x15; - public static final byte MAX_OP = 0x59; + public static final byte MAX_OP = 0x59; @Autowired HgStoreSessionImpl hgStoreSession; private HgStoreEngine storeEngine; - private AppConfig appConfig; + private final AppConfig appConfig; public HgStoreNodeService(@Autowired AppConfig appConfig) { this.appConfig = appConfig; } - public HgStoreEngine getStoreEngine(){ return this.storeEngine;} + public HgStoreEngine getStoreEngine() { + return this.storeEngine; + } + @PostConstruct - public void init(){ + public void init() { log.info("{}", appConfig.toString()); - HgStoreEngineOptions options = new HgStoreEngineOptions(){{ + HgStoreEngineOptions options = new HgStoreEngineOptions() {{ setRaftAddress(appConfig.getRaft().getAddress()); setDataPath(appConfig.getDataPath()); setRaftPath(appConfig.getRaftPath()); @@ -64,7 +87,7 @@ public class HgStoreNodeService implements RaftTaskHandler{ setRocksdbConfig(appConfig.getRocksdbConfig()); setGrpcAddress(appConfig.getStoreServerAddress()); setLabels(appConfig.getLabelConfig().getLabel()); - setRaftOptions(new RaftOptions(){{ + setRaftOptions(new RaftOptions() {{ setMetrics(appConfig.getRaft().isMetrics()); setRpcDefaultTimeout(appConfig.getRaft().getRpcTimeOut()); setSnapshotLogIndexMargin(appConfig.getRaft().getSnapshotLogIndexMargin()); @@ -76,7 +99,7 @@ public class HgStoreNodeService implements RaftTaskHandler{ setMaxSegmentFileSize(appConfig.getRaft().getMaxSegmentFileSize()); setMaxReplicatorInflightMsgs(appConfig.getRaft().getMaxReplicatorInflightMsgs()); }}); - setFakePdOptions(new FakePdOptions(){{ + setFakePdOptions(new FakePdOptions() {{ setStoreList(appConfig.getFakePdConfig().getStoreList()); setPeersList(appConfig.getFakePdConfig().getPeersList()); setPartitionCount(appConfig.getFakePdConfig().getPartitionCount()); @@ -95,19 +118,21 @@ public class HgStoreNodeService implements RaftTaskHandler{ } - public List getGraphLeaderPartitionIds(String graphName){ + public List getGraphLeaderPartitionIds(String graphName) { return storeEngine.getPartitionManager().getLeaderPartitionIds(graphName); } /** * 添加raft 任务，转发数据给raft + * * @return true 表示数据已被提交，false表示未提交，用于单副本入库减少批次拆分 */ public - void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, RaftClosure closure) { + void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, + RaftClosure closure) { if (!storeEngine.isClusterReady()) { closure.run(new Status(HgRaftError.CLUSTER_NOT_READY.getNumber(), - "The cluster is not ready, please check active stores number!")); + "The cluster is not ready, please check active stores number!")); log.error("The cluster is not ready, please check active stores number!"); return; } @@ -122,7 +147,7 @@ public class HgStoreNodeService implements RaftTaskHandler{ output.flush(); // 传送给raft storeEngine.addRaftTask(graphName, partitionId, - RaftOperation.create(methodId, buffer, req), closure); + RaftOperation.create(methodId, buffer, req), closure); } catch (Exception e) { closure.run(new Status(HgRaftError.UNKNOWN.getNumber(), e.getMessage())); @@ -135,14 +160,15 @@ public class HgStoreNodeService implements RaftTaskHandler{ * 来自日志的任务，一般是follower 或者 日志回滚的任务 */ @Override - public boolean invoke(int partId, byte[] request, RaftClosure response) throws HgStoreException { + public boolean invoke(int partId, byte[] request, RaftClosure response) throws + HgStoreException { try { CodedInputStream input = CodedInputStream.newInstance(request); byte methodId = input.readRawByte(); - switch (methodId){ + switch (methodId) { case HgStoreNodeService.BATCH_OP: invoke(partId, methodId, BatchReq.parseFrom(input), response); - break ; + break; case HgStoreNodeService.TABLE_OP: invoke(partId, methodId, TableReq.parseFrom(input), response); break; @@ -155,8 +181,8 @@ public class HgStoreNodeService implements RaftTaskHandler{ default: return false; // 未处理 } - }catch (IOException e){ - throw new HgStoreException(e.getMessage(), e); + } catch (IOException e) { + throw new HgStoreException(e.getMessage(), e); } return true; } @@ -165,8 +191,9 @@ public class HgStoreNodeService implements RaftTaskHandler{ * 处理raft传送过来的数据 */ @Override - public boolean invoke(int partId, byte methodId, Object req, RaftClosure response) throws HgStoreException { - switch (methodId){ + public boolean invoke(int partId, byte methodId, Object req, RaftClosure response) throws + HgStoreException { + switch (methodId) { case HgStoreNodeService.BATCH_OP: hgStoreSession.doBatch(partId, (BatchReq) req, response); break; @@ -189,20 +216,20 @@ public class HgStoreNodeService implements RaftTaskHandler{ } @PreDestroy - public void destroy(){ + public void destroy() { storeEngine.shutdown(); } private String getSerializingExceptionMessage(String target) { return "Serializing " - + getClass().getName() - + " to a " - + target - + " threw an IOException (should never happen)."; + + getClass().getName() + + " to a " + + target + + " threw an IOException (should never happen)."; } - public Map getNodeMetrics(){ + public Map getNodeMetrics() { return storeEngine.getNodeMetrics(); } } diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeState.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeState.java new file mode 100644 index 000000000..e396457b6 --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeState.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +import javax.annotation.concurrent.ThreadSafe; + +import com.baidu.hugegraph.store.grpc.state.NodeStateType; + +/** + * @author lynn.bond@hotmail.com created on 2021/11/3 + */ +@ThreadSafe +public final class HgStoreNodeState { + + private static NodeStateType curState = NodeStateType.STARTING; + + public static NodeStateType getState() { + return curState; + } + + private static void setState(NodeStateType state) { + curState = state; + change(); + } + + private static void change() { + HgStoreStateSubject.notifyAll(curState); + } + + public static void goOnline() { + setState(NodeStateType.ONLINE); + } + + public static void goStarting() { + setState(NodeStateType.STARTING); + } + + public static void goStopping() { + setState(NodeStateType.STOPPING); + } + +} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreSessionImpl.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreSessionImpl.java similarity index 74% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreSessionImpl.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreSessionImpl.java index ab55b3d33..9212ad8ba 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreSessionImpl.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreSessionImpl.java @@ -1,7 +1,21 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import static com.baidu.hugegraph.store.node.util.HgGrpc.toHgPair; -import static com.baidu.hugegraph.store.node.util.HgGrpc.toKv; +package org.apache.hugegraph.store.node.grpc; import java.util.HashMap; import java.util.LinkedList; @@ -9,6 +23,9 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hugegraph.store.node.AppConfig; +import org.apache.hugegraph.store.node.util.HgGrpc; +import org.apache.hugegraph.store.node.util.HgStoreNodeUtil; import org.lognet.springboot.grpc.GRpcService; import org.springframework.beans.factory.annotation.Autowired; @@ -33,9 +50,6 @@ import com.baidu.hugegraph.store.grpc.session.TableReq; import com.baidu.hugegraph.store.grpc.session.ValueResponse; import com.baidu.hugegraph.store.meta.Graph; import com.baidu.hugegraph.store.meta.GraphManager; -import com.baidu.hugegraph.store.node.AppConfig; -import com.baidu.hugegraph.store.node.util.HgGrpc; -import com.baidu.hugegraph.store.node.util.HgStoreNodeUtil; import com.baidu.hugegraph.store.pd.PdProvider; import com.baidu.hugegraph.store.raft.RaftClosure; import com.baidu.hugegraph.store.util.HgStoreConst; @@ -53,11 +67,14 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas private HgStoreNodeService storeService; private HgStoreWrapperEx wrapper; private PdProvider pdProvider; + private HgStoreWrapperEx getWrapper() { if (this.wrapper == null) { synchronized (this) { - if (this.wrapper == null) - this.wrapper = new HgStoreWrapperEx(storeService.getStoreEngine().getBusinessHandler()); + if (this.wrapper == null) { + this.wrapper = new HgStoreWrapperEx( + storeService.getStoreEngine().getBusinessHandler()); + } } } return this.wrapper; @@ -66,8 +83,9 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas private PdProvider getPD() { if (pdProvider == null) { synchronized (this) { - if (pdProvider == null) + if (pdProvider == null) { pdProvider = storeService.getStoreEngine().getPdProvider(); + } } } return pdProvider; @@ -86,20 +104,21 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas FeedbackRes res = null; if (value != null) { res = builder.setStatus(HgGrpc.success()) - .setValueResponse( - ValueResponse.newBuilder() - .setValue(ByteString.copyFrom(value)) - ).build(); + .setValueResponse( + ValueResponse.newBuilder() + .setValue(ByteString.copyFrom(value)) + ).build(); } else { res = builder.setStatus(HgGrpc.success()) - .setStatus(HgGrpc.not()) - .build(); + .setStatus(HgGrpc.not()) + .build(); } responseObserver.onNext(res); responseObserver.onCompleted(); } + @Override public void clean(CleanReq request, StreamObserver responseObserver) { @@ -109,20 +128,22 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas // 发给不同的raft执行 BatchGrpcClosure closure = new BatchGrpcClosure<>(1); storeService.addRaftTask(HgStoreNodeService.CLEAN_OP, graph, partition, - request, - closure.newRaftClosure()); + request, + closure.newRaftClosure()); // 等待返回结果 - closure.waitFinish(responseObserver, r -> closure.selectError(r), appConfig.getRaft().getRpcTimeOut()); + closure.waitFinish(responseObserver, r -> closure.selectError(r), + appConfig.getRaft().getRpcTimeOut()); } public void doClean(int partId, CleanReq request, RaftClosure response) { String graph = request.getHeader().getGraph(); FeedbackRes.Builder builder = FeedbackRes.newBuilder(); try { - if (getWrapper().doClean(graph, partId)) + if (getWrapper().doClean(graph, partId)) { builder.setStatus(HgGrpc.success()); - else + } else { builder.setStatus(HgGrpc.not()); + } } catch (Throwable t) { String msg = "Failed to doClean, graph: " + graph + "; partitionId = " + partId; log.error(msg, t); @@ -151,23 +172,28 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas AtomicInteger count = new AtomicInteger(-1); Kv.Builder kvBuilder = Kv.newBuilder(); getWrapper().batchGet(graph, table, - () -> { - if (count.getAndAdd(1) == max) { - return null; - } + () -> { + if (count.getAndAdd(1) == max) { + return null; + } - Key key = keyList.get(count.get()); - if(log.isDebugEnabled())log.debug("batch-get: " + HgStoreNodeUtil.toStr(key.getKey().toByteArray())); - return toHgPair(key); - }, - ( - pair -> { - if (pair.getValue() == null || pair.getKey() == null) { - return; - } - keyValueBuilder.addKv(toKv(pair, kvBuilder)); - } - ) + Key key = keyList.get(count.get()); + if (log.isDebugEnabled()) { + log.debug("batch-get: " + + HgStoreNodeUtil.toStr( + key.getKey() + .toByteArray())); + } + return HgGrpc.toHgPair(key); + }, + ( + pair -> { + if (pair.getValue() == null || pair.getKey() == null) { + return; + } + keyValueBuilder.addKv(HgGrpc.toKv(pair, kvBuilder)); + } + ) ); @@ -299,7 +325,7 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas builder.setStatus(HgGrpc.success()); } catch (Throwable t) { String msg = "Failed to doBatch, graph: " + graph + "; batchId= " + batchId; - log.error(msg,t); + log.error(msg, t); builder.setStatus(HgGrpc.fail(msg)); } GrpcClosure.setResult(response, builder.build()); @@ -342,11 +368,13 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas @Override public void table(TableReq request, StreamObserver observer) { - if(log.isDebugEnabled())log.debug("table: method = {}, graph = {}, table = {}" - , request.getMethod().name() - , request.getHeader().getGraph() - , request.getTableName() - ); + if (log.isDebugEnabled()) { + log.debug("table: method = {}, graph = {}, table = {}" + , request.getMethod().name() + , request.getHeader().getGraph() + , request.getTableName() + ); + } String graph = request.getHeader().getGraph(); // 所有Leader分区 @@ -362,17 +390,18 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas BatchGrpcClosure closure = new BatchGrpcClosure<>(groups.size()); groups.forEach((partition, entries) -> { storeService.addRaftTask(HgStoreNodeService.TABLE_OP, graph, partition, - TableReq.newBuilder(request).build(), - closure.newRaftClosure()); + TableReq.newBuilder(request).build(), + closure.newRaftClosure()); }); if (!groups.isEmpty()) { - // log.info(" table waiting raft..."); + // log.info(" table waiting raft..."); // 等待返回结果 - closure.waitFinish(observer, r -> closure.selectError(r), appConfig.getRaft().getRpcTimeOut()); - // log.info(" table ended waiting raft"); + closure.waitFinish(observer, r -> closure.selectError(r), + appConfig.getRaft().getRpcTimeOut()); + // log.info(" table ended waiting raft"); } else { - // log.info(" table none leader logic"); + // log.info(" table none leader logic"); ResStatus status = null; switch (request.getMethod()) { @@ -383,7 +412,7 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas status = HgGrpc.success(); } - // log.info(" table none leader status: {}", status.getCode()); + // log.info(" table none leader status: {}", status.getCode()); observer.onNext(FeedbackRes.newBuilder().setStatus(status).build()); observer.onCompleted(); } @@ -391,20 +420,22 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas } public void doTable(int partId, TableReq request, RaftClosure response) { - if(log.isDebugEnabled())log.debug(" - doTable[{}]: graph = {}, table = {}" - , request.getMethod().name() - , request.getHeader().getGraph() - , request.getTableName() - ); + if (log.isDebugEnabled()) { + log.debug(" - doTable[{}]: graph = {}, table = {}" + , request.getMethod().name() + , request.getHeader().getGraph() + , request.getTableName() + ); + } FeedbackRes.Builder builder = FeedbackRes.newBuilder(); try { log.debug(" - starting wrapper:doTable "); if (getWrapper().doTable(partId, - request.getMethod(), - request.getHeader().getGraph(), - request.getTableName())) { + request.getMethod(), + request.getHeader().getGraph(), + request.getTableName())) { builder.setStatus(HgGrpc.success()); } else { builder.setStatus(HgGrpc.not()); @@ -412,9 +443,9 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas log.debug(" - ended wrapper:doTable "); } catch (Throwable t) { String msg = "Failed to invoke doTable[ " - + request.getMethod().name() + " ], graph=" - + request.getHeader().getGraph() + " , table=" - + request.getTableName(); + + request.getMethod().name() + " ], graph=" + + request.getHeader().getGraph() + " , table=" + + request.getTableName(); log.error(msg, t); builder.setStatus(HgGrpc.fail(msg)); } @@ -426,11 +457,13 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas @Override public void graph(GraphReq request, StreamObserver observer) { - if(log.isDebugEnabled())log.debug("graph: method = {}, graph = {}, table = {}" - , request.getMethod().name() - , request.getHeader().getGraph() - , request.getGraphName() - ); + if (log.isDebugEnabled()) { + log.debug("graph: method = {}, graph = {}, table = {}" + , request.getMethod().name() + , request.getHeader().getGraph() + , request.getGraphName() + ); + } String graph = request.getHeader().getGraph(); // 所有Leader分区 @@ -446,13 +479,14 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas BatchGrpcClosure closure = new BatchGrpcClosure<>(groups.size()); groups.forEach((partition, entries) -> { storeService.addRaftTask(HgStoreNodeService.GRAPH_OP, graph, partition, - GraphReq.newBuilder(request).build(), - closure.newRaftClosure()); + GraphReq.newBuilder(request).build(), + closure.newRaftClosure()); }); if (!groups.isEmpty()) { // 等待返回结果 - closure.waitFinish(observer, r -> closure.selectError(r), appConfig.getRaft().getRpcTimeOut()); + closure.waitFinish(observer, r -> closure.selectError(r), + appConfig.getRaft().getRpcTimeOut()); } else { observer.onNext(FeedbackRes.newBuilder().setStatus(HgGrpc.success()).build()); @@ -462,26 +496,28 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas } public void doGraph(int partId, GraphReq request, RaftClosure response) { - if (log.isDebugEnabled()) log.debug(" - doGraph[{}]: graph = {}, table = {}" - , request.getMethod().name() - , request.getHeader().getGraph() - , request.getGraphName() - ); + if (log.isDebugEnabled()) { + log.debug(" - doGraph[{}]: graph = {}, table = {}" + , request.getMethod().name() + , request.getHeader().getGraph() + , request.getGraphName() + ); + } FeedbackRes.Builder builder = FeedbackRes.newBuilder(); try { if (getWrapper().doGraph(partId, - request.getMethod(), - request.getHeader().getGraph())) { + request.getMethod(), + request.getHeader().getGraph())) { builder.setStatus(HgGrpc.success()); } else { builder.setStatus(HgGrpc.not()); } } catch (Throwable t) { String msg = "Failed to invoke doGraph[ " - + request.getMethod().name() + " ], graph=" - + request.getHeader().getGraph(); + + request.getMethod().name() + " ], graph=" + + request.getHeader().getGraph(); log.error(msg, t); builder.setStatus(HgGrpc.fail(msg)); } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreStateService.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStateService.java similarity index 59% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreStateService.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStateService.java index e9fe95969..da20d7284 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreStateService.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStateService.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; import org.lognet.springboot.grpc.GRpcService; diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreStateSubject.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStateSubject.java similarity index 52% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreStateSubject.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStateSubject.java index a3df45bb3..707f2808d 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreStateSubject.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStateSubject.java @@ -1,16 +1,33 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.grpc.state.NodeStateRes; -import com.baidu.hugegraph.store.grpc.state.NodeStateType; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.node.grpc; import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import static com.baidu.hugegraph.store.node.util.HgAssert.isArgumentNotNull; -import static com.baidu.hugegraph.store.node.util.HgAssert.isArgumentValid; +import org.apache.hugegraph.store.node.util.HgAssert; + +import com.baidu.hugegraph.store.grpc.state.NodeStateRes; +import com.baidu.hugegraph.store.grpc.state.NodeStateType; + +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; /** * @author lynn.bond@hotmail.com created on 2021/11/3 @@ -22,22 +39,23 @@ public final class HgStoreStateSubject { public static void addObserver(String subId, StreamObserver observer) { - isArgumentValid(subId, "subId"); - isArgumentNotNull(observer == null, "observer"); + HgAssert.isArgumentValid(subId, "subId"); + HgAssert.isArgumentNotNull(observer == null, "observer"); subObserverHolder.put(subId, observer); } public static void removeObserver(String subId) { - isArgumentValid(subId, "subId"); + HgAssert.isArgumentValid(subId, "subId"); subObserverHolder.remove(subId); } public static void notifyAll(NodeStateType nodeState) { - isArgumentNotNull(nodeState == null, "nodeState"); + HgAssert.isArgumentNotNull(nodeState == null, "nodeState"); NodeStateRes res = NodeStateRes.newBuilder().setState(nodeState).build(); - Iterator>> iter = subObserverHolder.entrySet().iterator(); + Iterator>> iter = + subObserverHolder.entrySet().iterator(); while (iter.hasNext()) { Map.Entry> entry = iter.next(); @@ -45,7 +63,8 @@ public final class HgStoreStateSubject { try { entry.getValue().onNext(res); } catch (Throwable e) { - log.error("Failed to send node-state[" + nodeState + "] to subscriber[" + entry.getKey() + "].", e); + log.error("Failed to send node-state[" + nodeState + "] to subscriber[" + + entry.getKey() + "].", e); iter.remove(); log.error("Removed the subscriber[" + entry.getKey() + "].", e); } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreStreamImpl.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStreamImpl.java similarity index 76% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreStreamImpl.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStreamImpl.java index a185271bc..8d3dc0598 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreStreamImpl.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStreamImpl.java @@ -15,13 +15,15 @@ * under the License. */ -package com.baidu.hugegraph.store.node.grpc; +package org.apache.hugegraph.store.node.grpc; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc; import org.apache.hugegraph.store.grpc.stream.KvStream; +import org.apache.hugegraph.store.node.AppConfig; +import org.apache.hugegraph.store.node.util.HgExecutorUtil; import org.lognet.springboot.grpc.GRpcService; import org.springframework.beans.factory.annotation.Autowired; @@ -29,8 +31,6 @@ import com.baidu.hugegraph.store.grpc.state.ScanState; import com.baidu.hugegraph.store.grpc.stream.KvPageRes; import com.baidu.hugegraph.store.grpc.stream.ScanStreamBatchReq; import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq; -import com.baidu.hugegraph.store.node.AppConfig; -import com.baidu.hugegraph.store.node.util.HgExecutorUtil; import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; @@ -53,7 +53,8 @@ public class HgStoreStreamImpl extends HgStoreStreamGrpc.HgStoreStreamImplBase { if (this.wrapper == null) { synchronized (this) { if (this.wrapper == null) { - this.wrapper = new HgStoreWrapperEx(storeService.getStoreEngine().getBusinessHandler()); + this.wrapper = new HgStoreWrapperEx( + storeService.getStoreEngine().getBusinessHandler()); } } } @@ -69,8 +70,9 @@ public class HgStoreStreamImpl extends HgStoreStreamGrpc.HgStoreStreamImplBase { synchronized (this) { if (this.executor == null) { AppConfig.ThreadPoolScan scan = this.appConfig.getThreadPoolScan(); - this.executor = HgExecutorUtil.createExecutor("hg-scan", scan.getCore(), scan.getMax(), - scan.getQueue()); + this.executor = + HgExecutorUtil.createExecutor("hg-scan", scan.getCore(), scan.getMax(), + scan.getQueue()); } } } @@ -81,13 +83,15 @@ public class HgStoreStreamImpl extends HgStoreStreamGrpc.HgStoreStreamImplBase { ThreadPoolExecutor ex = getExecutor(); ScanState.Builder builder = ScanState.newBuilder(); BlockingQueue queue = ex.getQueue(); - ScanState state = builder.setActiveCount(ex.getActiveCount()).setTaskCount(ex.getTaskCount()) - .setCompletedTaskCount(ex.getCompletedTaskCount()) - .setMaximumPoolSize(ex.getMaximumPoolSize()) - .setLargestPoolSize(ex.getLargestPoolSize()).setPoolSize(ex.getPoolSize()) - .setAddress(appConfig.getStoreServerAddress()) - .setQueueSize(queue.size()).setQueueRemainingCapacity(queue.remainingCapacity()) - .build(); + ScanState state = + builder.setActiveCount(ex.getActiveCount()).setTaskCount(ex.getTaskCount()) + .setCompletedTaskCount(ex.getCompletedTaskCount()) + .setMaximumPoolSize(ex.getMaximumPoolSize()) + .setLargestPoolSize(ex.getLargestPoolSize()).setPoolSize(ex.getPoolSize()) + .setAddress(appConfig.getStoreServerAddress()) + .setQueueSize(queue.size()) + .setQueueRemainingCapacity(queue.remainingCapacity()) + .build(); return state; } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreWrapperEx.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreWrapperEx.java similarity index 68% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreWrapperEx.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreWrapperEx.java index dd40b711b..5e774efac 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreWrapperEx.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreWrapperEx.java @@ -1,10 +1,28 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; import java.util.List; import java.util.function.Consumer; import java.util.function.Supplier; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.ScanIterator; + import com.baidu.hugegraph.store.business.BusinessHandler; import com.baidu.hugegraph.store.business.FilterIterator; import com.baidu.hugegraph.store.grpc.common.GraphMethod; @@ -45,9 +63,11 @@ public class HgStoreWrapperEx { return FilterIterator.of(scanIterator, query); } - public ScanIterator scan(String graph, int partId, String table, byte[] start, byte[] end, int scanType, + public ScanIterator scan(String graph, int partId, String table, byte[] start, byte[] end, + int scanType, byte[] query) { - ScanIterator scanIterator = this.handler.scan(graph, partId, table, start, end, scanType, query); + ScanIterator scanIterator = + this.handler.scan(graph, partId, table, start, end, scanType, query); return FilterIterator.of(scanIterator, query); } @@ -58,9 +78,11 @@ public class HgStoreWrapperEx { })); } - public ScanIterator scanPrefix(String graph, int partition, String table, byte[] prefix, int scanType, + public ScanIterator scanPrefix(String graph, int partition, String table, byte[] prefix, + int scanType, byte[] query) { - ScanIterator scanIterator = this.handler.scanPrefix(graph, partition, table, prefix, scanType); + ScanIterator scanIterator = + this.handler.scanPrefix(graph, partition, table, prefix, scanType); return FilterIterator.of(scanIterator, query); } @@ -99,13 +121,10 @@ public class HgStoreWrapperEx { public boolean doGraph(int partId, GraphMethod method, String graph) { boolean flag = true; - switch (method) { - case GRAPH_METHOD_DELETE: - // 交给raft执行，此处不处理 - flag = true; - break; - default: - throw new UnsupportedOperationException("GraphMethod: " + method.name()); + if (method == GRAPH_METHOD_DELETE) {// 交给raft执行，此处不处理 + flag = true; + } else { + throw new UnsupportedOperationException("GraphMethod: " + method.name()); } return flag; } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ParallelScanIterator.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java similarity index 82% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ParallelScanIterator.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java index a8d86816e..f9121216c 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ParallelScanIterator.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; import java.util.ArrayList; import java.util.LinkedList; @@ -11,15 +28,16 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Supplier; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.node.util.HgAssert; +import org.apache.hugegraph.store.node.util.PropertyUtil; + import com.alipay.sofa.jraft.util.Utils; import com.baidu.hugegraph.pd.common.KVPair; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.store.buffer.KVByteBuffer; import com.baidu.hugegraph.store.grpc.common.ScanOrderType; import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest; -import com.baidu.hugegraph.store.node.util.HgAssert; -import com.baidu.hugegraph.store.node.util.PropertyUtil; import com.baidu.hugegraph.store.term.Bits; import lombok.extern.slf4j.Slf4j; @@ -30,31 +48,33 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class ParallelScanIterator implements ScanIterator { - private int batchSize = PropertyUtil.getInt("app.scan.stream.entries.size", 20000); - protected static int maxBodySize = PropertyUtil.getInt("app.scan.stream.body.size", 1 * 1024 * 1024); - private static int waitDataMaxTryTimes = 600; + private final int batchSize = PropertyUtil.getInt("app.scan.stream.entries.size", 20000); + protected static int maxBodySize = + PropertyUtil.getInt("app.scan.stream.body.size", 1024 * 1024); + private static final int waitDataMaxTryTimes = 600; private int maxWorkThreads = Utils.cpus() / 8; private int maxInQueue = maxWorkThreads * 2; private final Supplier> batchSupplier; private final Supplier limitSupplier; private volatile boolean finished; - private BlockingQueue> queue; + private final BlockingQueue> queue; private final ReentrantLock queueLock = new ReentrantLock(); final private ThreadPoolExecutor executor; private List current = null; - private ScanQueryRequest query; - private Queue scanners = new LinkedList<>(); - private Queue pauseScanners = new LinkedList<>(); + private final ScanQueryRequest query; + private final Queue scanners = new LinkedList<>(); + private final Queue pauseScanners = new LinkedList<>(); final private List NO_DATA = new ArrayList<>(); - private boolean orderVertex; - private boolean orderEdge; + private final boolean orderVertex; + private final boolean orderEdge; - public static ParallelScanIterator of(Supplier> iteratorSupplier, - Supplier limitSupplier, - ScanQueryRequest query, - ThreadPoolExecutor executor) { + public static ParallelScanIterator of( + Supplier> iteratorSupplier, + Supplier limitSupplier, + ScanQueryRequest query, + ThreadPoolExecutor executor) { HgAssert.isArgumentNotNull(iteratorSupplier, "iteratorSupplier"); HgAssert.isArgumentNotNull(limitSupplier, "limitSupplier"); return new ParallelScanIterator(iteratorSupplier, limitSupplier, query, executor); @@ -74,7 +94,8 @@ public class ParallelScanIterator implements ScanIterator { if (orderVertex) { this.maxWorkThreads = 1; } else { - this.maxWorkThreads = Math.max(1, Math.min(query.getConditionCount() / 16, maxWorkThreads)); + this.maxWorkThreads = + Math.max(1, Math.min(query.getConditionCount() / 16, maxWorkThreads)); } this.maxInQueue = maxWorkThreads * 2; // 边有序需要更大的队列 @@ -102,7 +123,8 @@ public class ParallelScanIterator implements ScanIterator { tryTimes++; } if (current == null && tryTimes >= waitDataMaxTryTimes) { - log.error("Wait data timeout!!!, scanner is {}/{}", scanners.size(), pauseScanners.size()); + log.error("Wait data timeout!!!, scanner is {}/{}", scanners.size(), + pauseScanners.size()); } return current != null && current != NO_DATA; } @@ -219,7 +241,7 @@ public class ParallelScanIterator implements ScanIterator { } } // 数据未结束，线程继续执行 - return hasNext ? true : this.queue.size() < maxInQueue; + return hasNext || this.queue.size() < maxInQueue; } private synchronized KVPair getIterator() { @@ -241,7 +263,7 @@ public class ParallelScanIterator implements ScanIterator { private long limit; private long counter; private volatile boolean closed = false; - private ReentrantLock iteratorLock = new ReentrantLock(); + private final ReentrantLock iteratorLock = new ReentrantLock(); private ScanIterator getIterator() { // 迭代器没有数据，或该点以达到limit，切换新的迭代器 @@ -270,7 +292,8 @@ public class ParallelScanIterator implements ScanIterator { if (iterator == null) { break; } - while (iterator.hasNext() && entriesSize < batchSize && bodySize < maxBodySize && + while (iterator.hasNext() && entriesSize < batchSize && + bodySize < maxBodySize && counter < limit && !closed) { KV kv = KV.of(iterator.next()); dataList.add(orderVertex ? kv.setNo(query.getSerialNo()) : kv); diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/QueryCondition.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/QueryCondition.java new file mode 100644 index 000000000..3d147469c --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/QueryCondition.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +/** + * @author lynn.bond@hotmail.com on 2023/2/8 + */ +public interface QueryCondition { + byte[] getStart(); + + byte[] getEnd(); + + byte[] getPrefix(); + + int getKeyCode(); + + int getScanType(); + + byte[] getQuery(); + + byte[] getPosition(); + + int getSerialNo(); +} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java similarity index 57% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java index 732449277..a7ba8b480 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java @@ -1,15 +1,33 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import static com.baidu.hugegraph.store.node.grpc.ScanUtil.getIterator; +package org.apache.hugegraph.store.node.grpc; + +import static org.apache.hugegraph.store.node.grpc.ScanUtil.getIterator; + +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.node.util.HgGrpc; +import org.apache.hugegraph.store.node.util.HgStoreNodeUtil; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.store.grpc.common.Kv; import com.baidu.hugegraph.store.grpc.stream.KvPageRes; import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest; import com.baidu.hugegraph.store.grpc.stream.ScanStreamBatchReq; -import com.baidu.hugegraph.store.node.util.HgGrpc; -import com.baidu.hugegraph.store.node.util.HgStoreNodeUtil; import com.google.protobuf.ByteString; import io.grpc.Status; @@ -45,7 +63,8 @@ public class ScanBatchOneShotResponse { if (limit <= 0) { limit = Integer.MAX_VALUE; - log.warn("As limit is less than or equals 0, default limit was effective:[ {} ]", Integer.MAX_VALUE); + log.warn("As limit is less than or equals 0, default limit was effective:[ {} ]", + Integer.MAX_VALUE); } int count = 0; @@ -60,9 +79,10 @@ public class ScanBatchOneShotResponse { RocksDBSession.BackendColumn col = iterator.next(); resBuilder.addData(kvBuilder - .setKey(ByteString.copyFrom(col.name)) - .setValue(ByteString.copyFrom(col.value)) - .setCode(HgStoreNodeUtil.toInt(iterator.position())) //position == partition-id. + .setKey(ByteString.copyFrom(col.name)) + .setValue(ByteString.copyFrom(col.value)) + .setCode(HgStoreNodeUtil.toInt(iterator.position())) +//position == partition-id. ); } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchResponse.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java similarity index 88% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchResponse.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java index 93eb0b671..428288ad2 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchResponse.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java @@ -15,24 +15,22 @@ * under the License. */ -package com.baidu.hugegraph.store.node.grpc; +package org.apache.hugegraph.store.node.grpc; -import static com.baidu.hugegraph.store.node.grpc.ScanUtil.getParallelIterator; - import java.util.List; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.locks.ReentrantLock; +import org.apache.hugegraph.rocksdb.access.ScanIterator; import org.apache.hugegraph.store.grpc.stream.KvStream; +import org.apache.hugegraph.store.node.util.HgGrpc; +import org.apache.hugegraph.store.node.util.PropertyUtil; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.store.buffer.ByteBufferAllocator; import com.baidu.hugegraph.store.buffer.KVByteBuffer; import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest; import com.baidu.hugegraph.store.grpc.stream.ScanStreamBatchReq; -import com.baidu.hugegraph.store.node.util.HgGrpc; -import com.baidu.hugegraph.store.node.util.PropertyUtil; import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; @@ -49,7 +47,7 @@ public class ScanBatchResponse implements StreamObserver { private final int activeTimeout = PropertyUtil.getInt("app.scan.stream.timeout", 60); //单位秒 static ByteBufferAllocator bfAllocator = - new ByteBufferAllocator(ParallelScanIterator.maxBodySize*3/2, 1000); + new ByteBufferAllocator(ParallelScanIterator.maxBodySize * 3 / 2, 1000); // 当前正在遍历的迭代器 private ScanIterator iterator; // 下一次发送的序号 @@ -65,14 +63,15 @@ public class ScanBatchResponse implements StreamObserver { private long activeTime; private volatile State state; private final Object stateLock = new Object(); - private ReentrantLock iteratorLock = new ReentrantLock(); + private final ReentrantLock iteratorLock = new ReentrantLock(); private final StreamObserver sender; private final HgStoreWrapperEx wrapper; private final ThreadPoolExecutor executor; private final long logId; - public ScanBatchResponse(StreamObserver response, HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) { + public ScanBatchResponse(StreamObserver response, HgStoreWrapperEx wrapper, + ThreadPoolExecutor executor) { this.sender = response; this.wrapper = wrapper; this.executor = executor; @@ -143,12 +142,13 @@ public class ScanBatchResponse implements StreamObserver { // log.info("Stream {} startQuery graphName is {}, query degree/keylimit/limit is " + // "{}/{}/{}, scanType is {}, orderType is {}", - // this.logId, graphName, query.getSkipDegree(), query.getPerKeyLimit(), query.getLimit(), + // this.logId, graphName, query.getSkipDegree(), query.getPerKeyLimit(), query + // .getLimit(), // query.getScanType(), query.getOrderType()); this.clientLimit = request.getLimit(); this.entriesCounter = 0; - this.iterator = getParallelIterator(graphName, request, this.wrapper, executor); + this.iterator = ScanUtil.getParallelIterator(graphName, request, this.wrapper, executor); synchronized (stateLock) { if (state == State.IDLE) { state = State.DOING; @@ -186,16 +186,16 @@ public class ScanBatchResponse implements StreamObserver { private void sendEntries() { iteratorLock.lock(); try { - if ( state == State.DONE || iterator == null) { + if (state == State.DONE || iterator == null) { setStateIdle(); return; } KvStream.Builder dataBuilder = KvStream.newBuilder() - .setVersion(1); + .setVersion(1); while (iterator.hasNext() - && (nextSeqNo - clientSeqNo < maxInFlightCount) - && this.entriesCounter < clientLimit - && state != State.DONE) { + && (nextSeqNo - clientSeqNo < maxInFlightCount) + && this.entriesCounter < clientLimit + && state != State.DONE) { KVByteBuffer buffer = new KVByteBuffer(bfAllocator.get()); List dataList = iterator.next(); dataList.forEach(kv -> { @@ -204,7 +204,7 @@ public class ScanBatchResponse implements StreamObserver { }); dataBuilder.setStream(buffer.flip().getBuffer()); dataBuilder.setSeqNo(nextSeqNo++); - dataBuilder.complete(e->{ + dataBuilder.complete(e -> { bfAllocator.release(buffer.getBuffer()); }); this.sender.onNext(dataBuilder.build()); @@ -216,12 +216,13 @@ public class ScanBatchResponse implements StreamObserver { } else { setStateIdle(); } - }catch (Throwable e){ + } catch (Throwable e) { log.error("exception ", e); setStateIdle(); - if ( this.sender != null) + if (this.sender != null) { this.sender.onError(e); - }finally { + } + } finally { iteratorLock.unlock(); } } @@ -229,7 +230,7 @@ public class ScanBatchResponse implements StreamObserver { private void sendNoDataEntries() { try { this.sender.onNext(KvStream.newBuilder().setOver(true).build()); - }catch (Exception e){ + } catch (Exception e) { } } @@ -242,8 +243,9 @@ public class ScanBatchResponse implements StreamObserver { private State setStateIdle() { synchronized (this.stateLock) { - if (this.state != State.DONE) + if (this.state != State.DONE) { this.state = State.IDLE; + } } return state; } @@ -252,7 +254,7 @@ public class ScanBatchResponse implements StreamObserver { * 检查是否活跃，超过一定时间客户端没有请求数据，认为已经不活跃，关闭连接释放资源 */ public void checkActiveTimeout() { - if ((System.currentTimeMillis() - activeTime) > activeTimeout * 1000) { + if ((System.currentTimeMillis() - activeTime) > activeTimeout * 1000L) { log.warn("The stream is not closed, and the timeout is forced to close"); closeQuery(); } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchResponse3.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java similarity index 73% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchResponse3.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java index 2b28da9d8..37dbdcb4c 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchResponse3.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java @@ -1,6 +1,21 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import static com.baidu.hugegraph.store.node.grpc.ScanUtil.getIterator; +package org.apache.hugegraph.store.node.grpc; import java.util.List; import java.util.concurrent.ThreadPoolExecutor; @@ -12,18 +27,19 @@ import java.util.concurrent.locks.ReentrantLock; import javax.annotation.concurrent.NotThreadSafe; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.node.util.Base58; +import org.apache.hugegraph.store.node.util.HgAssert; +import org.apache.hugegraph.store.node.util.HgGrpc; +import org.apache.hugegraph.store.node.util.HgStoreConst; +import org.apache.hugegraph.store.node.util.HgStoreNodeUtil; + import com.baidu.hugegraph.store.grpc.common.Kv; import com.baidu.hugegraph.store.grpc.stream.KvPageRes; import com.baidu.hugegraph.store.grpc.stream.ScanCondition; import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest; import com.baidu.hugegraph.store.grpc.stream.ScanStreamBatchReq; -import com.baidu.hugegraph.store.node.util.Base58; -import com.baidu.hugegraph.store.node.util.HgAssert; -import com.baidu.hugegraph.store.node.util.HgGrpc; -import com.baidu.hugegraph.store.node.util.HgStoreConst; -import com.baidu.hugegraph.store.node.util.HgStoreNodeUtil; import com.google.protobuf.ByteString; import io.grpc.Status; @@ -39,14 +55,16 @@ public class ScanBatchResponse3 { private final static long DEFAULT_PACKAGE_SIZE = 10_000; private final static int MAX_NOT_RECEIPT = 10; - public static StreamObserver of(StreamObserver responseObserver, HgStoreWrapperEx wrapper,ThreadPoolExecutor executor) { + public static StreamObserver of(StreamObserver responseObserver, + HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) { HgAssert.isArgumentNotNull(responseObserver, "responseObserver"); HgAssert.isArgumentNotNull(wrapper, "wrapper"); - return new Broker(responseObserver, wrapper,executor); + return new Broker(responseObserver, wrapper, executor); } private enum OrderState { - NEW(0), WORKING(1);//, PAUSE(2), COMPLETE(10); + NEW(0), + WORKING(1);//, PAUSE(2), COMPLETE(10); int value; OrderState(int value) { @@ -61,12 +79,13 @@ public class ScanBatchResponse3 { private final ThreadPoolExecutor executor; private String graph; - private OrderManager manager = new OrderManager(); + private final OrderManager manager = new OrderManager(); - Broker(StreamObserver responseObserver, HgStoreWrapperEx wrapper,ThreadPoolExecutor executor) { + Broker(StreamObserver responseObserver, HgStoreWrapperEx wrapper, + ThreadPoolExecutor executor) { this.responseObserver = responseObserver; this.wrapper = wrapper; - this.executor=executor; + this.executor = executor; } @Override @@ -113,7 +132,8 @@ public class ScanBatchResponse3 { ScanCondition c = conditions.get(0); if (c.getPrefix() != null && c.getPrefix().size() > 0) { deliverId = Base58.encode(c.getPrefix().toByteArray()); - log.info("[ANALYSIS DEAL] [{}] prefixLength: {}", deliverId, conditions.size()); + log.info("[ANALYSIS DEAL] [{}] prefixLength: {}", deliverId, + conditions.size()); } } @@ -123,7 +143,7 @@ public class ScanBatchResponse3 { OrderWorker worker = new OrderWorker( request.getLimit(), request.getPageSize(), - getIterator(this.graph, request, this.wrapper), + ScanUtil.getIterator(this.graph, request, this.wrapper), deliverer, this.executor); @@ -171,9 +191,9 @@ public class ScanBatchResponse3 { private static class OrderDeliverer { private final StreamObserver responseObserver; - private AtomicBoolean finishFlag = new AtomicBoolean(); + private final AtomicBoolean finishFlag = new AtomicBoolean(); private final String delivererId; - private AtomicLong count = new AtomicLong(); + private final AtomicLong count = new AtomicLong(); OrderDeliverer(String delivererId, StreamObserver responseObserver) { this.responseObserver = responseObserver; @@ -189,9 +209,10 @@ public class ScanBatchResponse3 { if (log.isDebugEnabled()) log.debug("deliver times : {}, over: {}", times, isOver); if (isOver) { - if(log.isDebugEnabled()){ + if (log.isDebugEnabled()) { if (delivererId != null && !delivererId.isEmpty()) { - log.debug("[ANALYSIS OVER] [{}] count: {}, times: {}", delivererId, count, times); + log.debug("[ANALYSIS OVER] [{}] count: {}, times: {}", delivererId, count, + times); } } this.finish(); @@ -199,19 +220,19 @@ public class ScanBatchResponse3 { } void finish() { - if (finishFlag.getAndSet(true) == false) { + if (!finishFlag.getAndSet(true)) { this.responseObserver.onCompleted(); } } void error(String msg) { - if (finishFlag.getAndSet(true) == false) { + if (!finishFlag.getAndSet(true)) { this.responseObserver.onError(HgGrpc.toErr(msg)); } } void error(String msg, Throwable t) { - if (finishFlag.getAndSet(true) == false) { + if (!finishFlag.getAndSet(true)) { this.responseObserver.onError(HgGrpc.toErr(Status.INTERNAL, msg, t)); } @@ -230,20 +251,24 @@ public class ScanBatchResponse3 { private final AtomicInteger receiptTimes = new AtomicInteger(); private final AtomicInteger curTimes = new AtomicInteger(); private final ThreadPoolExecutor executor; - private long limit; + private final long limit; private long packageSize; private long counter; - OrderWorker(long limit, long packageSize, ScanIterator iterator, OrderDeliverer deliverer,ThreadPoolExecutor executor) { + OrderWorker(long limit, long packageSize, ScanIterator iterator, OrderDeliverer deliverer, + ThreadPoolExecutor executor) { this.limit = limit; this.packageSize = packageSize; this.iterator = iterator; this.deliverer = deliverer; - this.executor=executor; + this.executor = executor; if (this.packageSize <= 0) { this.packageSize = DEFAULT_PACKAGE_SIZE; - log.warn("As page-Size is less than or equals 0, default package-size was effective.[ {} ]", DEFAULT_PACKAGE_SIZE); + log.warn( + "As page-Size is less than or equals 0, default package-size was " + + "effective.[ {} ]", + DEFAULT_PACKAGE_SIZE); } } @@ -292,7 +317,7 @@ public class ScanBatchResponse3 { } private void working() { - if (this.isWorking.getAndSet(true) == true) { + if (this.isWorking.getAndSet(true)) { return; } @@ -321,12 +346,17 @@ public class ScanBatchResponse3 { if (!this.checkContinue()) { long start = System.currentTimeMillis(); - iterator.wait(HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * 1000); + iterator.wait( + HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * + 1000); if (System.currentTimeMillis() - start - >= HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * 1000) { + >= + HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * 1000) { throw new TimeoutException("Waiting continue more than " - + HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS + " seconds."); + + + HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS + + " seconds."); } if (this.breakdown.get()) { @@ -342,9 +372,11 @@ public class ScanBatchResponse3 { RocksDBSession.BackendColumn col = iterator.next(); dataBuilder.addData(kvBuilder - .setKey(ByteString.copyFrom(col.name)) - .setValue(ByteString.copyFrom(col.value)) - .setCode(HgStoreNodeUtil.toInt(iterator.position())) //position == partition-id. + .setKey(ByteString.copyFrom(col.name)) + .setValue(ByteString.copyFrom(col.value)) + .setCode(HgStoreNodeUtil.toInt( + iterator.position())) +//position == partition-id. ); } @@ -361,7 +393,8 @@ public class ScanBatchResponse3 { } catch (TimeoutException t) { log.info(t.getMessage()); this.deliverer.error("Sever waiting exceeded [" - + HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS + "] seconds."); + + HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS + + "] seconds."); } catch (Throwable t) { log.error("Failed to do while for scanning, cause by:", t); this.deliverer.error("Failed to finish scanning ", t); diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchResponseFactory.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponseFactory.java similarity index 89% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchResponseFactory.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponseFactory.java index 35436034b..7ade55f84 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchResponseFactory.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponseFactory.java @@ -15,7 +15,7 @@ * under the License. */ -package com.baidu.hugegraph.store.node.grpc; +package org.apache.hugegraph.store.node.grpc; import java.util.Set; import java.util.concurrent.ThreadPoolExecutor; @@ -33,7 +33,7 @@ public class ScanBatchResponseFactory { return instance; } - private Set streamObservers = new ConcurrentHashSet<>(); + private final Set streamObservers = new ConcurrentHashSet<>(); public int addStreamObserver(StreamObserver observer) { streamObservers.add(observer); @@ -54,7 +54,8 @@ public class ScanBatchResponseFactory { }); } - public static StreamObserver of(StreamObserver responseObserver, HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) { + public static StreamObserver of(StreamObserver responseObserver, + HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) { StreamObserver observer = new ScanBatchResponse(responseObserver, wrapper, executor); getInstance().addStreamObserver(observer); getInstance().checkStreamActive(); diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanOneShotResponse.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanOneShotResponse.java new file mode 100644 index 000000000..cc4005d17 --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanOneShotResponse.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.node.util.HgGrpc; +import org.apache.hugegraph.store.node.util.HgStoreNodeUtil; + +import com.baidu.hugegraph.store.grpc.common.Kv; +import com.baidu.hugegraph.store.grpc.stream.KvPageRes; +import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq; +import com.google.protobuf.ByteString; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; + + +/** + * @author lynn.bond@hotmail.com created on 2022/02/17 + * @version 3.6.0 + */ +@Slf4j +public class ScanOneShotResponse { + + /** + * Handle one-shot scan + * + * @param request + * @param responseObserver + */ + public static void scanOneShot(ScanStreamReq request, + StreamObserver responseObserver, + HgStoreWrapperEx wrapper) { + KvPageRes.Builder resBuilder = KvPageRes.newBuilder(); + Kv.Builder kvBuilder = Kv.newBuilder(); + ScanIterator iterator = ScanUtil.getIterator(ScanUtil.toSq(request), wrapper); + + long limit = request.getLimit(); + + if (limit <= 0) { + responseObserver.onError(HgGrpc.toErr("limit<=0, please to invoke stream scan.")); + return; + } + + int count = 0; + + try { + while (iterator.hasNext()) { + + if (++count > limit) { + break; + } + + RocksDBSession.BackendColumn col = iterator.next(); + + resBuilder.addData(kvBuilder + .setKey(ByteString.copyFrom(col.name)) + .setValue(ByteString.copyFrom(col.value)) + .setCode(HgStoreNodeUtil.toInt(iterator.position())) +//position == partition-id. + ); + + } + + responseObserver.onNext(resBuilder.build()); + responseObserver.onCompleted(); + + } catch (Throwable t) { + String msg = "an exception occurred during data scanning"; + responseObserver.onError(HgGrpc.toErr(Status.INTERNAL, msg, t)); + } finally { + iterator.close(); + } + + } + + +} \ No newline at end of file diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanQuery.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanQuery.java new file mode 100644 index 000000000..92838ae31 --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanQuery.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +import java.util.Arrays; + +import com.baidu.hugegraph.store.grpc.common.ScanMethod; + +/** + * @author lynn.bond@hotmail.com on 2022/2/28 + */ +class ScanQuery implements QueryCondition { + String graph; + String table; + ScanMethod method; + + byte[] start; + byte[] end; + byte[] prefix; + int keyCode; + int scanType; + byte[] query; + byte[] position; + int serialNo; + + @Override + public byte[] getStart() { + return this.start; + } + + @Override + public byte[] getEnd() { + return this.end; + } + + @Override + public byte[] getPrefix() { + return this.prefix; + } + + @Override + public int getKeyCode() { + return this.keyCode; + } + + @Override + public int getScanType() { + return this.scanType; + } + + @Override + public byte[] getQuery() { + return this.query; + } + + @Override + public byte[] getPosition() { + return this.position; + } + + @Override + public int getSerialNo() { + return this.serialNo; + } + + static ScanQuery of() { + return new ScanQuery(); + } + + private ScanQuery() { + } + + @Override + public String toString() { + return "ScanQuery{" + + "graph='" + graph + '\'' + + ", table='" + table + '\'' + + ", method=" + method + + ", start=" + Arrays.toString(start) + + ", end=" + Arrays.toString(end) + + ", prefix=" + Arrays.toString(prefix) + + ", partition=" + keyCode + + ", scanType=" + scanType + + ", serialNo=" + serialNo + + ", query=" + Arrays.toString(query) + + ", position=" + Arrays.toString(position) + + '}'; + } +} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanQueryProducer.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanQueryProducer.java similarity index 82% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanQueryProducer.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanQueryProducer.java index a6a456925..0f163ec52 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanQueryProducer.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanQueryProducer.java @@ -1,17 +1,37 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.node.util.HgAssert; -import com.baidu.hugegraph.store.grpc.common.ScanMethod; -import com.baidu.hugegraph.store.grpc.stream.ScanCondition; -import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.node.grpc; -import javax.annotation.concurrent.NotThreadSafe; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; +import javax.annotation.concurrent.NotThreadSafe; + +import org.apache.hugegraph.store.node.util.HgAssert; + +import com.baidu.hugegraph.store.grpc.common.ScanMethod; +import com.baidu.hugegraph.store.grpc.stream.ScanCondition; +import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest; + +import lombok.extern.slf4j.Slf4j; + /** * Buffering the data of ScanQueryRequest and generating ScanQuery. * It will not hold the reference of ScanQueryRequest. @@ -34,7 +54,8 @@ class ScanQueryProducer implements Iterable { private ScanQueryProducer() { } - public static ScanQueryProducer requestOf(String graph, String[] tables, ScanQueryRequest request) { + public static ScanQueryProducer requestOf(String graph, String[] tables, + ScanQueryRequest request) { HgAssert.isArgumentValid(graph, "graph"); HgAssert.isArgumentNotNull(tables, "tables"); HgAssert.isArgumentNotNull(request, "ScanQueryRequest"); @@ -124,7 +145,7 @@ class ScanQueryProducer implements Iterable { @Override public ScanQuery[] next() { - if (! this.hasNext()) { + if (!this.hasNext()) { throw new NoSuchElementException(); } @@ -141,7 +162,8 @@ class ScanQueryProducer implements Iterable { } private class GroupedConditionsIterator implements Iterator { - private Iterator conditionIterator = ScanQueryProducer.this.conditionList.iterator(); + private final Iterator conditionIterator = + ScanQueryProducer.this.conditionList.iterator(); @Override public boolean hasNext() { @@ -154,7 +176,8 @@ class ScanQueryProducer implements Iterable { ScanQuery[] res = new ScanQuery[ScanQueryProducer.this.tables.length]; for (int i = 0; i < res.length; i++) { - res[i] = ScanQueryProducer.this.createQuery(ScanQueryProducer.this.tables[i], condition); + res[i] = ScanQueryProducer.this.createQuery(ScanQueryProducer.this.tables[i], + condition); } return res; @@ -198,7 +221,8 @@ class ScanQueryProducer implements Iterable { * TODO: no testing */ private class ConditionsIterator implements Iterator { - private Iterator conditionIterator = ScanQueryProducer.this.conditionList.iterator(); + private final Iterator conditionIterator = + ScanQueryProducer.this.conditionList.iterator(); private ScanCondition condition; private String tableName; private int tableIndex; diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanStreamResponse.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanStreamResponse.java similarity index 78% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanStreamResponse.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanStreamResponse.java index 5a9f366cf..9bd6257b8 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanStreamResponse.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanStreamResponse.java @@ -1,21 +1,39 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import static com.baidu.hugegraph.store.node.grpc.ScanUtil.getIterator; +package org.apache.hugegraph.store.node.grpc; + +import static org.apache.hugegraph.store.node.grpc.ScanUtil.getIterator; import java.util.Collections; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicBoolean; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.node.AppConfig; +import org.apache.hugegraph.store.node.util.HgAssert; +import org.apache.hugegraph.store.node.util.HgChannel; +import org.apache.hugegraph.store.node.util.HgGrpc; +import org.apache.hugegraph.store.node.util.HgStoreNodeUtil; + import com.baidu.hugegraph.store.grpc.common.Kv; import com.baidu.hugegraph.store.grpc.stream.KvPageRes; import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq; -import com.baidu.hugegraph.store.node.AppConfig; -import com.baidu.hugegraph.store.node.util.HgAssert; -import com.baidu.hugegraph.store.node.util.HgChannel; -import com.baidu.hugegraph.store.node.util.HgGrpc; -import com.baidu.hugegraph.store.node.util.HgStoreNodeUtil; import com.google.protobuf.ByteString; import io.grpc.Status; @@ -40,12 +58,13 @@ public class ScanStreamResponse implements StreamObserver { private int total = 0; private String graph; private String table; - private AtomicBoolean isStarted = new AtomicBoolean(); - private AtomicBoolean isStop = new AtomicBoolean(false); - private AppConfig config; - private int waitTime; - private HgChannel channel; - private static String msg = "to wait for client taking data exceeded max time: [{}] seconds,stop scanning."; + private final AtomicBoolean isStarted = new AtomicBoolean(); + private final AtomicBoolean isStop = new AtomicBoolean(false); + private final AppConfig config; + private final int waitTime; + private final HgChannel channel; + private static final String msg = + "to wait for client taking data exceeded max time: [{}] seconds,stop scanning."; public static ScanStreamResponse of(StreamObserver responseObserver, HgStoreWrapperEx wrapper, @@ -84,7 +103,8 @@ public class ScanStreamResponse implements StreamObserver { public void onError(Throwable t) { this.isStop.set(true); this.finishServer(); - log.warn("onError from client [ graph: {} , table: {}]; Reason: {}]", graph, table, t.getMessage()); + log.warn("onError from client [ graph: {} , table: {}]; Reason: {}]", graph, table, + t.getMessage()); } @Override @@ -104,7 +124,9 @@ public class ScanStreamResponse implements StreamObserver { this.limit = request.getLimit(); this.pageSize = request.getPageSize(); if (this.pageSize <= 0) { - log.warn("As page-Size is less than or equals 0, no data will be send to the client."); + log.warn( + "As page-Size is less than or equals 0, no data will be send to the " + + "client."); } /*** Start scanning loop ***/ Runnable scanning = () -> @@ -123,7 +145,7 @@ public class ScanStreamResponse implements StreamObserver { if (++pageCount > pageSize) { long start = System.currentTimeMillis(); if (!this.channel.send(dataBuilder)) { - if (System.currentTimeMillis() - start >= waitTime * 1000) { + if (System.currentTimeMillis() - start >= waitTime * 1000L) { log.warn(msg, waitTime); this.timeoutSever(); } @@ -140,7 +162,8 @@ public class ScanStreamResponse implements StreamObserver { this.channel.send(dataBuilder); } catch (Throwable t) { String msg = "an exception occurred while scanning data:"; - StatusRuntimeException ex = HgGrpc.toErr(Status.INTERNAL, msg + t.getMessage(), t); + StatusRuntimeException ex = + HgGrpc.toErr(Status.INTERNAL, msg + t.getMessage(), t); responseObserver.onError(ex); } finally { try { @@ -162,8 +185,6 @@ public class ScanStreamResponse implements StreamObserver { } catch (Exception exception) { } - } finally { - } /*** Scanning loop end ***/ @@ -207,7 +228,8 @@ public class ScanStreamResponse implements StreamObserver { return; } boolean isOver = false; - if (resBuilder == null || resBuilder.getDataList() == null || resBuilder.getDataList().isEmpty()) { + if (resBuilder == null || resBuilder.getDataList() == null || + resBuilder.getDataList().isEmpty()) { isOver = true; resBuilder = KvPageRes.newBuilder().addAllData(Collections.EMPTY_LIST); } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanUtil.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanUtil.java similarity index 56% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanUtil.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanUtil.java index a789a7fed..10c9e93af 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanUtil.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanUtil.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.node.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; import java.util.Arrays; import java.util.Collections; @@ -14,14 +31,14 @@ import java.util.stream.Collectors; import javax.annotation.concurrent.NotThreadSafe; +import org.apache.hugegraph.rocksdb.access.ScanIterator; + import com.baidu.hugegraph.pd.common.KVPair; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.store.business.SelectIterator; import com.baidu.hugegraph.store.grpc.common.ScanMethod; import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest; import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq; import com.baidu.hugegraph.store.grpc.stream.SelectParam; -import com.baidu.hugegraph.store.node.util.HgStoreNodeUtil; import lombok.extern.slf4j.Slf4j; @@ -82,10 +99,12 @@ class ScanUtil { iter = wrapper.scanAll(sq.graph, sq.table, sq.query); break; case PREFIX: - iter = wrapper.scanPrefix(sq.graph, sq.keyCode, sq.table, sq.prefix, sq.scanType, sq.query); + iter = wrapper.scanPrefix(sq.graph, sq.keyCode, sq.table, sq.prefix, sq.scanType, + sq.query); break; case RANGE: - iter = wrapper.scan(sq.graph, sq.keyCode, sq.table, sq.start, sq.end, sq.scanType, sq.query); + iter = wrapper.scan(sq.graph, sq.keyCode, sq.table, sq.start, sq.end, sq.scanType, + sq.query); break; } @@ -121,7 +140,8 @@ class ScanUtil { return res; } - static ScanIterator getIterator(String graph, ScanQueryRequest request, HgStoreWrapperEx wrapper) { + static ScanIterator getIterator(String graph, ScanQueryRequest request, + HgStoreWrapperEx wrapper) { ScanIteratorSupplier supplier = new ScanIteratorSupplier(graph, request, wrapper); return BatchScanIterator.of(supplier, supplier.getLimitSupplier()); } @@ -133,19 +153,20 @@ class ScanUtil { HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) { ScanIteratorSupplier supplier = new ScanIteratorSupplier(graph, request, wrapper); return ParallelScanIterator.of(supplier, supplier.getLimitSupplier(), - request, executor); + request, executor); } @NotThreadSafe - private static class ScanIteratorSupplier implements Supplier> { + private static class ScanIteratorSupplier implements + Supplier> { - private AtomicBoolean isEmpty = new AtomicBoolean(); + private final AtomicBoolean isEmpty = new AtomicBoolean(); - private String graph; + private final String graph; private long perKeyLimit; - private long perKeyMax; - private long skipDegree; - private HgStoreWrapperEx wrapper; + private final long perKeyMax; + private final long skipDegree; + private final HgStoreWrapperEx wrapper; private List sqs = new LinkedList<>(); private Iterator sqIterator; @@ -156,14 +177,15 @@ class ScanUtil { ScanIteratorSupplier(String graph, ScanQueryRequest request, HgStoreWrapperEx wrapper) { this.graph = graph; this.perKeyLimit = request.getPerKeyLimit(); - this.perKeyMax=request.getPerKeyMax(); + this.perKeyMax = request.getPerKeyMax(); this.skipDegree = request.getSkipDegree() == 0 ? Integer.MAX_VALUE : request.getSkipDegree(); this.wrapper = wrapper; if (this.perKeyLimit <= 0) { this.perKeyLimit = Integer.MAX_VALUE; - log.warn("as perKeyLimit <=0 so default perKeyLimit was effective: {}", Integer.MAX_VALUE); + log.warn("as perKeyLimit <=0 so default perKeyLimit was effective: {}", + Integer.MAX_VALUE); } //init(request); init2(request); @@ -171,47 +193,61 @@ class ScanUtil { private void init(ScanQueryRequest request) { this.sqs = Arrays.stream(request.getTable().split(",")) - .map(table -> { - if (table == null) return null; - if (table.isEmpty()) return null; + .map(table -> { + if (table == null) return null; + if (table.isEmpty()) return null; - List list = request.getConditionList() - .stream() - .map(condition -> { - ScanQuery sq = ScanQuery.of(); - sq.graph = this.graph; - sq.table = table; - sq.method = request.getMethod(); - sq.scanType = request.getScanType(); - sq.query = request.getQuery().toByteArray(); - sq.position = request.getPosition().toByteArray(); + List list = request.getConditionList() + .stream() + .map(condition -> { + ScanQuery sq = + ScanQuery.of(); + sq.graph = this.graph; + sq.table = table; + sq.method = + request.getMethod(); + sq.scanType = + request.getScanType(); + sq.query = + request.getQuery() + .toByteArray(); + sq.position = + request.getPosition() + .toByteArray(); - sq.keyCode = condition.getCode(); - sq.start = condition.getStart().toByteArray(); - sq.end = condition.getEnd().toByteArray(); - sq.prefix = condition.getPrefix().toByteArray(); - sq.serialNo = condition.getSerialNo(); - return sq; - }) - .filter(e -> e != null) - .collect(Collectors.toList()); + sq.keyCode = + condition.getCode(); + sq.start = + condition.getStart() + .toByteArray(); + sq.end = condition.getEnd() + .toByteArray(); + sq.prefix = + condition.getPrefix() + .toByteArray(); + sq.serialNo = + condition.getSerialNo(); + return sq; + }) + .filter(e -> e != null) + .collect(Collectors.toList()); - if (list == null || list.isEmpty()) { - ScanQuery sq = ScanQuery.of(); - sq.graph = this.graph; - sq.table = table; - sq.method = request.getMethod(); - sq.scanType = request.getScanType(); - sq.query = request.getQuery().toByteArray(); - sq.position = request.getPosition().toByteArray(); - list = Collections.singletonList(sq); - } - return list; + if (list == null || list.isEmpty()) { + ScanQuery sq = ScanQuery.of(); + sq.graph = this.graph; + sq.table = table; + sq.method = request.getMethod(); + sq.scanType = request.getScanType(); + sq.query = request.getQuery().toByteArray(); + sq.position = request.getPosition().toByteArray(); + list = Collections.singletonList(sq); + } + return list; - } - ) - .flatMap(e -> e.stream()) - .collect(Collectors.toList()); + } + ) + .flatMap(e -> e.stream()) + .collect(Collectors.toList()); this.sqIterator = this.sqs.iterator(); } @@ -239,8 +275,8 @@ class ScanUtil { private void init2(ScanQueryRequest request) { List tableList = Arrays.stream(request.getTable().split(",")) - .filter(e -> e != null && !e.isEmpty()) - .collect(Collectors.toList()); + .filter(e -> e != null && !e.isEmpty()) + .collect(Collectors.toList()); if (tableList.isEmpty()) { throw new RuntimeException("table name is invalid"); diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/scan/GraphStoreImpl.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/GraphStoreImpl.java similarity index 65% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/scan/GraphStoreImpl.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/GraphStoreImpl.java index c6a655bb5..240799007 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/scan/GraphStoreImpl.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/GraphStoreImpl.java @@ -1,7 +1,26 @@ -package com.baidu.hugegraph.store.node.grpc.scan; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc.scan; import java.util.concurrent.ThreadPoolExecutor; +import org.apache.hugegraph.store.node.grpc.HgStoreNodeService; +import org.apache.hugegraph.store.node.grpc.HgStoreStreamImpl; import org.lognet.springboot.grpc.GRpcService; import org.springframework.beans.factory.annotation.Autowired; @@ -11,8 +30,6 @@ import com.baidu.hugegraph.store.grpc.Graphpb; import com.baidu.hugegraph.store.grpc.Graphpb.ResponseHeader; import com.baidu.hugegraph.store.grpc.Graphpb.ScanPartitionRequest; import com.baidu.hugegraph.store.grpc.Graphpb.ScanResponse; -import com.baidu.hugegraph.store.node.grpc.HgStoreNodeService; -import com.baidu.hugegraph.store.node.grpc.HgStoreStreamImpl; import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; @@ -30,7 +47,7 @@ public class GraphStoreImpl extends GraphStoreImplBase { private HgStoreStreamImpl storeStream; BusinessHandler handler; - private ResponseHeader okHeader = + private final ResponseHeader okHeader = ResponseHeader.newBuilder().setError( Graphpb.Error.newBuilder().setType(Graphpb.ErrorType.OK)) .build(); diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/scan/ScanResponseObserver.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/ScanResponseObserver.java similarity index 84% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/scan/ScanResponseObserver.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/ScanResponseObserver.java index c9284b0eb..3c2c79d72 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/scan/ScanResponseObserver.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/ScanResponseObserver.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.node.grpc.scan; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc.scan; import java.util.ArrayList; import java.util.concurrent.Future; @@ -9,9 +26,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; -import com.baidu.hugegraph.store.business.GraphStoreIterator; import com.baidu.hugegraph.store.business.BusinessHandler; -import com.baidu.hugegraph.store.grpc.Graphpb.Error; +import com.baidu.hugegraph.store.business.GraphStoreIterator; import com.baidu.hugegraph.store.grpc.Graphpb.ErrorType; import com.baidu.hugegraph.store.grpc.Graphpb.ResponseHeader; import com.baidu.hugegraph.store.grpc.Graphpb.ScanPartitionRequest; @@ -35,26 +51,26 @@ public class ScanResponseObserver implements private final BusinessHandler handler; private static final int BATCH_SIZE = 100000; private static final int MAX_PAGE = 8; // - private AtomicInteger nextSeqNo = new AtomicInteger(0); - private AtomicInteger cltSeqNo = new AtomicInteger(0); + private final AtomicInteger nextSeqNo = new AtomicInteger(0); + private final AtomicInteger cltSeqNo = new AtomicInteger(0); private final ThreadPoolExecutor executor; private ScanPartitionRequest scanReq; private GraphStoreIterator iter; - private static Error ok = Error.newBuilder().setType(ErrorType.OK).build(); - private static ResponseHeader okHeader = + private static final Error ok = Error.newBuilder().setType(ErrorType.OK).build(); + private static final ResponseHeader okHeader = ResponseHeader.newBuilder().setError(ok).build(); private volatile long leftCount; - private volatile AtomicBoolean readOver = new AtomicBoolean(false); + private final AtomicBoolean readOver = new AtomicBoolean(false); private volatile Future sendTask; private volatile Future readTask; private final LinkedBlockingQueue packages = new LinkedBlockingQueue(MAX_PAGE * 2); - private Descriptors.FieldDescriptor vertexField = + private final Descriptors.FieldDescriptor vertexField = ScanResponse.getDescriptor().findFieldByNumber(3); - private Descriptors.FieldDescriptor edgeField = + private final Descriptors.FieldDescriptor edgeField = ScanResponse.getDescriptor().findFieldByNumber(4); - private ReentrantLock readLock =new ReentrantLock(); - private ReentrantLock sendLock =new ReentrantLock(); + private final ReentrantLock readLock = new ReentrantLock(); + private final ReentrantLock sendLock = new ReentrantLock(); /* * 2022年11月1日 @@ -116,7 +132,7 @@ public class ScanResponseObserver implements } } } catch (Exception e) { - log.warn("read data with error: ",e); + log.warn("read data with error: ", e); sender.onError(e); } } @@ -246,8 +262,6 @@ public class ScanResponseObserver implements iter.close(); } catch (Exception e) { log.warn("on Complete with error:", e); - } finally { - } } } diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/ContextClosedListener.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/ContextClosedListener.java new file mode 100644 index 000000000..174d948cf --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/ContextClosedListener.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.listener; + +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.hugegraph.store.node.grpc.HgStoreStreamImpl; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationListener; +import org.springframework.context.event.ContextClosedEvent; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author zhangyingjie + * @date 2023/2/17 + **/ +@Slf4j +public class ContextClosedListener implements ApplicationListener { + + @Autowired + HgStoreStreamImpl storeStream; + + @Override + public void onApplicationEvent(ContextClosedEvent event) { + try { + log.info("closing scan threads...."); + ThreadPoolExecutor executor = storeStream.getRealExecutor(); + if (executor != null) { + try { + executor.shutdownNow(); + } catch (Exception e) { + + } + } + } catch (Exception ignored) { + + } finally { + log.info("closed scan threads"); + } + } +} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/listener/PdConfigureListener.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/PdConfigureListener.java similarity index 81% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/listener/PdConfigureListener.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/PdConfigureListener.java index 1dd25e24b..4c2fc3501 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/listener/PdConfigureListener.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/PdConfigureListener.java @@ -1,20 +1,21 @@ -package com.baidu.hugegraph.store.node.listener; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.pd.client.KvClient; -import com.baidu.hugegraph.pd.client.PDConfig; -import com.baidu.hugegraph.pd.common.PDException; -import com.baidu.hugegraph.pd.grpc.kv.ScanPrefixResponse; -import com.baidu.hugegraph.pd.grpc.kv.WatchResponse; -import com.google.common.base.Charsets; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; -import org.springframework.boot.context.event.ApplicationEnvironmentPreparedEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.core.env.MutablePropertySources; -import org.springframework.core.env.PropertiesPropertySource; -import org.yaml.snakeyaml.Yaml; +package org.apache.hugegraph.store.node.listener; import java.io.File; import java.io.IOException; @@ -26,24 +27,44 @@ import java.util.Map; import java.util.Properties; import java.util.function.Consumer; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.springframework.boot.context.event.ApplicationEnvironmentPreparedEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.core.env.MutablePropertySources; +import org.springframework.core.env.PropertiesPropertySource; +import org.yaml.snakeyaml.Yaml; + +import com.baidu.hugegraph.pd.client.KvClient; +import com.baidu.hugegraph.pd.client.PDConfig; +import com.baidu.hugegraph.pd.common.PDException; +import com.baidu.hugegraph.pd.grpc.kv.ScanPrefixResponse; +import com.baidu.hugegraph.pd.grpc.kv.WatchResponse; +import com.google.common.base.Charsets; + +import lombok.extern.slf4j.Slf4j; + /** * @author zhangyingjie * @date 2022/7/5 **/ @Slf4j -public class PdConfigureListener implements ApplicationListener { +public class PdConfigureListener implements + ApplicationListener { private static final String CONFIG_PREFIX = "S:"; private static final String CONFIG_FIX_PREFIX = "S:FS"; private ConfigurableApplicationContext context; private static final String TIMESTAMP_KEY = "S:Timestamp"; private static final String PD_CONFIG_FILE_NAME = "application-pd.yml"; - private String workDir = System.getProperty("user.dir"); - private String fileSeparator = System.getProperty("file.separator"); + private final String workDir = System.getProperty("user.dir"); + private final String fileSeparator = System.getProperty("file.separator"); private File pdConfFile; - private String configFilePath = workDir + fileSeparator + "conf" + fileSeparator + PD_CONFIG_FILE_NAME; - private String restartShellPath = workDir + fileSeparator + "bin" + fileSeparator - + "restart-hugegraph-store.sh"; + private final String configFilePath = + workDir + fileSeparator + "conf" + fileSeparator + PD_CONFIG_FILE_NAME; + private final String restartShellPath = workDir + fileSeparator + "bin" + fileSeparator + + "restart-hugegraph-store.sh"; // private String restartPath = workDir + fileSeparator + "lib" + fileSeparator; @Override @@ -62,7 +83,8 @@ public class PdConfigureListener implements ApplicationListener appSourceMap = (Map) appSource + // Map appSourceMap = (Map) + // appSource // .getSource(); // OriginTrackedValue pdTrackedValue = appSourceMap.get("pdserver.address"); // String pdAddress = pdTrackedValue.getValue().toString(); @@ -102,9 +124,10 @@ public class PdConfigureListener implements ApplicationListener> metrics() { + File[] rootDrive = File.listRoots(); + + if (rootDrive == null) { + return new LinkedHashMap(0); + } + + Map> metrics = new HashMap<>(); + + for (File d : rootDrive) { + Map buf = new HashMap<>(); + buf.put("total_space", d.getTotalSpace() / MIB); + buf.put("free_space", d.getFreeSpace() / MIB); + buf.put("usable_space", d.getUsableSpace() / MIB); + buf.put("size_unit", "MB"); + + metrics.put(d.getPath().replace("\\", ""), buf); + + } + + return metrics; + + } + +} diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/GRpcExMetrics.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/GRpcExMetrics.java new file mode 100644 index 000000000..3e9122f9a --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/GRpcExMetrics.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.metrics; + +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.hugegraph.store.node.grpc.GRpcServerConfig; +import org.apache.hugegraph.store.node.util.HgExecutorUtil; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; + +/** + * @author lynn.bond@hotmail.com on 2022/3/8 + */ +public class GRpcExMetrics { + public final static String PREFIX = "grpc"; + private static MeterRegistry registry; + private final static ExecutorWrapper wrapper = new ExecutorWrapper(); + + private GRpcExMetrics() { + } + + public synchronized static void init(MeterRegistry meterRegistry) { + if (registry == null) { + registry = meterRegistry; + registerMeters(); + } + } + + private static void registerMeters() { + registerExecutor(); + + } + + private static void registerExecutor() { + + Gauge.builder(PREFIX + ".executor.pool.size", wrapper, (e) -> e.getPoolSize()) + .description("The current number of threads in the pool.") + .register(registry); + + Gauge.builder(PREFIX + ".executor.core.pool.size", wrapper, (e) -> e.getCorePoolSize()) + .description( + "The largest number of threads that have ever simultaneously been in the " + + "pool.") + .register(registry); + + Gauge.builder(PREFIX + ".executor.active.count", wrapper, (e) -> e.getActiveCount()) + .description("The approximate number of threads that are actively executing tasks.") + .register(registry); + } + + private static class ExecutorWrapper { + ThreadPoolExecutor pool; + + void init() { + if (this.pool == null) { + pool = HgExecutorUtil.getThreadPoolExecutor(GRpcServerConfig.EXECUTOR_NAME); + } + } + + double getPoolSize() { + init(); + return this.pool == null ? 0d : this.pool.getPoolSize(); + } + + int getCorePoolSize() { + init(); + return this.pool == null ? 0 : this.pool.getCorePoolSize(); + } + + int getActiveCount() { + init(); + return this.pool == null ? 0 : this.pool.getActiveCount(); + } + + } + +} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/JRaftMetrics.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/JRaftMetrics.java similarity index 63% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/JRaftMetrics.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/JRaftMetrics.java index 30877bc39..0c14663eb 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/JRaftMetrics.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/JRaftMetrics.java @@ -1,22 +1,45 @@ -package com.baidu.hugegraph.store.node.metrics; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.metrics; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hugegraph.store.node.util.HgRegexUtil; import com.alipay.sofa.jraft.core.NodeMetrics; import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.node.util.HgRegexUtil; import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; import com.codahale.metrics.Snapshot; import com.codahale.metrics.Timer; + import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; import lombok.extern.slf4j.Slf4j; -import java.util.*; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiFunction; - /** * @author lynn.bond@hotmail.com on 2022/1/4 */ @@ -52,9 +75,9 @@ public class JRaftMetrics { private static void registerMeters() { Gauge.builder(PREFIX + ".groups", JRaftMetrics::updateGroups) - .description("Number of raft-groups, which handled the data of graph.") - .tags(Collections.singleton(handleDataTag)) - .register(registry); + .description("Number of raft-groups, which handled the data of graph.") + .tags(Collections.singleton(handleDataTag)) + .register(registry); } @@ -88,18 +111,23 @@ public class JRaftMetrics { map.forEach((group, metrics) -> { if (!groupSet.add(group)) return; - metrics.getMetricRegistry().getGauges().forEach((k, v) -> registerGauge(group, k, v)); - metrics.getMetricRegistry().getMeters().forEach((k, v) -> registerMeter(group, k, v)); - metrics.getMetricRegistry().getCounters().forEach((k, v) -> registerCounter(group, k, v)); - metrics.getMetricRegistry().getTimers().forEach((k, v) -> registerTimer(group, k, v)); - metrics.getMetricRegistry().getHistograms().forEach((k, v) -> registerHistogram(group, k, v)); + metrics.getMetricRegistry().getGauges() + .forEach((k, v) -> registerGauge(group, k, v)); + metrics.getMetricRegistry().getMeters() + .forEach((k, v) -> registerMeter(group, k, v)); + metrics.getMetricRegistry().getCounters() + .forEach((k, v) -> registerCounter(group, k, v)); + metrics.getMetricRegistry().getTimers() + .forEach((k, v) -> registerTimer(group, k, v)); + metrics.getMetricRegistry().getHistograms() + .forEach((k, v) -> registerHistogram(group, k, v)); }); } } private static class HistogramWrapper { - private com.codahale.metrics.Histogram histogram; + private final com.codahale.metrics.Histogram histogram; private Snapshot snapshot; private long ts = System.currentTimeMillis(); @@ -147,7 +175,8 @@ public class JRaftMetrics { return res; } - private static void registerHistogram(String group, String name, com.codahale.metrics.Histogram histogram) { + private static void registerHistogram(String group, String name, + com.codahale.metrics.Histogram histogram) { if (histogram == null) return; List tags = new LinkedList<>(); @@ -158,35 +187,36 @@ public class JRaftMetrics { String baseName = PREFIX + "." + name.toLowerCase(); - HistogramWrapper wrapper=toWrapper(histogram); + HistogramWrapper wrapper = toWrapper(histogram); - Gauge.builder(baseName + ".median",wrapper, (d) -> d.getSnapshot().getMedian()) - .tags(tags).register(registry); + Gauge.builder(baseName + ".median", wrapper, (d) -> d.getSnapshot().getMedian()) + .tags(tags).register(registry); Gauge.builder(baseName + ".min", wrapper, (d) -> d.getSnapshot().getMin()) - .tags(tags).register(registry); + .tags(tags).register(registry); Gauge.builder(baseName + ".max", wrapper, (d) -> d.getSnapshot().getMax()) - .tags(tags).register(registry); + .tags(tags).register(registry); Gauge.builder(baseName + ".mean", wrapper, (d) -> d.getSnapshot().getMean()) - .tags(tags).register(registry); + .tags(tags).register(registry); baseName = baseName + ".summary"; Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().getMedian()) - .tags(tags).tag(LABELS, LABEL_50).register(registry); + .tags(tags).tag(LABELS, LABEL_50).register(registry); Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get75thPercentile()) - .tags(tags).tag(LABELS, LABEL_75).register(registry); + .tags(tags).tag(LABELS, LABEL_75).register(registry); Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get95thPercentile()) - .tags(tags).tag(LABELS, LABEL_95).register(registry); + .tags(tags).tag(LABELS, LABEL_95).register(registry); Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get98thPercentile()) - .tags(tags).tag(LABELS, LABEL_98).register(registry); + .tags(tags).tag(LABELS, LABEL_98).register(registry); Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get99thPercentile()) - .tags(tags).tag(LABELS, LABEL_99).register(registry); + .tags(tags).tag(LABELS, LABEL_99).register(registry); Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get999thPercentile()) - .tags(tags).tag(LABELS, LABEL_999).register(registry); + .tags(tags).tag(LABELS, LABEL_999).register(registry); - Gauge.builder(baseName + ".sum", wrapper, (d) -> Arrays.stream(d.getSnapshot().getValues()).sum()) - .tags(tags).register(registry); + Gauge.builder(baseName + ".sum", wrapper, + (d) -> Arrays.stream(d.getSnapshot().getValues()).sum()) + .tags(tags).register(registry); Gauge.builder(baseName + ".count", wrapper, (d) -> d.getSnapshot().size()) - .tags(tags).register(registry); + .tags(tags).register(registry); } @@ -200,16 +230,16 @@ public class JRaftMetrics { String baseName = PREFIX + "." + name.toLowerCase(); Gauge.builder(baseName + ".count", timer, Timer::getCount) - .tags(tags).register(registry); + .tags(tags).register(registry); Gauge.builder(baseName + ".timer", timer, Timer::getCount) - .tags(tags).tag("rate", "1m").register(registry); + .tags(tags).tag("rate", "1m").register(registry); Gauge.builder(baseName + ".timer", timer, Timer::getCount) - .tags(tags).tag("rate", "5m").register(registry); + .tags(tags).tag("rate", "5m").register(registry); Gauge.builder(baseName + ".timer", timer, Timer::getCount) - .tags(tags).tag("rate", "15m").register(registry); + .tags(tags).tag("rate", "15m").register(registry); Gauge.builder(baseName + ".timer", timer, Timer::getCount) - .tags(tags).tag("rate", "mean").register(registry); + .tags(tags).tag("rate", "mean").register(registry); } @@ -223,25 +253,26 @@ public class JRaftMetrics { String baseName = PREFIX + "." + name.toLowerCase(); Gauge.builder(baseName + ".count", meter, Meter::getCount) - .tags(tags) - .register(registry); + .tags(tags) + .register(registry); Gauge.builder(baseName + ".rate", meter, Meter::getCount) - .tags(tags).tag("rate", "1m") - .register(registry); + .tags(tags).tag("rate", "1m") + .register(registry); Gauge.builder(baseName + ".rate", meter, Meter::getCount) - .tags(tags).tag("rate", "5m") - .register(registry); + .tags(tags).tag("rate", "5m") + .register(registry); Gauge.builder(baseName + ".rate", meter, Meter::getCount) - .tags(tags).tag("rate", "15m") - .register(registry); + .tags(tags).tag("rate", "15m") + .register(registry); Gauge.builder(baseName + ".rate", meter, Meter::getCount) - .tags(tags).tag("rate", "mean") - .register(registry); + .tags(tags).tag("rate", "mean") + .register(registry); } - private static void registerCounter(String group, String name, com.codahale.metrics.Counter counter) { + private static void registerCounter(String group, String name, + com.codahale.metrics.Counter counter) { List tags = new LinkedList<>(); tags.add(handleDataTag); tags.add(Tag.of("group", group)); @@ -252,10 +283,11 @@ public class JRaftMetrics { //Adapted a counter to be a gauge. Gauge.builder(PREFIX + "." + name + ".count", counter, Counter::getCount) - .tags(tags).register(registry); + .tags(tags).register(registry); } - private static void registerGauge(String group, String name, com.codahale.metrics.Gauge gauge) { + private static void registerGauge(String group, String name, + com.codahale.metrics.Gauge gauge) { List tags = new LinkedList<>(); tags.add(handleDataTag); tags.add(Tag.of("group", group)); @@ -266,11 +298,11 @@ public class JRaftMetrics { if (gauge.getValue() instanceof Number) { Gauge.builder(PREFIX + "." + name, gauge, (g) -> ((Number) g.getValue()).doubleValue()) - .tags(tags).register(registry); + .tags(tags).register(registry); } else { Gauge.builder(PREFIX + "." + name, () -> 1.0) - .tags(tags) - .tag("str.gauge", String.valueOf(gauge.getValue())).register(registry); + .tags(tags) + .tag("str.gauge", String.valueOf(gauge.getValue())).register(registry); } } diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/MetricsConfig.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/MetricsConfig.java new file mode 100644 index 000000000..a80741ab6 --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/MetricsConfig.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.metrics; + +import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import io.micrometer.core.instrument.MeterRegistry; + +/** + * @author lynn.bond@hotmail.com on 2021/11/24 + */ +@Configuration +public class MetricsConfig { + + @Bean + public MeterRegistryCustomizer metricsCommonTags() { + return (registry) -> registry.config().commonTags("hg", "store"); + } + + @Bean + public MeterRegistryCustomizer registerMeters() { + return (registry) -> { + StoreMetrics.init(registry); + RocksDBMetrics.init(registry); + JRaftMetrics.init(registry); + ProcfsMetrics.init(registry); + GRpcExMetrics.init(registry); + }; + } + +} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/MetricsUtil.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/MetricsUtil.java similarity index 86% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/MetricsUtil.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/MetricsUtil.java index b22357f37..987d3080d 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/MetricsUtil.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/MetricsUtil.java @@ -1,6 +1,4 @@ /* - * Copyright 2017 HugeGraph Authors - * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -17,9 +15,14 @@ * under the License. */ -package com.baidu.hugegraph.store.node.metrics; +package org.apache.hugegraph.store.node.metrics; -import com.codahale.metrics.*; +import com.codahale.metrics.Counter; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; @Deprecated public class MetricsUtil { diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/ProcfsEntry.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsEntry.java similarity index 93% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/ProcfsEntry.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsEntry.java index edd615093..48e07934f 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/ProcfsEntry.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsEntry.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.baidu.hugegraph.store.node.metrics; +package org.apache.hugegraph.store.node.metrics; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.apache.hugegraph.store.node.metrics.ProcfsReader.ReadResult; import java.io.IOException; import java.util.Collection; import java.util.Objects; -import static com.baidu.hugegraph.store.node.metrics.ProcfsReader.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; abstract class ProcfsEntry { diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsMetrics.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsMetrics.java new file mode 100644 index 000000000..d434c90c7 --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsMetrics.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.metrics; + + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; + +/** + * @author lynn.bond@hotmail.com on 2022/3/1 + * @version 0.1.0 + */ +public class ProcfsMetrics { + + private static MeterRegistry registry; + public final static String PREFIX = "process_memory"; + + private final static ProcfsSmaps smaps = new ProcfsSmaps(); + + private ProcfsMetrics() { + + } + + public synchronized static void init(MeterRegistry meterRegistry) { + if (registry == null) { + registry = meterRegistry; + registerMeters(); + } + } + + private static void registerMeters() { + registerProcessGauge(); + } + + private static void registerProcessGauge() { + Gauge.builder(PREFIX + ".rss.bytes", () -> smaps.get(ProcfsSmaps.KEY.RSS)) + .register(registry); + + Gauge.builder(PREFIX + ".pss.bytes", () -> smaps.get(ProcfsSmaps.KEY.PSS)) + .register(registry); + + Gauge.builder(PREFIX + ".vss.bytes", () -> smaps.get(ProcfsSmaps.KEY.VSS)) + .register(registry); + + Gauge.builder(PREFIX + ".swap.bytes", () -> smaps.get(ProcfsSmaps.KEY.SWAP)) + .register(registry); + + Gauge.builder(PREFIX + ".swappss.bytes", () -> smaps.get(ProcfsSmaps.KEY.SWAPPSS)) + .register(registry); + } + +} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/ProcfsReader.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsReader.java similarity index 90% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/ProcfsReader.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsReader.java index 1163c63b0..a5ea7347f 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/ProcfsReader.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsReader.java @@ -13,13 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.baidu.hugegraph.store.node.metrics; +package org.apache.hugegraph.store.node.metrics; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; class ProcfsReader { @@ -56,7 +61,8 @@ class ProcfsReader { this.entryPath = base.resolve(entry); this.osSupport = forceOSSupport - || System.getProperty("os.name").toLowerCase(Locale.ENGLISH).startsWith("linux"); + || System.getProperty("os.name").toLowerCase(Locale.ENGLISH) + .startsWith("linux"); } /* default */ Path getEntryPath() { @@ -104,7 +110,8 @@ class ProcfsReader { return System.currentTimeMillis(); } - /* default */ static ProcfsReader getInstance(String entry) { + /* default */ + static ProcfsReader getInstance(String entry) { Objects.requireNonNull(entry); synchronized (instancesLock) { diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/ProcfsSmaps.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsSmaps.java similarity index 94% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/ProcfsSmaps.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsSmaps.java index 0573f4a86..9c34174e9 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/ProcfsSmaps.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsSmaps.java @@ -13,9 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.baidu.hugegraph.store.node.metrics; +package org.apache.hugegraph.store.node.metrics; -import java.util.*; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; import java.util.function.LongUnaryOperator; diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/RocksDBMetrics.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetrics.java similarity index 59% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/RocksDBMetrics.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetrics.java index f52a8f0be..47e78e2af 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/RocksDBMetrics.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetrics.java @@ -1,23 +1,56 @@ -package com.baidu.hugegraph.store.node.metrics; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.rocksdb.access.RocksDBFactory; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.node.util.HgAssert; -import io.micrometer.core.instrument.Gauge; -import io.micrometer.core.instrument.Meter; -import io.micrometer.core.instrument.MeterRegistry; -import lombok.extern.slf4j.Slf4j; -import org.rocksdb.*; +package org.apache.hugegraph.store.node.metrics; -import java.util.*; +import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.HISTOGRAMS; +import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABELS; +import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_50; +import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_95; +import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_99; +import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.PREFIX; +import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.TICKERS; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; -import static com.baidu.hugegraph.store.node.metrics.RocksDBMetricsConst.*; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.node.util.HgAssert; +import org.rocksdb.HistogramData; +import org.rocksdb.HistogramType; +import org.rocksdb.MemoryUsageType; +import org.rocksdb.Statistics; +import org.rocksdb.TickerType; + +import com.baidu.hugegraph.store.HgStoreEngine; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; /** * @author lynn.bond@hotmail.com on 2021/12/30 @@ -49,8 +82,8 @@ public class RocksDBMetrics { registry = meterRegistry; Gauge.builder("rocks.num", RocksDBMetrics::updateRocks) - .description("Number of instance of RocksDB running in this node") - .register(registry); + .description("Number of instance of RocksDB running in this node") + .register(registry); registerMemoryUse(); } @@ -83,7 +116,9 @@ public class RocksDBMetrics { Set graphs = getGraphs(); if (graphs == null) { - log.error("Failed to fetch the collection of names of graph, when invoking to register RocksDB gauge."); + log.error( + "Failed to fetch the collection of names of graph, when invoking to register " + + "RocksDB gauge."); return; } @@ -99,10 +134,10 @@ public class RocksDBMetrics { String gaugeName = PREFIX + "." + ticker.name().toLowerCase(); saveGraphMeter(g, - Gauge.builder(gaugeName, () -> stats.getTickerCount(ticker)) - .description("RocksDB reported statistics for " + ticker.name()) - .tag("graph", g) - .register(registry) + Gauge.builder(gaugeName, () -> stats.getTickerCount(ticker)) + .description("RocksDB reported statistics for " + ticker.name()) + .tag("graph", g) + .register(registry) ); } @@ -115,14 +150,14 @@ public class RocksDBMetrics { }); graphSet.removeAll(graphSet.stream().filter(g -> !graphs.contains(g)) - .peek(g -> removeMeters(g)) - .collect(Collectors.toList()) + .peek(g -> removeMeters(g)) + .collect(Collectors.toList()) ); } private static void saveGraphMeter(String g, Meter meter) { - graphMeterMap.computeIfAbsent(g,k->new HashSet<>()).add(meter); + graphMeterMap.computeIfAbsent(g, k -> new HashSet<>()).add(meter); } private static void removeMeters(String g) { @@ -132,39 +167,59 @@ public class RocksDBMetrics { private static void registerHistogram(String graph, MeterRegistry registry, HistogramType histogramType, StatisticsWrapper stats) { - HistogramDataWrapper histogram = new HistogramDataWrapper(histogramType, () -> stats.getHistogramData(histogramType)); + HistogramDataWrapper histogram = new HistogramDataWrapper(histogramType, + () -> stats.getHistogramData( + histogramType)); histogramHolder.put(histogram, histogramType); String baseName = PREFIX + "." + histogramType.name().toLowerCase(); - saveGraphMeter(graph, Gauge.builder(baseName + ".max", histogram, HistogramDataWrapper::getMax).tag("graph", graph).register(registry)); - saveGraphMeter(graph, Gauge.builder(baseName + ".mean", histogram, HistogramDataWrapper::getAverage).tag("graph", graph).register(registry)); - saveGraphMeter(graph, Gauge.builder(baseName + ".min", histogram, HistogramDataWrapper::getMin).tag("graph", graph).register(registry)); + saveGraphMeter(graph, + Gauge.builder(baseName + ".max", histogram, HistogramDataWrapper::getMax) + .tag("graph", graph).register(registry)); + saveGraphMeter(graph, Gauge.builder(baseName + ".mean", histogram, + HistogramDataWrapper::getAverage).tag("graph", graph) + .register(registry)); + saveGraphMeter(graph, + Gauge.builder(baseName + ".min", histogram, HistogramDataWrapper::getMin) + .tag("graph", graph).register(registry)); baseName = baseName + ".summary"; - saveGraphMeter(graph, Gauge.builder(baseName, histogram, HistogramDataWrapper::getMedian).tags("graph", graph, LABELS, LABEL_50).register(registry)); - saveGraphMeter(graph, Gauge.builder(baseName, histogram, HistogramDataWrapper::getPercentile95).tags("graph", graph, LABELS, LABEL_95).register(registry)); - saveGraphMeter(graph, Gauge.builder(baseName, histogram, HistogramDataWrapper::getPercentile99).tags("graph", graph, LABELS, LABEL_99).register(registry)); - saveGraphMeter(graph, Gauge.builder(baseName + ".sum", histogram, HistogramDataWrapper::getSum).tags("graph", graph).register(registry)); - saveGraphMeter(graph, Gauge.builder(baseName + ".count", histogram, HistogramDataWrapper::getCount).tags("graph", graph).register(registry)); + saveGraphMeter(graph, Gauge.builder(baseName, histogram, HistogramDataWrapper::getMedian) + .tags("graph", graph, LABELS, LABEL_50).register(registry)); + saveGraphMeter(graph, + Gauge.builder(baseName, histogram, HistogramDataWrapper::getPercentile95) + .tags("graph", graph, LABELS, LABEL_95).register(registry)); + saveGraphMeter(graph, + Gauge.builder(baseName, histogram, HistogramDataWrapper::getPercentile99) + .tags("graph", graph, LABELS, LABEL_99).register(registry)); + saveGraphMeter(graph, + Gauge.builder(baseName + ".sum", histogram, HistogramDataWrapper::getSum) + .tags("graph", graph).register(registry)); + saveGraphMeter(graph, + Gauge.builder(baseName + ".count", histogram, HistogramDataWrapper::getCount) + .tags("graph", graph).register(registry)); } private static void registerMemoryUse() { - Gauge.builder(PREFIX + ".table.reader.total", memoryUseWrapper, (e) -> e.getTableReaderTotal()) - .description("The current number of threads in the pool.") - .register(registry); + Gauge.builder(PREFIX + ".table.reader.total", memoryUseWrapper, + (e) -> e.getTableReaderTotal()) + .description("The current number of threads in the pool.") + .register(registry); Gauge.builder(PREFIX + ".mem.table.total", memoryUseWrapper, (e) -> e.getMemTableTotal()) - .description("The current number of threads in the pool.") - .register(registry); - Gauge.builder(PREFIX + ".mem.table.unFlushed", memoryUseWrapper, (e) -> e.getMemTableUnFlushed()) - .description("The current number of threads in the pool.") - .register(registry); + .description("The current number of threads in the pool.") + .register(registry); + Gauge.builder(PREFIX + ".mem.table.unFlushed", memoryUseWrapper, + (e) -> e.getMemTableUnFlushed()) + .description("The current number of threads in the pool.") + .register(registry); Gauge.builder(PREFIX + ".cache.total", memoryUseWrapper, (e) -> e.getCacheTotal()) - .description("The current number of threads in the pool.") - .register(registry); - Gauge.builder(PREFIX + ".block.cache.pinned-usage", memoryUseWrapper, (e) -> e.getProperty("rocksdb.block-cache-pinned-usage")) - .description("The current number of threads in the pool.") - .register(registry); + .description("The current number of threads in the pool.") + .register(registry); + Gauge.builder(PREFIX + ".block.cache.pinned-usage", memoryUseWrapper, + (e) -> e.getProperty("rocksdb.block-cache-pinned-usage")) + .description("The current number of threads in the pool.") + .register(registry); } @@ -172,11 +227,12 @@ public class RocksDBMetrics { SessionWrapper sessionWrapper = new SessionWrapper(graph); saveGraphMeter(graph, - Gauge.builder(PREFIX + ".session.ref.count", sessionWrapper, (e) -> e.getRefCount() - 1) - .description("The current amount of reference of session") - .tag("ref", "self").tag("graph",graph) - .strongReference(true) - .register(registry) + Gauge.builder(PREFIX + ".session.ref.count", sessionWrapper, + (e) -> e.getRefCount() - 1) + .description("The current amount of reference of session") + .tag("ref", "self").tag("graph", graph) + .strongReference(true) + .register(registry) ); } @@ -248,8 +304,8 @@ public class RocksDBMetrics { private final String graphName; long lastTime = 0; - private Map tickerCounteMap = new ConcurrentHashMap<>(); - private Map histogramDataMap = new ConcurrentHashMap<>(); + private final Map tickerCounteMap = new ConcurrentHashMap<>(); + private final Map histogramDataMap = new ConcurrentHashMap<>(); StatisticsWrapper(String graph) { @@ -258,21 +314,22 @@ public class RocksDBMetrics { } - private void loadData(){ + private void loadData() { if (System.currentTimeMillis() - lastTime < 30000) return; lastTime = System.currentTimeMillis(); try (RocksDBSession session = getRocksDBSession(graphName)) { if (session == null) { - // log.error("Failed to fetch the RocksDBSession with graph's name: [ " + graph + " ]"); - return ; + // log.error("Failed to fetch the RocksDBSession with graph's name: [ " + + // graph + " ]"); + return; } - Statistics statistics = session.getRocksDbStats(); - for (final TickerType ticker : TICKERS){ + Statistics statistics = session.getRocksDbStats(); + for (final TickerType ticker : TICKERS) { tickerCounteMap.put(ticker, statistics.getTickerCount(ticker)); } - for (final HistogramType histogram : HISTOGRAMS){ + for (final HistogramType histogram : HISTOGRAMS) { histogramDataMap.put(histogram, statistics.getHistogramData(histogram)); } } @@ -298,7 +355,7 @@ public class RocksDBMetrics { private static class HistogramDataWrapper { private HistogramData data = new HistogramData(0d, 0d, 0d, 0d, 0d); private long ts = System.currentTimeMillis() - 30_000; - private Supplier supplier; + private final Supplier supplier; private final HistogramType histogramType; HistogramDataWrapper(HistogramType histogramType, Supplier supplier) { diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/RocksDBMetricsConst.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java similarity index 84% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/RocksDBMetricsConst.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java index bf62e2e9f..553d42f93 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/RocksDBMetricsConst.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java @@ -1,13 +1,30 @@ -package com.baidu.hugegraph.store.node.metrics; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.metrics; import org.rocksdb.HistogramType; import org.rocksdb.TickerType; /** - * @author lynn.bond@hotmail.com on 2021/12/30 + * @author lynn.bond@hotmail.com on 2021/12/30 */ public final class RocksDBMetricsConst { - public static final String PREFIX="rocks.stats"; + public static final String PREFIX = "rocks.stats"; public static final String LABELS = "quantile"; public static final String LABEL_50 = "0.5"; public static final String LABEL_95 = "0.95"; @@ -109,7 +126,7 @@ public final class RocksDBMetricsConst { TickerType.NUMBER_RATE_LIMITER_DRAINS, TickerType.NUMBER_ITER_SKIP, TickerType.NUMBER_MULTIGET_KEYS_FOUND, - }; + }; // Histograms - treated as prometheus summaries public static final HistogramType[] HISTOGRAMS = { @@ -144,5 +161,5 @@ public final class RocksDBMetricsConst { HistogramType.COMPRESSION_TIMES_NANOS, HistogramType.DECOMPRESSION_TIMES_NANOS, HistogramType.READ_NUM_MERGE_OPERANDS, - }; + }; } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/StoreMetrics.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/StoreMetrics.java similarity index 62% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/StoreMetrics.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/StoreMetrics.java index 1655eb5da..a12c46dad 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/StoreMetrics.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/StoreMetrics.java @@ -1,16 +1,34 @@ -package com.baidu.hugegraph.store.node.metrics; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hugegraph.store.node.metrics; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.meta.Partition; -import io.micrometer.core.instrument.Gauge; -import io.micrometer.core.instrument.MeterRegistry; import java.util.Collections; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; +import com.baidu.hugegraph.store.HgStoreEngine; +import com.baidu.hugegraph.store.meta.Partition; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; + /** * @author lynn.bond@hotmail.com on 2021/12/28 */ @@ -33,8 +51,8 @@ public final class StoreMetrics { private static void registerMeters() { Gauge.builder(PREFIX + ".up", () -> 1).register(registry); Gauge.builder(PREFIX + ".graphs", StoreMetrics::updateGraphs) - .description("Number of graphs stored in this node") - .register(registry); + .description("Number of graphs stored in this node") + .register(registry); } @@ -55,9 +73,9 @@ public final class StoreMetrics { Map> map = getGraphPartitions(); map.forEach((k, v) -> Gauge.builder(PREFIX + ".partitions", new PartitionsGetter(k)) - .description("Number of partitions stored in the node") - .tag("graph", k) - .register(registry)); + .description("Number of partitions stored in the node") + .tag("graph", k) + .register(registry)); } @@ -71,7 +89,8 @@ public final class StoreMetrics { } private static Map> getGraphPartitions() { - Map> map = storeEngine.getPartitionManager().getPartitions(); + Map> map = + storeEngine.getPartitionManager().getPartitions(); if (map == null) { return Collections.emptyMap(); } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/SystemMetrics.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/SystemMetrics.java similarity index 87% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/SystemMetrics.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/SystemMetrics.java index 3bd239ea0..bcaa2d3bb 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/SystemMetrics.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/SystemMetrics.java @@ -1,6 +1,4 @@ /* - * Copyright 2017 HugeGraph Authors - * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -17,15 +15,20 @@ * under the License. */ -package com.baidu.hugegraph.store.node.metrics; +package org.apache.hugegraph.store.node.metrics; + +import java.lang.management.ClassLoadingMXBean; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryUsage; +import java.lang.management.ThreadMXBean; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; -import com.baidu.hugegraph.util.Bytes; import org.apache.commons.lang3.StringUtils; -import java.io.File; -import java.lang.management.*; -import java.util.*; -import java.util.stream.Collectors; +import com.baidu.hugegraph.util.Bytes; @Deprecated public class SystemMetrics { @@ -61,16 +64,16 @@ public class SystemMetrics { metrics.put("processors", runtime.availableProcessors()); metrics.put("uptime", ManagementFactory.getRuntimeMXBean().getUptime()); metrics.put("systemload_average", - ManagementFactory.getOperatingSystemMXBean() - .getSystemLoadAverage()); + ManagementFactory.getOperatingSystemMXBean() + .getSystemLoadAverage()); return metrics; } private static long totalNonHeapMemory() { try { return ManagementFactory.getMemoryMXBean() - .getNonHeapMemoryUsage() - .getCommitted(); + .getNonHeapMemoryUsage() + .getCommitted(); } catch (Throwable ignored) { return 0; } @@ -79,7 +82,7 @@ public class SystemMetrics { private Map getHeapMetrics() { Map metrics = new LinkedHashMap<>(); MemoryUsage memoryUsage = ManagementFactory.getMemoryMXBean() - .getHeapMemoryUsage(); + .getHeapMemoryUsage(); metrics.put("committed", memoryUsage.getCommitted() / MB); metrics.put("init", memoryUsage.getInit() / MB); metrics.put("used", memoryUsage.getUsed() / MB); @@ -90,7 +93,7 @@ public class SystemMetrics { private Map getNonHeapMetrics() { Map metrics = new LinkedHashMap<>(); MemoryUsage memoryUsage = ManagementFactory.getMemoryMXBean() - .getNonHeapMemoryUsage(); + .getNonHeapMemoryUsage(); metrics.put("committed", memoryUsage.getCommitted() / MB); metrics.put("init", memoryUsage.getInit() / MB); metrics.put("used", memoryUsage.getUsed() / MB); diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/model/HgNodeStatus.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/model/HgNodeStatus.java similarity index 52% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/model/HgNodeStatus.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/model/HgNodeStatus.java index dd697bdbc..57bc53c3d 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/model/HgNodeStatus.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/model/HgNodeStatus.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.node.model; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.model; import java.util.Objects; @@ -48,8 +65,8 @@ public class HgNodeStatus { @Override public String toString() { return "HgNodeStatus{" + - "status=" + status + - ", text='" + text + '\'' + - '}'; + "status=" + status + + ", text='" + text + '\'' + + '}'; } } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/Base58.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Base58.java similarity index 77% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/Base58.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Base58.java index e2f5feca4..5fdd6231c 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/Base58.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Base58.java @@ -1,7 +1,25 @@ -package com.baidu.hugegraph.store.node.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.util; import java.io.UnsupportedEncodingException; import java.math.BigInteger; +import java.nio.charset.StandardCharsets; /** * Created with IntelliJ IDEA. @@ -12,7 +30,8 @@ import java.math.BigInteger; */ public class Base58 { - public static final char[] ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray(); + public static final char[] ALPHABET = + "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray(); private static final int[] INDEXES = new int[128]; static { @@ -60,11 +79,7 @@ public class Base58 { } byte[] output = copyOfRange(temp, j, temp.length); - try { - return new String(output, "US-ASCII"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); // Cannot happen. - } + return new String(output, StandardCharsets.US_ASCII); } public static byte[] decode(String input) throws IllegalArgumentException { diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Err.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Err.java new file mode 100644 index 000000000..2c276f54e --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Err.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.util; + +/** + * @author lynn.bond@hotmail.com + */ +class Err { + private final String msg; + + public static Err of(String msg) { + return new Err(msg); + } + + private Err(String msg) { + this.msg = msg; + } + + @Override + public String toString() { + return "Err{" + + "msg='" + msg + '\'' + + '}'; + } +} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgAssert.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgAssert.java similarity index 68% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgAssert.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgAssert.java index 47a9b1554..5363d36b0 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgAssert.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgAssert.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.node.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.util; import java.util.Collection; import java.util.Map; @@ -10,7 +27,7 @@ import java.util.function.Supplier; public final class HgAssert { @Deprecated public static void isTrue(boolean expression, String message) { - if (message== null) throw new IllegalArgumentException("message is null"); + if (message == null) throw new IllegalArgumentException("message is null"); if (!expression) { throw new IllegalArgumentException(message); } @@ -46,15 +63,15 @@ public final class HgAssert { } public static void isArgumentValid(byte[] bytes, String parameter) { - isFalse(isInvalid(bytes), ()-> "The argument is invalid: " + parameter); + isFalse(isInvalid(bytes), () -> "The argument is invalid: " + parameter); } public static void isArgumentValid(String str, String parameter) { - isFalse(isInvalid(str), ()-> "The argument is invalid: " + parameter); + isFalse(isInvalid(str), () -> "The argument is invalid: " + parameter); } public static void isArgumentNotNull(Object obj, String parameter) { - isTrue(obj != null, ()-> "The argument is null: " + parameter); + isTrue(obj != null, () -> "The argument is null: " + parameter); } public static void istValid(byte[] bytes, String msg) { @@ -88,18 +105,15 @@ public final class HgAssert { } public static boolean isInvalid(byte[] bytes) { - if (bytes == null || bytes.length == 0) return true; - return false; + return bytes == null || bytes.length == 0; } public static boolean isInvalid(Map map) { - if (map == null || map.isEmpty()) return true; - return false; + return map == null || map.isEmpty(); } public static boolean isInvalid(Collection list) { - if (list == null || list.isEmpty()) return true; - return false; + return list == null || list.isEmpty(); } public static boolean isContains(Collection list, T item) { diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgBufferProxy.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgBufferProxy.java similarity index 72% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgBufferProxy.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgBufferProxy.java index b15395889..03cb5f721 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgBufferProxy.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgBufferProxy.java @@ -1,8 +1,22 @@ -package com.baidu.hugegraph.store.node.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.node.util; -import javax.annotation.CheckForNull; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executor; import java.util.concurrent.LinkedBlockingQueue; @@ -12,9 +26,12 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.function.Supplier; +import javax.annotation.CheckForNull; + +import lombok.extern.slf4j.Slf4j; + /** * @author lynn.bond@hotmail.com on 2022/3/15 - * @version 0.1.0 * @version 0.2.0 */ @Slf4j @@ -25,14 +42,14 @@ public final class HgBufferProxy { private final Runnable applier; private final Executor executor; - public static HgBufferProxy of(Executor executor,Runnable applier) { + public static HgBufferProxy of(Executor executor, Runnable applier) { HgAssert.isArgumentNotNull(applier, "applier"); HgAssert.isArgumentNotNull(executor, "executor"); - return new HgBufferProxy(executor,applier); + return new HgBufferProxy(executor, applier); } - private HgBufferProxy(Executor executor,Runnable applier) { - this.executor=executor; + private HgBufferProxy(Executor executor, Runnable applier) { + this.executor = executor; this.applier = applier; this.queue = new LinkedBlockingQueue<>(); } @@ -121,7 +138,8 @@ public final class HgBufferProxy { s = this.queue.poll(); } else { if (timeoutCallBack == null) { - throw new RuntimeException("Timeout, max time: " + timeoutSeconds + " seconds;"); + throw new RuntimeException( + "Timeout, max time: " + timeoutSeconds + " seconds;"); } else { timeoutCallBack.accept(timeoutSeconds); } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgChannel.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgChannel.java similarity index 78% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgChannel.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgChannel.java index 896e467ba..a49553658 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgChannel.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgChannel.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.node.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.util; import java.util.concurrent.BlockingQueue; import java.util.concurrent.SynchronousQueue; @@ -97,7 +114,8 @@ public final class HgChannel { } if (s == null) { if (timeoutCallBack == null) { - throw new RuntimeException("Timeout, max time: " + timeoutSeconds + " seconds;"); + throw new RuntimeException( + "Timeout, max time: " + timeoutSeconds + " seconds;"); } else { timeoutCallBack.accept(timeoutSeconds); } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgExecutorUtil.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgExecutorUtil.java similarity index 70% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgExecutorUtil.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgExecutorUtil.java index ea74d0667..7a67be4d6 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgExecutorUtil.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgExecutorUtil.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.node.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.util; import java.util.Map; import java.util.concurrent.BlockingQueue; @@ -31,7 +48,8 @@ public final class HgExecutorUtil { return threadPoolMap.get(name); } - public static ThreadPoolExecutor createExecutor(String name, int coreThreads, int maxThreads, int queueSize) { + public static ThreadPoolExecutor createExecutor(String name, int coreThreads, int maxThreads, + int queueSize) { ThreadPoolExecutor res = threadPoolMap.get(name); if (res != null) { return res; @@ -47,7 +65,8 @@ public final class HgExecutorUtil { } else { queue = new LinkedBlockingQueue<>(queueSize); } - res = new ThreadPoolExecutor(coreThreads, maxThreads, 60L, TimeUnit.SECONDS, queue, newThreadFactory(name)); + res = new ThreadPoolExecutor(coreThreads, maxThreads, 60L, TimeUnit.SECONDS, queue, + newThreadFactory(name)); threadPoolMap.put(name, res); } return res; diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgGrpc.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgGrpc.java similarity index 71% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgGrpc.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgGrpc.java index 56e721056..c136a3b52 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgGrpc.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgGrpc.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.node.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.util; import com.baidu.hugegraph.store.grpc.common.Key; import com.baidu.hugegraph.store.grpc.common.Kv; @@ -19,7 +36,7 @@ import jline.internal.Log; * @author lynn.bond@hotmail.com */ public abstract class HgGrpc { - private static ResCode OK = ResCode.RES_CODE_OK; + private static final ResCode OK = ResCode.RES_CODE_OK; public static ResStatus not() { return toStatus(ResCode.RES_CODE_NOT_EXIST, "not exist"); @@ -43,8 +60,8 @@ public abstract class HgGrpc { public static ResStatus toStatus(ResCode code, String msg) { return ResStatus.newBuilder() - .setCode(code) - .setMsg(msg).build(); + .setCode(code) + .setMsg(msg).build(); } public static HgPair toHgPair(Key key) { @@ -63,9 +80,9 @@ public abstract class HgGrpc { public static Kv toKv(HgPair pair, Kv.Builder builder) { return builder.clear() - .setKey(ByteString.copyFrom(pair.getKey())) - .setValue(ByteString.copyFrom(pair.getValue())) - .build(); + .setKey(ByteString.copyFrom(pair.getKey())) + .setValue(ByteString.copyFrom(pair.getValue())) + .build(); } public static HgPair toHgPair(Tse tse) { @@ -75,7 +92,7 @@ public abstract class HgGrpc { ); } - public static StatusRuntimeException toErr(String msg){ + public static StatusRuntimeException toErr(String msg) { return toErr(Status.INTERNAL, msg, null); } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgRegexUtil.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgRegexUtil.java similarity index 65% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgRegexUtil.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgRegexUtil.java index 9b7842657..c83f93d72 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgRegexUtil.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgRegexUtil.java @@ -1,7 +1,23 @@ -package com.baidu.hugegraph.store.node.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.util; import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -60,7 +76,8 @@ public class HgRegexUtil { public static void main(String[] args) { - List res = toGroupValues("(replicator)(.+?:\\d+)(.*)", "replicator_10.14.139.10:8081_append_entries_times"); + List res = toGroupValues("(replicator)(.+?:\\d+)(.*)", + "replicator_10.14.139.10:8081_append_entries_times"); if (res != null) { res.stream().forEach(e -> System.out.println(e)); } diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgStoreConst.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgStoreConst.java new file mode 100644 index 000000000..ee569ff85 --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgStoreConst.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.util; + +import java.util.Collections; +import java.util.List; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/22 + */ +public final class HgStoreConst { + + public final static int SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS = 300; + + public final static byte[] EMPTY_BYTES = new byte[0]; + + public static final List EMPTY_LIST = Collections.EMPTY_LIST; + + public final static int SCAN_ALL_PARTITIONS_ID = -1; // means scan all partitions. + + private HgStoreConst() { + } + +} diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgStoreNodeUtil.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgStoreNodeUtil.java similarity index 62% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgStoreNodeUtil.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgStoreNodeUtil.java index a153ac7d7..485e7a5db 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgStoreNodeUtil.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgStoreNodeUtil.java @@ -1,11 +1,28 @@ -package com.baidu.hugegraph.store.node.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.node.util; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.UUID; +import lombok.extern.slf4j.Slf4j; + /** * @author lynn.bond@hotmail.com created on 2021/10/29 */ @@ -52,7 +69,7 @@ public final class HgStoreNodeUtil { try { return UUID.nameUUIDFromBytes(bytes).toString(); } catch (Throwable t) { - log.error("Failed to parse bytes to UUID",t); + log.error("Failed to parse bytes to UUID", t); } return null; } diff --git a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/PropertyUtil.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/PropertyUtil.java similarity index 66% rename from hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/PropertyUtil.java rename to hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/PropertyUtil.java index 01fe012fa..5d24ba2aa 100644 --- a/hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/PropertyUtil.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/PropertyUtil.java @@ -1,13 +1,31 @@ -package com.baidu.hugegraph.store.node.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +package org.apache.hugegraph.store.node.util; import java.security.AccessController; import java.security.PrivilegedAction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class PropertyUtil { private static final Logger LOG = LoggerFactory.getLogger(PropertyUtil.class); + public static String get(String key) { return get(key, null); } @@ -25,7 +43,8 @@ public class PropertyUtil { if (System.getSecurityManager() == null) { value = System.getProperty(key); } else { - value = AccessController.doPrivileged((PrivilegedAction) () -> System.getProperty(key)); + value = AccessController.doPrivileged( + (PrivilegedAction) () -> System.getProperty(key)); } } catch (Exception e) { LOG.error("exception {}", e); diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Result.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Result.java new file mode 100644 index 000000000..5a98e48a4 --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Result.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.util; + +/** + * @author lynn.bond@hotmail.com + */ +public class Result { + private Err err; + private T t; + + public static Result of() { + return new Result(); + } + + private Result() { + } + + public T get() { + return t; + } + + public void set(T t) { + this.t = t; + } + + public void err(String msg) { + this.err = Err.of(msg); + } +} diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/TkEntry.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/TkEntry.java new file mode 100644 index 000000000..19caf37ad --- /dev/null +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/TkEntry.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.util; + +import java.util.Arrays; +import java.util.Objects; + +/** + * Table Key pair. + */ +public class TkEntry { + private final String table; + private final byte[] key; + + public TkEntry(String table, byte[] key) { + this.table = table; + this.key = key; + } + + public String getTable() { + return table; + } + + public byte[] getKey() { + return key; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TkEntry)) return false; + TkEntry tk = (TkEntry) o; + return Objects.equals(table, tk.table) && Arrays.equals(key, tk.key); + } + + @Override + public int hashCode() { + int result = Objects.hash(table); + result = 31 * result + Arrays.hashCode(key); + return result; + } + + @Override + public String toString() { + return "Tk{" + + "table='" + table + '\'' + + ", key=" + Arrays.toString(key) + + '}'; + } +} diff --git a/hg-store-node/src/main/resources/log4j2-dev.xml b/hg-store-node/src/main/resources/log4j2-dev.xml index 46a9df4dc..3e5e01743 100644 --- a/hg-store-node/src/main/resources/log4j2-dev.xml +++ b/hg-store-node/src/main/resources/log4j2-dev.xml @@ -1,5 +1,22 @@ + + @@ -9,7 +26,7 @@ - + @@ -25,7 +42,7 @@ - + @@ -33,8 +50,8 @@ - - + + @@ -52,7 +69,7 @@ - + @@ -60,8 +77,8 @@ - - + + @@ -80,7 +97,7 @@ - + @@ -88,8 +105,8 @@ - - + + diff --git a/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer00.java b/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer00.java deleted file mode 100644 index 92af87666..000000000 --- a/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer00.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.baidu.hugegraph.store.boot; - -import com.alipay.remoting.util.StringUtils; -import com.baidu.hugegraph.store.node.StoreNodeApplication; -import org.springframework.boot.SpringApplication; -import java.io.File; - -public class StoreNodeServer00 { - - public static void main(String[] args) { - // deleteDir(new File("tmp/8500")); - String logPath = System.getProperty("logging.path"); - if (StringUtils.isBlank(logPath)) { - System.setProperty("logging.path", "logs/8500"); - System.setProperty("com.alipay.remoting.client.log.level", "WARN"); - } - if (System.getProperty("bolt.channel_write_buf_low_water_mark") == null) - System.setProperty("bolt.channel_write_buf_low_water_mark", Integer.toString(4 * 1024 * 1024)); - if (System.getProperty("bolt.channel_write_buf_high_water_mark") == null) - System.setProperty("bolt.channel_write_buf_high_water_mark", Integer.toString(8 * 1024 * 1024)); - SpringApplication.run(StoreNodeApplication.class,"--spring.profiles.active=server00"); - System.out.println("StoreNodeServer00 started."); - } - private static boolean deleteDir(File dir) { - if (dir.isDirectory()) { - for(File file : dir.listFiles()){ - deleteDir(file); - } - } - return dir.delete(); - } -} \ No newline at end of file diff --git a/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer01.java b/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer01.java deleted file mode 100644 index 48794a390..000000000 --- a/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer01.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.baidu.hugegraph.store.boot; - -import com.alipay.remoting.util.StringUtils; -import com.baidu.hugegraph.store.node.StoreNodeApplication; -import org.springframework.boot.SpringApplication; -import java.io.File; - -public class StoreNodeServer01 { - - public static void main(String[] args) { - // deleteDir(new File("tmp/8501")); - String logPath = System.getProperty("logging.path"); - if (StringUtils.isBlank(logPath)) { - System.setProperty("logging.path", "logs/8501"); - System.setProperty("com.alipay.remoting.client.log.level", "WARN"); - } - if (System.getProperty("bolt.channel_write_buf_low_water_mark") == null) - System.setProperty("bolt.channel_write_buf_low_water_mark", Integer.toString(4 * 1024 * 1024)); - if (System.getProperty("bolt.channel_write_buf_high_water_mark") == null) - System.setProperty("bolt.channel_write_buf_high_water_mark", Integer.toString(8 * 1024 * 1024)); - - - SpringApplication.run(StoreNodeApplication.class, "--spring.profiles.active=server01"); - System.out.println("StoreNodeServer01 started."); - } - - private static boolean deleteDir(File dir) { - if (dir.isDirectory()) { - for (File file : dir.listFiles()) { - deleteDir(file); - } - } - return dir.delete(); - } -} \ No newline at end of file diff --git a/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer03.java b/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer03.java deleted file mode 100644 index 3a12fd93e..000000000 --- a/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer03.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.baidu.hugegraph.store.boot; - -import com.alipay.remoting.util.StringUtils; -import com.baidu.hugegraph.store.node.StoreNodeApplication; -import org.springframework.boot.SpringApplication; - -import java.io.File; - -public class StoreNodeServer03 { - - public static void main(String[] args) { - // deleteDir(new File("tmp/8503")); - String logPath = System.getProperty("logging.path"); - if (StringUtils.isBlank(logPath)) { - System.setProperty("logging.path", "logs/8503"); - System.setProperty("com.alipay.remoting.client.log.level", "WARN"); - } - if (System.getProperty("bolt.channel_write_buf_low_water_mark") == null) - System.setProperty("bolt.channel_write_buf_low_water_mark", Integer.toString(4 * 1024 * 1024)); - if (System.getProperty("bolt.channel_write_buf_high_water_mark") == null) - System.setProperty("bolt.channel_write_buf_high_water_mark", Integer.toString(8 * 1024 * 1024)); - - SpringApplication.run(StoreNodeApplication.class, "--spring.profiles.active=server03"); - System.out.println("StoreNodeServer03 started."); - } - - private static boolean deleteDir(File dir) { - if (dir.isDirectory()) { - for (File file : dir.listFiles()) { - deleteDir(file); - } - } - return dir.delete(); - } -} \ No newline at end of file diff --git a/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer04.java b/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer04.java deleted file mode 100644 index a38245f01..000000000 --- a/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer04.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.baidu.hugegraph.store.boot; - -import com.baidu.hugegraph.store.node.StoreNodeApplication; -import org.springframework.boot.SpringApplication; - -import java.io.File; - -public class StoreNodeServer04 { - - public static void main(String[] args) { - // deleteDir(new File("tmp/8504")); - SpringApplication.run(StoreNodeApplication.class,"--spring.profiles.active=server04"); - System.out.println("StoreNodeServer04 started."); - } - private static boolean deleteDir(File dir) { - if (dir.isDirectory()) { - for(File file : dir.listFiles()){ - deleteDir(file); - } - } - return dir.delete(); - } -} \ No newline at end of file diff --git a/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer05.java b/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer05.java deleted file mode 100644 index 7994de4ca..000000000 --- a/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer05.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.baidu.hugegraph.store.boot; - -import com.baidu.hugegraph.store.node.StoreNodeApplication; -import org.springframework.boot.SpringApplication; - -import java.io.File; - -public class StoreNodeServer05 { - - public static void main(String[] args) { - // deleteDir(new File("tmp/8503")); - SpringApplication.run(StoreNodeApplication.class,"--spring.profiles.active=server05"); - System.out.println("StoreNodeServer03 started."); - } - private static boolean deleteDir(File dir) { - if (dir.isDirectory()) { - for(File file : dir.listFiles()){ - deleteDir(file); - } - } - return dir.delete(); - } -} \ No newline at end of file diff --git a/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer06.java b/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer06.java deleted file mode 100644 index 6e2e02f71..000000000 --- a/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer06.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.baidu.hugegraph.store.boot; - -import com.baidu.hugegraph.store.node.StoreNodeApplication; -import org.springframework.boot.SpringApplication; - -import java.io.File; - -public class StoreNodeServer06 { - - public static void main(String[] args) { - // deleteDir(new File("tmp/8503")); - SpringApplication.run(StoreNodeApplication.class,"--spring.profiles.active=server06"); - System.out.println("StoreNodeServer03 started."); - } - private static boolean deleteDir(File dir) { - if (dir.isDirectory()) { - for(File file : dir.listFiles()){ - deleteDir(file); - } - } - return dir.delete(); - } -} \ No newline at end of file diff --git a/hg-store-node/src/test/java/com/baidu/hugegraph/store/node/HgStoreNodeServiceTest.java b/hg-store-node/src/test/java/com/baidu/hugegraph/store/node/HgStoreNodeServiceTest.java deleted file mode 100644 index 38c2a2aac..000000000 --- a/hg-store-node/src/test/java/com/baidu/hugegraph/store/node/HgStoreNodeServiceTest.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.baidu.hugegraph.store.node; - -// import org.junit.Test; -import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; -import org.yaml.snakeyaml.Yaml; - -import java.io.IOException; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -/** - * HgStore单元测试 - * 1、测试raft多副本入库 - * 2、测试快照同步 - * 3、测试副本增减 - * 4、测试单幅本关闭日志入库 - */ -public class HgStoreNodeServiceTest { - - // @Test - public void testRaft() { - - } - - String yml = "rocksdb:\n" + " # rocksdb 使用的总内存大小\n" + " total_memory_size: 32000000000\n" + " max_background_jobs: 8\n" + " max_subcompactions: 4\n" + " target_file_size_multiplier: 4\n" + " min_write_buffer_number_to_merge: 8\n" + " target_file_size_base: 512000000"; - - // @Test - public void testYaml() throws InterruptedException, IOException { - - ExecutorService executor = new ThreadPoolExecutor(1000, 1000, - 10L, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(10000)); - CountDownLatch latch = new CountDownLatch(100); - for(int i =0 ;i < 100; i++){ - - executor.execute(()->{ - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - System.out.println(executor); - System.out.println(Thread.activeCount()); - latch.countDown(); - }); - - } - latch.await(); - System.in.read(); - } -} diff --git a/hg-store-node/src/test/java/com/baidu/hugegraph/store/node/metrics/JraftMetricsTest.java b/hg-store-node/src/test/java/com/baidu/hugegraph/store/node/metrics/JraftMetricsTest.java deleted file mode 100644 index ee9242f66..000000000 --- a/hg-store-node/src/test/java/com/baidu/hugegraph/store/node/metrics/JraftMetricsTest.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.baidu.hugegraph.store.node.metrics; - -import com.baidu.hugegraph.store.node.util.HgRegexUtil; -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - - -/** - * @author lynn.bond@hotmail.com on 2022/3/7 - */ -public class JraftMetricsTest { - - @Test - public void testRefineLabel() { - String regex = "(replicator)(.+?:\\d+)(.*)"; - - String[] sources = { - "replicator_10.14.139.10:8081_append_entries_times", - "replicator10.14.139.10:8081appendentriestimes", - "replicator.10.14.139.10:8081.append.entries.times", - "replicator-10-14-139-10:8081-append-entries-times", - "replicator_hg_0_10_14_139_10:8081::100_replicate_inflights_count_min", - "replicasdf14-13dasfies-times", - }; - - - Arrays.stream(sources).forEach(e -> { - System.out.println("--- "+e+" ---"); - List list = HgRegexUtil.toGroupValues(regex, e); - if (list != null) { - list.forEach(System.out::println); - }else{ - System.out.println("NONE"); - } - System.out.println(""); - }); - - } - -} diff --git a/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer00.java b/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer00.java new file mode 100644 index 000000000..ce2858b35 --- /dev/null +++ b/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer00.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.boot; + +import java.io.File; +import java.util.Objects; + +import org.apache.hugegraph.store.node.StoreNodeApplication; +import org.springframework.boot.SpringApplication; + +import com.alipay.remoting.util.StringUtils; + +public class StoreNodeServer00 { + + public static void main(String[] args) { + // deleteDir(new File("tmp/8500")); + String logPath = System.getProperty("logging.path"); + if (StringUtils.isBlank(logPath)) { + System.setProperty("logging.path", "logs/8500"); + System.setProperty("com.alipay.remoting.client.log.level", "WARN"); + } + if (System.getProperty("bolt.channel_write_buf_low_water_mark") == null) { + System.setProperty("bolt.channel_write_buf_low_water_mark", + Integer.toString(4 * 1024 * 1024)); + } + if (System.getProperty("bolt.channel_write_buf_high_water_mark") == null) { + System.setProperty("bolt.channel_write_buf_high_water_mark", + Integer.toString(8 * 1024 * 1024)); + } + SpringApplication.run(StoreNodeApplication.class, "--spring.profiles.active=server00"); + System.out.println("StoreNodeServer00 started."); + } + + private static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + for (File file : Objects.requireNonNull(dir.listFiles())) { + deleteDir(file); + } + } + return dir.delete(); + } +} \ No newline at end of file diff --git a/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer01.java b/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer01.java new file mode 100644 index 000000000..a8fbe96e5 --- /dev/null +++ b/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer01.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.boot; + +import java.io.File; +import java.util.Objects; + +import org.apache.hugegraph.store.node.StoreNodeApplication; +import org.springframework.boot.SpringApplication; + +import com.alipay.remoting.util.StringUtils; + +public class StoreNodeServer01 { + + public static void main(String[] args) { + // deleteDir(new File("tmp/8501")); + String logPath = System.getProperty("logging.path"); + if (StringUtils.isBlank(logPath)) { + System.setProperty("logging.path", "logs/8501"); + System.setProperty("com.alipay.remoting.client.log.level", "WARN"); + } + if (System.getProperty("bolt.channel_write_buf_low_water_mark") == null) { + System.setProperty("bolt.channel_write_buf_low_water_mark", + Integer.toString(4 * 1024 * 1024)); + } + if (System.getProperty("bolt.channel_write_buf_high_water_mark") == null) { + System.setProperty("bolt.channel_write_buf_high_water_mark", + Integer.toString(8 * 1024 * 1024)); + } + + + SpringApplication.run(StoreNodeApplication.class, "--spring.profiles.active=server01"); + System.out.println("StoreNodeServer01 started."); + } + + private static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + for (File file : Objects.requireNonNull(dir.listFiles())) { + deleteDir(file); + } + } + return dir.delete(); + } +} \ No newline at end of file diff --git a/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer02.java b/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer02.java similarity index 50% rename from hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer02.java rename to hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer02.java index 9da329537..b1c5e87c4 100644 --- a/hg-store-node/src/test/java/com/baidu/hugegraph/store/boot/StoreNodeServer02.java +++ b/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer02.java @@ -1,11 +1,29 @@ -package com.baidu.hugegraph.store.boot; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.alipay.remoting.util.StringUtils; -import com.baidu.hugegraph.store.node.StoreNodeApplication; -import org.springframework.boot.SpringApplication; +package org.apache.hugegraph.store.boot; import java.io.File; +import org.apache.hugegraph.store.node.StoreNodeApplication; +import org.springframework.boot.SpringApplication; + +import com.alipay.remoting.util.StringUtils; + public class StoreNodeServer02 { public static void main(String[] args) { @@ -15,10 +33,14 @@ public class StoreNodeServer02 { System.setProperty("logging.path", "logs/8502"); System.setProperty("com.alipay.remoting.client.log.level", "WARN"); } - if (System.getProperty("bolt.channel_write_buf_low_water_mark") == null) - System.setProperty("bolt.channel_write_buf_low_water_mark", Integer.toString(4 * 1024 * 1024)); - if (System.getProperty("bolt.channel_write_buf_high_water_mark") == null) - System.setProperty("bolt.channel_write_buf_high_water_mark", Integer.toString(8 * 1024 * 1024)); + if (System.getProperty("bolt.channel_write_buf_low_water_mark") == null) { + System.setProperty("bolt.channel_write_buf_low_water_mark", + Integer.toString(4 * 1024 * 1024)); + } + if (System.getProperty("bolt.channel_write_buf_high_water_mark") == null) { + System.setProperty("bolt.channel_write_buf_high_water_mark", + Integer.toString(8 * 1024 * 1024)); + } SpringApplication.run(StoreNodeApplication.class, "--spring.profiles.active=server02"); System.out.println("StoreNodeServer02 started."); diff --git a/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer03.java b/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer03.java new file mode 100644 index 000000000..1abac6618 --- /dev/null +++ b/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer03.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.boot; + +import java.io.File; + +import org.apache.hugegraph.store.node.StoreNodeApplication; +import org.springframework.boot.SpringApplication; + +import com.alipay.remoting.util.StringUtils; + +public class StoreNodeServer03 { + + public static void main(String[] args) { + // deleteDir(new File("tmp/8503")); + String logPath = System.getProperty("logging.path"); + if (StringUtils.isBlank(logPath)) { + System.setProperty("logging.path", "logs/8503"); + System.setProperty("com.alipay.remoting.client.log.level", "WARN"); + } + if (System.getProperty("bolt.channel_write_buf_low_water_mark") == null) { + System.setProperty("bolt.channel_write_buf_low_water_mark", + Integer.toString(4 * 1024 * 1024)); + } + if (System.getProperty("bolt.channel_write_buf_high_water_mark") == null) { + System.setProperty("bolt.channel_write_buf_high_water_mark", + Integer.toString(8 * 1024 * 1024)); + } + + SpringApplication.run(StoreNodeApplication.class, "--spring.profiles.active=server03"); + System.out.println("StoreNodeServer03 started."); + } + + private static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + for (File file : dir.listFiles()) { + deleteDir(file); + } + } + return dir.delete(); + } +} \ No newline at end of file diff --git a/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer04.java b/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer04.java new file mode 100644 index 000000000..1730c9505 --- /dev/null +++ b/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer04.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.boot; + +import java.io.File; + +import org.apache.hugegraph.store.node.StoreNodeApplication; +import org.springframework.boot.SpringApplication; + +public class StoreNodeServer04 { + + public static void main(String[] args) { + // deleteDir(new File("tmp/8504")); + SpringApplication.run(StoreNodeApplication.class, "--spring.profiles.active=server04"); + System.out.println("StoreNodeServer04 started."); + } + + private static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + for (File file : dir.listFiles()) { + deleteDir(file); + } + } + return dir.delete(); + } +} \ No newline at end of file diff --git a/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer05.java b/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer05.java new file mode 100644 index 000000000..61e6e86a1 --- /dev/null +++ b/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer05.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.boot; + +import java.io.File; + +import org.apache.hugegraph.store.node.StoreNodeApplication; +import org.springframework.boot.SpringApplication; + +public class StoreNodeServer05 { + + public static void main(String[] args) { + // deleteDir(new File("tmp/8503")); + SpringApplication.run(StoreNodeApplication.class, "--spring.profiles.active=server05"); + System.out.println("StoreNodeServer03 started."); + } + + private static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + for (File file : dir.listFiles()) { + deleteDir(file); + } + } + return dir.delete(); + } +} \ No newline at end of file diff --git a/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer06.java b/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer06.java new file mode 100644 index 000000000..3f3b5acc4 --- /dev/null +++ b/hg-store-node/src/test/java/org/apache/hugegraph/store/boot/StoreNodeServer06.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.boot; + +import java.io.File; + +import org.apache.hugegraph.store.node.StoreNodeApplication; +import org.springframework.boot.SpringApplication; + +public class StoreNodeServer06 { + + public static void main(String[] args) { + // deleteDir(new File("tmp/8503")); + SpringApplication.run(StoreNodeApplication.class, "--spring.profiles.active=server06"); + System.out.println("StoreNodeServer03 started."); + } + + private static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + for (File file : dir.listFiles()) { + deleteDir(file); + } + } + return dir.delete(); + } +} \ No newline at end of file diff --git a/hg-store-node/src/test/java/org/apache/hugegraph/store/node/HgStoreNodeServiceTest.java b/hg-store-node/src/test/java/org/apache/hugegraph/store/node/HgStoreNodeServiceTest.java new file mode 100644 index 000000000..a35efb331 --- /dev/null +++ b/hg-store-node/src/test/java/org/apache/hugegraph/store/node/HgStoreNodeServiceTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node; + +// import org.junit.Test; + +import java.io.IOException; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * HgStore单元测试 + * 1、测试raft多副本入库 + * 2、测试快照同步 + * 3、测试副本增减 + * 4、测试单幅本关闭日志入库 + */ +public class HgStoreNodeServiceTest { + + // @Test + public void testRaft() { + + } + + String yml = + "rocksdb:\n" + " # rocksdb 使用的总内存大小\n" + " total_memory_size: 32000000000\n" + + " max_background_jobs: 8\n" + " max_subcompactions: 4\n" + + " target_file_size_multiplier: 4\n" + " min_write_buffer_number_to_merge: 8\n" + + " target_file_size_base: 512000000"; + + // @Test + public void testYaml() throws InterruptedException, IOException { + + ExecutorService executor = new ThreadPoolExecutor(1000, 1000, + 10L, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(10000)); + CountDownLatch latch = new CountDownLatch(100); + for (int i = 0; i < 100; i++) { + + executor.execute(() -> { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + System.out.println(executor); + System.out.println(Thread.activeCount()); + latch.countDown(); + }); + + } + latch.await(); + System.in.read(); + } +} diff --git a/hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/JraftMetricsTest.java b/hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/JraftMetricsTest.java new file mode 100644 index 000000000..bd7a8b1a0 --- /dev/null +++ b/hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/JraftMetricsTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.metrics; + +import java.util.Arrays; +import java.util.List; + +import org.apache.hugegraph.store.node.util.HgRegexUtil; +import org.junit.Test; + + +/** + * @author lynn.bond@hotmail.com on 2022/3/7 + */ +public class JraftMetricsTest { + + @Test + public void testRefineLabel() { + String regex = "(replicator)(.+?:\\d+)(.*)"; + + String[] sources = { + "replicator_10.14.139.10:8081_append_entries_times", + "replicator10.14.139.10:8081appendentriestimes", + "replicator.10.14.139.10:8081.append.entries.times", + "replicator-10-14-139-10:8081-append-entries-times", + "replicator_hg_0_10_14_139_10:8081::100_replicate_inflights_count_min", + "replicasdf14-13dasfies-times", + }; + + + Arrays.stream(sources).forEach(e -> { + System.out.println("--- " + e + " ---"); + List list = HgRegexUtil.toGroupValues(regex, e); + if (list != null) { + list.forEach(System.out::println); + } else { + System.out.println("NONE"); + } + System.out.println(); + }); + + } + +} diff --git a/hg-store-node/src/test/resources/log4j2-dev.xml b/hg-store-node/src/test/resources/log4j2-dev.xml index dd8af69fe..59e8ff552 100644 --- a/hg-store-node/src/test/resources/log4j2-dev.xml +++ b/hg-store-node/src/test/resources/log4j2-dev.xml @@ -1,5 +1,22 @@ + + @@ -9,7 +26,7 @@ - + @@ -25,7 +42,7 @@ - + @@ -33,8 +50,8 @@ - - + + @@ -52,7 +69,7 @@ - + @@ -60,8 +77,8 @@ - - + + @@ -80,7 +97,7 @@ - + @@ -88,8 +105,8 @@ - - + + diff --git a/hg-store-rocksdb/pom.xml b/hg-store-rocksdb/pom.xml index d3c28e1a3..e281a1b0a 100644 --- a/hg-store-rocksdb/pom.xml +++ b/hg-store-rocksdb/pom.xml @@ -1,6 +1,24 @@ + + + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 diff --git a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/DBSnapshotMeta.java b/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/DBSnapshotMeta.java deleted file mode 100644 index 42c65e9c5..000000000 --- a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/DBSnapshotMeta.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.baidu.hugegraph.rocksdb.access; - -import java.util.Date; -import java.util.HashMap; - -import lombok.Data; -import lombok.NonNull; - -/** - * metadata of a Rocksdb snapshot - */ -@Data -public class DBSnapshotMeta { - /** - * graph name - */ - private String graphName; - /** - * partition id - */ - private int partitionId; - /** - * star key with base64 encoding - */ - private byte[] startKey; - /** - * end key with base64 encoding - */ - private byte[] endKey; - /** - * created time of the snapshot - */ - private Date createdDate; - /** - * sst files, key is column family name value is the sst file name of the column - * family - */ - private HashMap sstFiles; -} diff --git a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/ScanIterator.java b/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/ScanIterator.java deleted file mode 100644 index 37abbd0db..000000000 --- a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/ScanIterator.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.baidu.hugegraph.rocksdb.access; - -import java.io.Closeable; - -public interface ScanIterator extends Closeable { - - public abstract class Trait { - public static final int SCAN_ANY = 0x80; - public static final int SCAN_PREFIX_BEGIN = 0x01; - public static final int SCAN_PREFIX_END = 0x02; - public static final int SCAN_GT_BEGIN = 0x04; - public static final int SCAN_GTE_BEGIN = 0x0c; - public static final int SCAN_LT_END = 0x10; - public static final int SCAN_LTE_END = 0x30; - public static final int SCAN_KEYONLY = 0x40; - public static final int SCAN_HASHCODE = 0x100; - } - - boolean hasNext(); - - boolean isValid(); - - T next(); - - default long count(){return 0;}; - - default byte[] position(){return new byte[0];} - - default void seek(byte[] position){}; - - void close(); -} diff --git a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/util/Asserts.java b/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/util/Asserts.java deleted file mode 100644 index 1687bb506..000000000 --- a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/util/Asserts.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.baidu.hugegraph.rocksdb.access.util; - -import java.util.Collection; -import java.util.Map; -import java.util.function.Supplier; - -/** - * @author lynn.bond@hotmail.com - */ -public final class Asserts { - - public static void isTrue(boolean expression, String message) { - if (message== null) throw new IllegalArgumentException("message is null"); - if (!expression) { - throw new IllegalArgumentException(message); - } - } - - public static void isTrue(boolean expression, Supplier s) { - if (s == null) throw new IllegalArgumentException("Supplier is null"); - if (!expression) { - throw s.get(); - } - } - -} \ No newline at end of file diff --git a/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/DBSnapshotMeta.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/DBSnapshotMeta.java new file mode 100644 index 000000000..1e9ec8500 --- /dev/null +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/DBSnapshotMeta.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access; + +import java.util.Date; +import java.util.HashMap; + +import lombok.Data; +import lombok.NonNull; + +/** + * metadata of a Rocksdb snapshot + */ +@Data +public class DBSnapshotMeta { + /** + * graph name + */ + private String graphName; + /** + * partition id + */ + private int partitionId; + /** + * star key with base64 encoding + */ + private byte[] startKey; + /** + * end key with base64 encoding + */ + private byte[] endKey; + /** + * created time of the snapshot + */ + private Date createdDate; + /** + * sst files, key is column family name value is the sst file name of the column + * family + */ + private HashMap sstFiles; +} diff --git a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/DBStoreException.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/DBStoreException.java similarity index 59% rename from hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/DBStoreException.java rename to hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/DBStoreException.java index 4dfbe48e7..5d51d95f6 100644 --- a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/DBStoreException.java +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/DBStoreException.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.rocksdb.access; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access; import java.io.InterruptedIOException; diff --git a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/RocksDBFactory.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBFactory.java similarity index 92% rename from hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/RocksDBFactory.java rename to hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBFactory.java index a59941a72..380d9302f 100644 --- a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/RocksDBFactory.java +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBFactory.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.rocksdb.access; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access; import java.io.File; import java.util.ArrayList; diff --git a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/RocksDBOptions.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java similarity index 95% rename from hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/RocksDBOptions.java rename to hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java index 4ab1bcf15..d3e988f3e 100644 --- a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/RocksDBOptions.java +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.rocksdb.access; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access; import static com.baidu.hugegraph.config.OptionChecker.allowValues; import static com.baidu.hugegraph.config.OptionChecker.disallowEmpty; diff --git a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/RocksDBScanIterator.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBScanIterator.java similarity index 88% rename from hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/RocksDBScanIterator.java rename to hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBScanIterator.java index d26e597b5..2c1f24418 100644 --- a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/RocksDBScanIterator.java +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBScanIterator.java @@ -1,6 +1,23 @@ -package com.baidu.hugegraph.rocksdb.access; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; +package org.apache.hugegraph.rocksdb.access; + +import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; import com.baidu.hugegraph.util.Bytes; import com.baidu.hugegraph.util.E; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/RocksDBSession.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java similarity index 97% rename from hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/RocksDBSession.java rename to hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java index 74d7be4ad..5e2e11383 100644 --- a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/RocksDBSession.java +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.rocksdb.access; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access; import java.io.Closeable; import java.io.File; @@ -19,6 +36,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.rocksdb.access.util.Asserts; import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.Checkpoint; import org.rocksdb.ColumnFamilyDescriptor; @@ -45,7 +63,6 @@ import org.rocksdb.WriteBufferManager; import org.rocksdb.WriteOptions; import com.baidu.hugegraph.config.HugeConfig; -import com.baidu.hugegraph.rocksdb.access.util.Asserts; import com.baidu.hugegraph.store.term.HgPair; import com.baidu.hugegraph.util.Bytes; import com.baidu.hugegraph.util.E; diff --git a/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/ScanIterator.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/ScanIterator.java new file mode 100644 index 000000000..a4ee2488d --- /dev/null +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/ScanIterator.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access; + +import java.io.Closeable; + +public interface ScanIterator extends Closeable { + + public abstract class Trait { + public static final int SCAN_ANY = 0x80; + public static final int SCAN_PREFIX_BEGIN = 0x01; + public static final int SCAN_PREFIX_END = 0x02; + public static final int SCAN_GT_BEGIN = 0x04; + public static final int SCAN_GTE_BEGIN = 0x0c; + public static final int SCAN_LT_END = 0x10; + public static final int SCAN_LTE_END = 0x30; + public static final int SCAN_KEYONLY = 0x40; + public static final int SCAN_HASHCODE = 0x100; + } + + boolean hasNext(); + + boolean isValid(); + + T next(); + + default long count(){return 0;}; + + default byte[] position(){return new byte[0];} + + default void seek(byte[] position){}; + + void close(); +} diff --git a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/SessionOperator.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperator.java similarity index 66% rename from hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/SessionOperator.java rename to hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperator.java index 42ea13735..b17b1d471 100644 --- a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/SessionOperator.java +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperator.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.rocksdb.access; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access; import com.baidu.hugegraph.store.term.HgPair; diff --git a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/SessionOperatorImpl.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java similarity index 92% rename from hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/SessionOperatorImpl.java rename to hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java index 9d7a7e9ab..bc886681c 100644 --- a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/SessionOperatorImpl.java +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java @@ -1,9 +1,27 @@ -package com.baidu.hugegraph.rocksdb.access; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Iterator; +import org.apache.hugegraph.rocksdb.access.util.Asserts; import org.rocksdb.ReadOptions; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; @@ -12,8 +30,8 @@ import org.rocksdb.Slice; import org.rocksdb.Snapshot; import org.rocksdb.WriteBatch; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession.CFHandleLock; -import com.baidu.hugegraph.rocksdb.access.util.Asserts; +import org.apache.hugegraph.rocksdb.access.RocksDBSession.CFHandleLock; + import com.baidu.hugegraph.store.term.HgPair; import com.baidu.hugegraph.util.Bytes; diff --git a/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/util/Asserts.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/util/Asserts.java new file mode 100644 index 000000000..92480d96e --- /dev/null +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/util/Asserts.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access.util; + +import java.util.Collection; +import java.util.Map; +import java.util.function.Supplier; + +/** + * @author lynn.bond@hotmail.com + */ +public final class Asserts { + + public static void isTrue(boolean expression, String message) { + if (message== null) throw new IllegalArgumentException("message is null"); + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + public static void isTrue(boolean expression, Supplier s) { + if (s == null) throw new IllegalArgumentException("Supplier is null"); + if (!expression) { + throw s.get(); + } + } + +} \ No newline at end of file diff --git a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/util/CRC64.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/util/CRC64.java similarity index 89% rename from hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/util/CRC64.java rename to hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/util/CRC64.java index 4f5aeb345..97f4a98bc 100644 --- a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/util/CRC64.java +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/util/CRC64.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.rocksdb.access.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access.util; import java.util.zip.Checksum; diff --git a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/util/ZipUtils.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/util/ZipUtils.java similarity index 83% rename from hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/util/ZipUtils.java rename to hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/util/ZipUtils.java index acb82ba7d..822824739 100644 --- a/hg-store-rocksdb/src/main/java/com/baidu/hugegraph/rocksdb/access/util/ZipUtils.java +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/util/ZipUtils.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.rocksdb.access.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access.util; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; @@ -6,11 +23,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.Comparator; import java.util.zip.CheckedInputStream; import java.util.zip.CheckedOutputStream; import java.util.zip.Checksum; diff --git a/hg-store-rocksdb/src/test/java/com/baidu/hugegraph/rocksdb/access/RocksDBFactoryTest.java b/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/RocksDBFactoryTest.java similarity index 72% rename from hg-store-rocksdb/src/test/java/com/baidu/hugegraph/rocksdb/access/RocksDBFactoryTest.java rename to hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/RocksDBFactoryTest.java index 14896accc..ebbb6b3e9 100644 --- a/hg-store-rocksdb/src/test/java/com/baidu/hugegraph/rocksdb/access/RocksDBFactoryTest.java +++ b/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/RocksDBFactoryTest.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.rocksdb.access; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access; import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.config.OptionSpace; @@ -13,7 +30,7 @@ public class RocksDBFactoryTest { // @BeforeClass public static void init() { OptionSpace.register("rocksdb", - "com.baidu.hugegraph.rocksdb.access.RocksDBOptions"); + "org.apache.hugegraph.rocksdb.access.RocksDBOptions"); RocksDBOptions.instance(); Map configMap = new HashMap<>(); diff --git a/hg-store-rocksdb/src/test/java/com/baidu/hugegraph/rocksdb/access/RocksDBSessionTest.java b/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/RocksDBSessionTest.java similarity index 93% rename from hg-store-rocksdb/src/test/java/com/baidu/hugegraph/rocksdb/access/RocksDBSessionTest.java rename to hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/RocksDBSessionTest.java index 3ea3c2932..19d1fc49a 100644 --- a/hg-store-rocksdb/src/test/java/com/baidu/hugegraph/rocksdb/access/RocksDBSessionTest.java +++ b/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/RocksDBSessionTest.java @@ -1,8 +1,24 @@ -package com.baidu.hugegraph.rocksdb.access; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access; import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.config.OptionSpace; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; // import org.junit.BeforeClass; // import org.junit.Test; @@ -22,7 +38,7 @@ public class RocksDBSessionTest { // @BeforeClass public static void init() { OptionSpace.register("rocksdb", - "com.baidu.hugegraph.rocksdb.access.RocksDBOptions"); + "org.apache.hugegraph.rocksdb.access.RocksDBOptions"); RocksDBOptions.instance(); Map configMap = new HashMap<>(); configMap.put("rocksdb.write_buffer_size", "1048576"); @@ -192,7 +208,7 @@ public class RocksDBSessionTest { //scan table ScanIterator it = sessionOp.scan(tName); while (it.hasNext()) { - BackendColumn col = it.next(); + RocksDBSession.BackendColumn col = it.next(); System.out.println(new String(col.name) + " : " + new String(col.value)); } @@ -231,7 +247,7 @@ public class RocksDBSessionTest { //prefix scan ScanIterator it = sessionOp.scan(tName, "bo".getBytes()); while (it.hasNext()) { - BackendColumn col = it.next(); + RocksDBSession.BackendColumn col = it.next(); System.out.println(new String(col.name) + " : " + new String(col.value)); } @@ -266,7 +282,7 @@ public class RocksDBSessionTest { ScanIterator it = sessionOp.scan(tName, "box2".getBytes(), "box5".getBytes(), ScanIterator.Trait.SCAN_GTE_BEGIN | ScanIterator.Trait.SCAN_LTE_END); while (it.hasNext()) { - BackendColumn col = it.next(); + RocksDBSession.BackendColumn col = it.next(); System.out.println(new String(col.name) + " : " + new String(col.value)); } @@ -342,7 +358,7 @@ public class RocksDBSessionTest { ScanIterator it = sessionOp.scan(tName, "p".getBytes()); System.out.println("after put ------>"); while (it.hasNext()) { - BackendColumn col = it.next(); + RocksDBSession.BackendColumn col = it.next(); System.out.println(new String(col.name) + ":" + new String(col.value)); } @@ -398,7 +414,7 @@ public class RocksDBSessionTest { ScanIterator it = op.scan("f2"); while (it.hasNext()) { - BackendColumn col = it.next(); + RocksDBSession.BackendColumn col = it.next(); System.out.println(Arrays.toString(col.name) + "=>" + Arrays.toString(col.value)); } diff --git a/hg-store-rocksdb/src/test/java/com/baidu/hugegraph/rocksdb/access/RocksdbDump.java b/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/RocksdbDump.java similarity index 66% rename from hg-store-rocksdb/src/test/java/com/baidu/hugegraph/rocksdb/access/RocksdbDump.java rename to hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/RocksdbDump.java index b21b273be..4f9eeee82 100644 --- a/hg-store-rocksdb/src/test/java/com/baidu/hugegraph/rocksdb/access/RocksdbDump.java +++ b/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/RocksdbDump.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.rocksdb.access; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access; import com.baidu.hugegraph.util.Bytes; // import org.junit.Test; diff --git a/hg-store-rocksdb/src/test/java/com/baidu/hugegraph/rocksdb/access/SnapshotManagerTest.java b/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/SnapshotManagerTest.java similarity index 94% rename from hg-store-rocksdb/src/test/java/com/baidu/hugegraph/rocksdb/access/SnapshotManagerTest.java rename to hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/SnapshotManagerTest.java index de9fa60d9..d66a1158b 100644 --- a/hg-store-rocksdb/src/test/java/com/baidu/hugegraph/rocksdb/access/SnapshotManagerTest.java +++ b/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/SnapshotManagerTest.java @@ -1,24 +1,33 @@ -package com.baidu.hugegraph.rocksdb.access; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.rocksdb.access; import static org.junit.Assert.assertEquals; import java.io.File; -import java.io.IOException; -import java.net.URL; import java.nio.file.Paths; import java.util.*; -import java.util.zip.Checksum; -import com.alibaba.fastjson.JSON; import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.config.OptionSpace; -import com.baidu.hugegraph.rocksdb.access.util.CRC64; -import com.baidu.hugegraph.rocksdb.access.util.ZipUtils; import com.baidu.hugegraph.store.term.HgPair; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; -import org.junit.AfterClass; -import org.junit.Assert; // import org.junit.BeforeClass; // import org.junit.Test; @@ -155,7 +164,7 @@ public class SnapshotManagerTest { // public static void beforeClass() { // // Register config // OptionSpace.register("rocksdb", -// "com.baidu.hugegraph.rocksdb.access.RocksDBOptions"); +// "org.apache.hugegraph.rocksdb.access.RocksDBOptions"); // // Register backend // URL configPath = SnapshotManagerTest.class.getResource("/hugegraph.properties"); // URL configPath2 = SnapshotManagerTest.class.getResource("/hugegraph-2.properties"); diff --git a/hg-store-rocksdb/src/test/resources/hugegraph-2.properties b/hg-store-rocksdb/src/test/resources/hugegraph-2.properties index 03787f9aa..4b90d123e 100644 --- a/hg-store-rocksdb/src/test/resources/hugegraph-2.properties +++ b/hg-store-rocksdb/src/test/resources/hugegraph-2.properties @@ -1,3 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# # rocksdb backend config rocksdb.data_path=./tmp/rocksdb-2/data rocksdb.wal_path=./tmp/rocksdb-2/data diff --git a/hg-store-rocksdb/src/test/resources/hugegraph.properties b/hg-store-rocksdb/src/test/resources/hugegraph.properties index eba015fcb..b2d37ad4f 100644 --- a/hg-store-rocksdb/src/test/resources/hugegraph.properties +++ b/hg-store-rocksdb/src/test/resources/hugegraph.properties @@ -1,3 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# # rocksdb backend config rocksdb.data_path=./tmp/rocksdb/data rocksdb.wal_path=./tmp/rocksdb/data diff --git a/hg-store-rocksdb/src/test/resources/log4j2.xml b/hg-store-rocksdb/src/test/resources/log4j2.xml index 1bcbc671b..30f1ebd3d 100644 --- a/hg-store-rocksdb/src/test/resources/log4j2.xml +++ b/hg-store-rocksdb/src/test/resources/log4j2.xml @@ -1,13 +1,30 @@ - - - - - - - - - - - - + + + + + + + + + + + + + + diff --git a/hg-store-test/pom.xml b/hg-store-test/pom.xml index 84ecfceef..de96e461f 100644 --- a/hg-store-test/pom.xml +++ b/hg-store-test/pom.xml @@ -1,4 +1,21 @@ + + diff --git a/hg-store-test/src/main/java/client/BaseClientTest.java b/hg-store-test/src/main/java/client/BaseClientTest.java index 37e7be1ed..dd6285abf 100644 --- a/hg-store-test/src/main/java/client/BaseClientTest.java +++ b/hg-store-test/src/main/java/client/BaseClientTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package client; import com.baidu.hugegraph.pd.client.PDClient; diff --git a/hg-store-test/src/main/java/client/ChangeShardNumTest.java b/hg-store-test/src/main/java/client/ChangeShardNumTest.java index 288bb12ee..508bb4e01 100644 --- a/hg-store-test/src/main/java/client/ChangeShardNumTest.java +++ b/hg-store-test/src/main/java/client/ChangeShardNumTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package client; import com.baidu.hugegraph.pd.common.PDException; diff --git a/hg-store-test/src/main/java/client/ClientSuiteTest.java b/hg-store-test/src/main/java/client/ClientSuiteTest.java index 91042c852..5b5724fc1 100644 --- a/hg-store-test/src/main/java/client/ClientSuiteTest.java +++ b/hg-store-test/src/main/java/client/ClientSuiteTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package client; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-test/src/main/java/client/ClientTest.java b/hg-store-test/src/main/java/client/ClientTest.java index 8aec91f09..782322568 100644 --- a/hg-store-test/src/main/java/client/ClientTest.java +++ b/hg-store-test/src/main/java/client/ClientTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package client; import org.junit.Assert; diff --git a/hg-store-test/src/main/java/client/HgAssertTest.java b/hg-store-test/src/main/java/client/HgAssertTest.java index 5bb03cec5..86a2f4fa4 100644 --- a/hg-store-test/src/main/java/client/HgAssertTest.java +++ b/hg-store-test/src/main/java/client/HgAssertTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package client; import static org.junit.Assert.assertFalse; diff --git a/hg-store-test/src/main/java/client/HgKvStoreTest.java b/hg-store-test/src/main/java/client/HgKvStoreTest.java index 5123774c7..02f9bffce 100644 --- a/hg-store-test/src/main/java/client/HgKvStoreTest.java +++ b/hg-store-test/src/main/java/client/HgKvStoreTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package client; import com.baidu.hugegraph.store.HgKvEntry; diff --git a/hg-store-test/src/main/java/client/HgPairTest.java b/hg-store-test/src/main/java/client/HgPairTest.java index cef5906aa..540747945 100644 --- a/hg-store-test/src/main/java/client/HgPairTest.java +++ b/hg-store-test/src/main/java/client/HgPairTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package client; import static org.junit.Assert.assertFalse; diff --git a/hg-store-test/src/main/java/client/HgSessionManagerRaftPDTest.java b/hg-store-test/src/main/java/client/HgSessionManagerRaftPDTest.java index 9ccd02262..32f73f08b 100644 --- a/hg-store-test/src/main/java/client/HgSessionManagerRaftPDTest.java +++ b/hg-store-test/src/main/java/client/HgSessionManagerRaftPDTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package client; import com.baidu.hugegraph.pd.client.PDClient; diff --git a/hg-store-test/src/main/java/client/HgStoreClientTest.java b/hg-store-test/src/main/java/client/HgStoreClientTest.java index 8e32a8436..575533d04 100644 --- a/hg-store-test/src/main/java/client/HgStoreClientTest.java +++ b/hg-store-test/src/main/java/client/HgStoreClientTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package client; import com.baidu.hugegraph.pd.common.PDException; diff --git a/hg-store-test/src/main/java/client/HgStoreNodeStateTest.java b/hg-store-test/src/main/java/client/HgStoreNodeStateTest.java index c65154239..e68aeeaa9 100644 --- a/hg-store-test/src/main/java/client/HgStoreNodeStateTest.java +++ b/hg-store-test/src/main/java/client/HgStoreNodeStateTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package client; import com.baidu.hugegraph.store.client.HgStoreNodeManager; diff --git a/hg-store-test/src/main/java/client/graph/GraphStoreClientTest.java b/hg-store-test/src/main/java/client/graph/GraphStoreClientTest.java index d8ba9d4cd..97e3931f9 100644 --- a/hg-store-test/src/main/java/client/graph/GraphStoreClientTest.java +++ b/hg-store-test/src/main/java/client/graph/GraphStoreClientTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package client.graph; import java.util.ArrayList; diff --git a/hg-store-test/src/main/java/common/BaseCommonTest.java b/hg-store-test/src/main/java/common/BaseCommonTest.java index 2c678200e..2b48ace2a 100644 --- a/hg-store-test/src/main/java/common/BaseCommonTest.java +++ b/hg-store-test/src/main/java/common/BaseCommonTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package common; import org.junit.After; diff --git a/hg-store-test/src/main/java/common/ByteBufferAllocatorTest.java b/hg-store-test/src/main/java/common/ByteBufferAllocatorTest.java index e5c58cebf..8722d6b06 100644 --- a/hg-store-test/src/main/java/common/ByteBufferAllocatorTest.java +++ b/hg-store-test/src/main/java/common/ByteBufferAllocatorTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package common; import com.baidu.hugegraph.store.buffer.ByteBufferAllocator; diff --git a/hg-store-test/src/main/java/common/CommonSuiteTest.java b/hg-store-test/src/main/java/common/CommonSuiteTest.java index 8a7122493..99c3d1318 100644 --- a/hg-store-test/src/main/java/common/CommonSuiteTest.java +++ b/hg-store-test/src/main/java/common/CommonSuiteTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package common; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-test/src/main/java/common/KVByteBufferTest.java b/hg-store-test/src/main/java/common/KVByteBufferTest.java index 0a10ec6e5..621186e81 100644 --- a/hg-store-test/src/main/java/common/KVByteBufferTest.java +++ b/hg-store-test/src/main/java/common/KVByteBufferTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package common; import com.baidu.hugegraph.store.buffer.KVByteBuffer; diff --git a/hg-store-test/src/main/java/core/BaseCoreTest.java b/hg-store-test/src/main/java/core/BaseCoreTest.java index b4827c825..737bfba4d 100644 --- a/hg-store-test/src/main/java/core/BaseCoreTest.java +++ b/hg-store-test/src/main/java/core/BaseCoreTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core; import org.apache.commons.io.FileUtils; diff --git a/hg-store-test/src/main/java/core/CoreSuiteTest.java b/hg-store-test/src/main/java/core/CoreSuiteTest.java index 93522ed77..43ade054c 100644 --- a/hg-store-test/src/main/java/core/CoreSuiteTest.java +++ b/hg-store-test/src/main/java/core/CoreSuiteTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core; import core.raft.RaftOperationTest; diff --git a/hg-store-test/src/main/java/core/CoreTest.java b/hg-store-test/src/main/java/core/CoreTest.java index da936bc42..70e3011d7 100644 --- a/hg-store-test/src/main/java/core/CoreTest.java +++ b/hg-store-test/src/main/java/core/CoreTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core; import org.junit.Assert; diff --git a/hg-store-test/src/main/java/core/HgBusinessImplTest.java b/hg-store-test/src/main/java/core/HgBusinessImplTest.java index 04891e29c..3eac87018 100644 --- a/hg-store-test/src/main/java/core/HgBusinessImplTest.java +++ b/hg-store-test/src/main/java/core/HgBusinessImplTest.java @@ -1,12 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core; import lombok.extern.slf4j.Slf4j; import org.junit.Assert; import org.junit.Test; -import com.baidu.hugegraph.rocksdb.access.RocksDBFactory; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.store.business.BusinessHandler; import com.baidu.hugegraph.util.Bytes; diff --git a/hg-store-test/src/main/java/core/HgCmdClientTest.java b/hg-store-test/src/main/java/core/HgCmdClientTest.java index 08d669beb..199b93e94 100644 --- a/hg-store-test/src/main/java/core/HgCmdClientTest.java +++ b/hg-store-test/src/main/java/core/HgCmdClientTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core; import com.alipay.sofa.jraft.JRaftUtils; diff --git a/hg-store-test/src/main/java/core/MetricServiceTest.java b/hg-store-test/src/main/java/core/MetricServiceTest.java index 4673a3394..f1d9c8835 100644 --- a/hg-store-test/src/main/java/core/MetricServiceTest.java +++ b/hg-store-test/src/main/java/core/MetricServiceTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core; import java.util.Map; diff --git a/hg-store-test/src/main/java/core/StoreEngineTestBase.java b/hg-store-test/src/main/java/core/StoreEngineTestBase.java index 89169d782..ff71cb10a 100644 --- a/hg-store-test/src/main/java/core/StoreEngineTestBase.java +++ b/hg-store-test/src/main/java/core/StoreEngineTestBase.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core; import com.alipay.sofa.jraft.util.StorageOptionsFactory; diff --git a/hg-store-test/src/main/java/core/raft/HgStoreStateMachineTest.java b/hg-store-test/src/main/java/core/raft/HgStoreStateMachineTest.java index 0ebaab5d3..2f67f1097 100644 --- a/hg-store-test/src/main/java/core/raft/HgStoreStateMachineTest.java +++ b/hg-store-test/src/main/java/core/raft/HgStoreStateMachineTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.raft; import com.alipay.sofa.jraft.Closure; diff --git a/hg-store-test/src/main/java/core/raft/MockStateMachine.java b/hg-store-test/src/main/java/core/raft/MockStateMachine.java index 94e4f3f36..804794f5d 100644 --- a/hg-store-test/src/main/java/core/raft/MockStateMachine.java +++ b/hg-store-test/src/main/java/core/raft/MockStateMachine.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.raft; import com.alipay.sofa.jraft.Iterator; diff --git a/hg-store-test/src/main/java/core/raft/RaftOperationTest.java b/hg-store-test/src/main/java/core/raft/RaftOperationTest.java index c4cae898a..81cea63b3 100644 --- a/hg-store-test/src/main/java/core/raft/RaftOperationTest.java +++ b/hg-store-test/src/main/java/core/raft/RaftOperationTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.raft; import com.baidu.hugegraph.pd.grpc.Metapb; diff --git a/hg-store-test/src/main/java/core/raft/RaftUtilsTest.java b/hg-store-test/src/main/java/core/raft/RaftUtilsTest.java index 5b84a4b8c..28005f0df 100644 --- a/hg-store-test/src/main/java/core/raft/RaftUtilsTest.java +++ b/hg-store-test/src/main/java/core/raft/RaftUtilsTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.raft; import com.alipay.sofa.jraft.Node; diff --git a/hg-store-test/src/main/java/core/snapshot/HgSnapshotHandlerTest.java b/hg-store-test/src/main/java/core/snapshot/HgSnapshotHandlerTest.java index 3132f41e5..c0fbf7c38 100644 --- a/hg-store-test/src/main/java/core/snapshot/HgSnapshotHandlerTest.java +++ b/hg-store-test/src/main/java/core/snapshot/HgSnapshotHandlerTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.snapshot; import com.alipay.sofa.jraft.entity.RaftOutter; diff --git a/hg-store-test/src/main/java/core/store/HgStoreEngineTest.java b/hg-store-test/src/main/java/core/store/HgStoreEngineTest.java index ec4e13181..fb7f5efb0 100644 --- a/hg-store-test/src/main/java/core/store/HgStoreEngineTest.java +++ b/hg-store-test/src/main/java/core/store/HgStoreEngineTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.store; import com.baidu.hugegraph.pd.grpc.Metapb; diff --git a/hg-store-test/src/main/java/core/store/PartitionEngineTest.java b/hg-store-test/src/main/java/core/store/PartitionEngineTest.java index b05dc40f3..01b0daa63 100644 --- a/hg-store-test/src/main/java/core/store/PartitionEngineTest.java +++ b/hg-store-test/src/main/java/core/store/PartitionEngineTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.store; import com.baidu.hugegraph.store.PartitionEngine; diff --git a/hg-store-test/src/main/java/core/store/PartitionInstructionProcessorTest.java b/hg-store-test/src/main/java/core/store/PartitionInstructionProcessorTest.java index 903b61f73..36772f98a 100644 --- a/hg-store-test/src/main/java/core/store/PartitionInstructionProcessorTest.java +++ b/hg-store-test/src/main/java/core/store/PartitionInstructionProcessorTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.store; import com.baidu.hugegraph.pd.grpc.Metapb; diff --git a/hg-store-test/src/main/java/core/store/meta/MetadataKeyHelperTest.java b/hg-store-test/src/main/java/core/store/meta/MetadataKeyHelperTest.java index fb69487ca..6c8aa843a 100644 --- a/hg-store-test/src/main/java/core/store/meta/MetadataKeyHelperTest.java +++ b/hg-store-test/src/main/java/core/store/meta/MetadataKeyHelperTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.store.meta; import com.baidu.hugegraph.store.meta.MetadataKeyHelper; diff --git a/hg-store-test/src/main/java/core/store/meta/PartitionManagerTest.java b/hg-store-test/src/main/java/core/store/meta/PartitionManagerTest.java index d34ef7360..45756bbbb 100644 --- a/hg-store-test/src/main/java/core/store/meta/PartitionManagerTest.java +++ b/hg-store-test/src/main/java/core/store/meta/PartitionManagerTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.store.meta; import com.baidu.hugegraph.pd.common.PDException; diff --git a/hg-store-test/src/main/java/core/store/meta/TaskManagerTest.java b/hg-store-test/src/main/java/core/store/meta/TaskManagerTest.java index b0e66d3a6..3e2b4d19a 100644 --- a/hg-store-test/src/main/java/core/store/meta/TaskManagerTest.java +++ b/hg-store-test/src/main/java/core/store/meta/TaskManagerTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.store.meta; import com.baidu.hugegraph.pd.grpc.MetaTask; diff --git a/hg-store-test/src/main/java/core/store/meta/asynctask/CleanTaskTest.java b/hg-store-test/src/main/java/core/store/meta/asynctask/CleanTaskTest.java index f40f0b00a..169e061bc 100644 --- a/hg-store-test/src/main/java/core/store/meta/asynctask/CleanTaskTest.java +++ b/hg-store-test/src/main/java/core/store/meta/asynctask/CleanTaskTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.store.meta.asynctask; import com.baidu.hugegraph.store.meta.asynctask.AbstractAsyncTask; diff --git a/hg-store-test/src/main/java/core/store/util/MiscUtilClassTest.java b/hg-store-test/src/main/java/core/store/util/MiscUtilClassTest.java index 114b66ac2..74d5ecb15 100644 --- a/hg-store-test/src/main/java/core/store/util/MiscUtilClassTest.java +++ b/hg-store-test/src/main/java/core/store/util/MiscUtilClassTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.store.util; import com.alipay.sofa.jraft.Status; diff --git a/hg-store-test/src/main/java/core/store/util/PartitionMetaStoreWrapperTest.java b/hg-store-test/src/main/java/core/store/util/PartitionMetaStoreWrapperTest.java index 62de7712f..20401d4a9 100644 --- a/hg-store-test/src/main/java/core/store/util/PartitionMetaStoreWrapperTest.java +++ b/hg-store-test/src/main/java/core/store/util/PartitionMetaStoreWrapperTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.store.util; import com.baidu.hugegraph.pd.grpc.Metapb; diff --git a/hg-store-test/src/main/java/core/store/util/ZipUtilsTest.java b/hg-store-test/src/main/java/core/store/util/ZipUtilsTest.java index ade26088d..867189508 100644 --- a/hg-store-test/src/main/java/core/store/util/ZipUtilsTest.java +++ b/hg-store-test/src/main/java/core/store/util/ZipUtilsTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package core.store.util; import com.baidu.hugegraph.store.util.ZipUtils; diff --git a/hg-store-test/src/main/java/raftcore/BytesCarrierTest.java b/hg-store-test/src/main/java/raftcore/BytesCarrierTest.java index 46f49501a..a68fe6ea1 100644 --- a/hg-store-test/src/main/java/raftcore/BytesCarrierTest.java +++ b/hg-store-test/src/main/java/raftcore/BytesCarrierTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package raftcore; import com.google.protobuf.BytesCarrier; diff --git a/hg-store-test/src/main/java/raftcore/RaftSuiteTest.java b/hg-store-test/src/main/java/raftcore/RaftSuiteTest.java index 4668a79a8..86c6de3bc 100644 --- a/hg-store-test/src/main/java/raftcore/RaftSuiteTest.java +++ b/hg-store-test/src/main/java/raftcore/RaftSuiteTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package raftcore; import org.junit.runner.RunWith; diff --git a/hg-store-test/src/main/java/raftcore/ZeroByteStringHelperTest.java b/hg-store-test/src/main/java/raftcore/ZeroByteStringHelperTest.java index c851c9ee4..f47cfa6c2 100644 --- a/hg-store-test/src/main/java/raftcore/ZeroByteStringHelperTest.java +++ b/hg-store-test/src/main/java/raftcore/ZeroByteStringHelperTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package raftcore; import com.google.protobuf.ZeroByteStringHelper; diff --git a/hg-store-test/src/main/java/rocksdb/BaseRocksDbTest.java b/hg-store-test/src/main/java/rocksdb/BaseRocksDbTest.java index d33fa9170..481bf862e 100644 --- a/hg-store-test/src/main/java/rocksdb/BaseRocksDbTest.java +++ b/hg-store-test/src/main/java/rocksdb/BaseRocksDbTest.java @@ -1,9 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package rocksdb; import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.config.OptionSpace; -import com.baidu.hugegraph.rocksdb.access.RocksDBFactory; -import com.baidu.hugegraph.rocksdb.access.RocksDBOptions; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; import org.junit.After; import org.junit.BeforeClass; @@ -14,7 +31,7 @@ public class BaseRocksDbTest { @BeforeClass public static void init() { OptionSpace.register("rocksdb", - "com.baidu.hugegraph.rocksdb.access.RocksDBOptions"); + "org.apache.hugegraph.rocksdb.access.RocksDBOptions"); RocksDBOptions.instance(); Map configMap = new HashMap<>(); diff --git a/hg-store-test/src/main/java/rocksdb/RocksDBFactoryTest.java b/hg-store-test/src/main/java/rocksdb/RocksDBFactoryTest.java index 91c199dce..aff26836b 100644 --- a/hg-store-test/src/main/java/rocksdb/RocksDBFactoryTest.java +++ b/hg-store-test/src/main/java/rocksdb/RocksDBFactoryTest.java @@ -1,8 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package rocksdb; -import com.baidu.hugegraph.rocksdb.access.RocksDBFactory; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.rocksdb.access.SessionOperator; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.SessionOperator; import org.junit.Test; public class RocksDBFactoryTest extends BaseRocksDbTest{ diff --git a/hg-store-test/src/main/java/rocksdb/RocksDbSuiteTest.java b/hg-store-test/src/main/java/rocksdb/RocksDbSuiteTest.java index fe5af9a56..a5f5011b0 100644 --- a/hg-store-test/src/main/java/rocksdb/RocksDbSuiteTest.java +++ b/hg-store-test/src/main/java/rocksdb/RocksDbSuiteTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package rocksdb; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-test/src/main/java/service/BaseServerTest.java b/hg-store-test/src/main/java/service/BaseServerTest.java index d2d9bc64b..b17b4c3ab 100644 --- a/hg-store-test/src/main/java/service/BaseServerTest.java +++ b/hg-store-test/src/main/java/service/BaseServerTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package service; import org.junit.After; diff --git a/hg-store-test/src/main/java/service/ServerSuiteTest.java b/hg-store-test/src/main/java/service/ServerSuiteTest.java index 5b5597abc..f148d32c5 100644 --- a/hg-store-test/src/main/java/service/ServerSuiteTest.java +++ b/hg-store-test/src/main/java/service/ServerSuiteTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package service; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-test/src/main/java/service/SessionTest.java b/hg-store-test/src/main/java/service/SessionTest.java index 321ad6207..9e0c1cb97 100644 --- a/hg-store-test/src/main/java/service/SessionTest.java +++ b/hg-store-test/src/main/java/service/SessionTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package service; import org.junit.Test; diff --git a/hg-store-test/src/main/java/util/CopyOnWriteCacheTest.java b/hg-store-test/src/main/java/util/CopyOnWriteCacheTest.java index f53512796..195891a6b 100644 --- a/hg-store-test/src/main/java/util/CopyOnWriteCacheTest.java +++ b/hg-store-test/src/main/java/util/CopyOnWriteCacheTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package util; import static org.junit.Assert.assertFalse; diff --git a/hg-store-test/src/main/java/util/HgStoreTestUtil.java b/hg-store-test/src/main/java/util/HgStoreTestUtil.java index 2d763c292..e1752a8d9 100644 --- a/hg-store-test/src/main/java/util/HgStoreTestUtil.java +++ b/hg-store-test/src/main/java/util/HgStoreTestUtil.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package util; import com.baidu.hugegraph.store.HgKvEntry; diff --git a/hg-store-test/src/main/java/util/UnitTestBase.java b/hg-store-test/src/main/java/util/UnitTestBase.java index 1610a85f3..cf5159cf1 100644 --- a/hg-store-test/src/main/java/util/UnitTestBase.java +++ b/hg-store-test/src/main/java/util/UnitTestBase.java @@ -1,11 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package util; import java.io.File; import java.util.HashMap; import java.util.Map; -import com.baidu.hugegraph.rocksdb.access.RocksDBFactory; -import com.baidu.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; import com.baidu.hugegraph.store.business.BusinessHandler; import com.baidu.hugegraph.store.business.BusinessHandlerImpl; import com.baidu.hugegraph.store.meta.PartitionManager; diff --git a/hg-store-test/src/main/java/util/UnsafeUtilTest.java b/hg-store-test/src/main/java/util/UnsafeUtilTest.java index be57a95bb..abaa0ddb6 100644 --- a/hg-store-test/src/main/java/util/UnsafeUtilTest.java +++ b/hg-store-test/src/main/java/util/UnsafeUtilTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + package util; import static org.junit.Assert.assertEquals;