diff --git a/hg-store-cli/pom.xml b/hg-store-cli/pom.xml index ed2d472b3..0749c0516 100644 --- a/hg-store-cli/pom.xml +++ b/hg-store-cli/pom.xml @@ -20,9 +20,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + org.apache.hugegraph hugegraph-store-root - com.baidu.hugegraph - 3.6.3-SNAPSHOT + ${revision} 4.0.0 @@ -40,19 +40,19 @@ - com.baidu.hugegraph + org.apache.hugegraph hg-store-client - ${project.version} + ${revision} - com.baidu.hugegraph + org.apache.hugegraph hg-store-grpc - ${project.version} + ${revision} - com.baidu.hugegraph + org.apache.hugegraph hg-pd-client - ${project.version} + ${revision} diff --git a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/StoreConsoleApplication.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/StoreConsoleApplication.java index 362f9ff6b..858bf9ddb 100644 --- a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/StoreConsoleApplication.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/StoreConsoleApplication.java @@ -19,6 +19,7 @@ package org.apache.hugegraph.store.cli; import java.io.IOException; +import org.apache.hugegraph.store.HgStoreClient; import org.apache.hugegraph.store.cli.loader.HgThread2DB; import org.apache.hugegraph.store.cli.scan.GrpcShardScanner; import org.apache.hugegraph.store.cli.scan.HgStoreCommitter; @@ -30,7 +31,6 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import com.baidu.hugegraph.pd.client.PDConfig; import com.baidu.hugegraph.pd.common.PDException; -import com.baidu.hugegraph.store.HgStoreClient; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/loader/HgThread2DB.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/loader/HgThread2DB.java index 0b75c308b..807a6b3d1 100644 --- a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/loader/HgThread2DB.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/loader/HgThread2DB.java @@ -36,18 +36,18 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.HgStoreClient; +import org.apache.hugegraph.store.HgStoreSession; import org.apache.hugegraph.store.cli.util.HgCliUtil; +import org.apache.hugegraph.store.client.grpc.KvCloseableIterator; +import org.apache.hugegraph.store.client.util.MetricX; import com.baidu.hugegraph.pd.client.PDClient; import com.baidu.hugegraph.pd.client.PDConfig; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgScanQuery; -import com.baidu.hugegraph.store.HgStoreClient; -import com.baidu.hugegraph.store.HgStoreSession; -import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator; -import com.baidu.hugegraph.store.client.util.MetricX; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreCommitter.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreCommitter.java index 0189aa752..48ace5444 100644 --- a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreCommitter.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreCommitter.java @@ -17,12 +17,11 @@ package org.apache.hugegraph.store.cli.scan; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgSessionManager; +import org.apache.hugegraph.store.HgStoreSession; import org.apache.hugegraph.store.cli.util.HgCliUtil; - -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgSessionManager; -import com.baidu.hugegraph.store.HgStoreSession; -import com.baidu.hugegraph.store.client.HgStoreNodeManager; +import org.apache.hugegraph.store.client.HgStoreNodeManager; /** * @author lynn.bond@hotmail.com on 2022/2/28 diff --git a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreScanner.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreScanner.java index d80de3696..a61730906 100644 --- a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreScanner.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreScanner.java @@ -20,22 +20,22 @@ package org.apache.hugegraph.store.cli.scan; import java.util.Arrays; import java.util.List; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvStore; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.HgSessionManager; +import org.apache.hugegraph.store.HgStoreClient; +import org.apache.hugegraph.store.HgStoreSession; import org.apache.hugegraph.store.cli.util.HgCliUtil; import org.apache.hugegraph.store.cli.util.HgMetricX; +import org.apache.hugegraph.store.client.grpc.KvCloseableIterator; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.client.util.MetricX; import com.baidu.hugegraph.pd.client.PDClient; import com.baidu.hugegraph.pd.common.PDException; import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgKvStore; -import com.baidu.hugegraph.store.HgScanQuery; -import com.baidu.hugegraph.store.HgSessionManager; -import com.baidu.hugegraph.store.HgStoreClient; -import com.baidu.hugegraph.store.HgStoreSession; -import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator; -import com.baidu.hugegraph.store.client.util.HgStoreClientConfig; -import com.baidu.hugegraph.store.client.util.MetricX; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgCliUtil.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgCliUtil.java index 38e2256ec..188c524bc 100644 --- a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgCliUtil.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgCliUtil.java @@ -26,10 +26,10 @@ import java.util.List; import java.util.Map; import java.util.function.Function; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgStoreSession; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; /** diff --git a/hg-store-client/deploy-snapshot.sh b/hg-store-client/deploy-snapshot.sh index f35659e36..d4f643995 100644 --- a/hg-store-client/deploy-snapshot.sh +++ b/hg-store-client/deploy-snapshot.sh @@ -1,5 +1,22 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + readonly REPO_URL=http://10.14.139.8:8081/artifactory/star-snapshot mvn --settings ../settings.xml -Dmaven.test.skip=true -DaltDeploymentRepository=star-snapshot::default::${REPO_URL} clean deploy diff --git a/hg-store-client/pom.xml b/hg-store-client/pom.xml index 930a1ef30..5f9e3666e 100644 --- a/hg-store-client/pom.xml +++ b/hg-store-client/pom.xml @@ -1,12 +1,29 @@ + + 4.0.0 - com.baidu.hugegraph + org.apache.hugegraph hg-store-client - 3.6.3-SNAPSHOT + ${revision} 11 @@ -18,15 +35,26 @@ - com.baidu.hugegraph + org.apache.hugegraph hg-store-grpc - ${project.version} + ${revision} - com.baidu.hugegraph + org.apache.hugegraph hg-store-common - ${project.version} + ${revision} + + org.apache.hugegraph + hg-pd-client + ${revision} + + + org.apache.hugegraph + hg-pd-grpc + ${revision} + + org.projectlombok lombok @@ -37,18 +65,6 @@ log4j-slf4j-impl ${log4j2.version} - - com.baidu.hugegraph - hg-pd-client - ${project.version} - - - - - com.baidu.hugegraph - hg-pd-grpc - ${project.version} - junit diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvEntry.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvEntry.java deleted file mode 100644 index 68afae352..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvEntry.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.baidu.hugegraph.store; - -/** - * @author lynn.bond@hotmail.com - */ -public interface HgKvEntry { - byte[] key(); - - byte[] value(); - - default int code(){ - return -1; - } -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvIterator.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvIterator.java deleted file mode 100644 index af2323aea..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvIterator.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.baidu.hugegraph.store; - -import java.io.Closeable; -import java.util.Iterator; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/21 - */ -public interface HgKvIterator extends Iterator, HgSeekAble, Closeable { - - byte[] key(); - - byte[] value(); - - void close(); - -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvOrderedIterator.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvOrderedIterator.java deleted file mode 100644 index 52cc5bc6c..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvOrderedIterator.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.baidu.hugegraph.store; - -/** - * @author lynn.bond@hotmail.com created on 2022/03/10 - */ -public interface HgKvOrderedIterator extends HgKvIterator, Comparable { - long getSequence(); -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvPagingIterator.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvPagingIterator.java deleted file mode 100644 index 7b647ce22..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvPagingIterator.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.baidu.hugegraph.store; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/24 - */ -public interface HgKvPagingIterator extends HgKvIterator, HgPageSize { -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgPageSize.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgPageSize.java deleted file mode 100644 index 150d9924b..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgPageSize.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.baidu.hugegraph.store; - -/** - * Return the amount of records returned by one query in pageable-query. - * - * @author lynn.bond@hotmail.com created on 2021/10/24 - */ -public interface HgPageSize { - long getPageSize(); - default boolean isPageEmpty(){ - return false; - } -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgPrivate.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgPrivate.java deleted file mode 100644 index 898571696..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgPrivate.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.baidu.hugegraph.store; - -/** - * @author lynn.bond@hotmail.com - */ -public final class HgPrivate { - - private static HgPrivate instance = new HgPrivate(); - - private HgPrivate() { - } - - static HgPrivate of() { - return instance; - } - -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgSeekAble.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgSeekAble.java deleted file mode 100644 index f26bc3e51..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgSeekAble.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.baidu.hugegraph.store; - -/** - * @author lynn.bond@hotmail.com created on 2022/03/11 - */ -public interface HgSeekAble { - byte[] position(); - - void seek(byte[] position); -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgSessionManager.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgSessionManager.java deleted file mode 100644 index a4aaf4bf7..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgSessionManager.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.baidu.hugegraph.store; - -import com.baidu.hugegraph.store.client.HgStoreSessionProvider; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * Maintain HgStoreSession instances. - * // TODO: Holding more than one HgSessionManager is available,if you want to connect multi HgStore-clusters. - * - * @author lynn.bond@hotmail.com - */ - -@ThreadSafe -public final class HgSessionManager { - private HgSessionProvider sessionProvider; - private final static HgSessionManager instance =new HgSessionManager(); - - private HgSessionManager(){ - // TODO: constructed by SPI - this.sessionProvider=new HgStoreSessionProvider(); - } - - - public static HgSessionManager getInstance(){ - return instance; - } - - /** - * Retrieve or create a HgStoreSession. - * @param graphName - * @return - */ - public HgStoreSession openSession(String graphName){ - return this.sessionProvider.createSession(graphName); - } - -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgSessionProvider.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgSessionProvider.java deleted file mode 100644 index 779c845be..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgSessionProvider.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.baidu.hugegraph.store; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/12 - */ -@ThreadSafe -public interface HgSessionProvider { - HgStoreSession createSession(String graphName); -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgStoreSession.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgStoreSession.java deleted file mode 100644 index 90bcc47d5..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgStoreSession.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.baidu.hugegraph.store; - -import com.baidu.hugegraph.store.client.type.HgStoreClientException; - -/** - * @author lynn.bond@hotmail.com - */ -public interface HgStoreSession extends HgKvStore { - - void beginTx(); - - /** - * @throws IllegalStateException when the tx hasn't been beginning. - * @throws HgStoreClientException when failed to commit . - */ - void commit(); - - /** - * @throws IllegalStateException when the tx hasn't been beginning. - * @throws HgStoreClientException when failed to rollback. - */ - void rollback(); - - boolean isTx(); -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgTkvEntry.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgTkvEntry.java deleted file mode 100644 index fd6699107..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgTkvEntry.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.baidu.hugegraph.store; - -/** - * @author lynn.bond@hotmail.com - */ -public interface HgTkvEntry { - String table(); - byte[] key(); - byte[] value(); -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgTokvEntry.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgTokvEntry.java deleted file mode 100644 index f2d1e264d..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgTokvEntry.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.baidu.hugegraph.store; - -/** - * @author lynn.bond@hotmail.com - */ -public interface HgTokvEntry { - String table(); - HgOwnerKey ownerKey(); - byte[] value(); -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgPrivate.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgPrivate.java deleted file mode 100644 index 184686c25..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgPrivate.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.baidu.hugegraph.store.client; - -import java.util.Arrays; -import java.util.Objects; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/26 - */ -public class HgPrivate { - private final static HgPrivate instance = new HgPrivate(); - - static HgPrivate getInstance() { - return instance; - } - - private HgPrivate() { - - } - -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNode.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNode.java deleted file mode 100644 index c9ab58577..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNode.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.baidu.hugegraph.store.client; - -import com.baidu.hugegraph.store.HgStoreSession; - -import java.util.Set; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/11 - * @version 0.2.0 - */ -public interface HgStoreNode { - - /** - * Return boolean value of being online or not - * @return - */ - default boolean isHealthy(){ - return true; - } - - /** - * Return the unique ID of store-node. - * @return - */ - Long getNodeId(); - - /** - * A string value concatenated by host and port: "host:port" - * @return - */ - String getAddress(); - - /** - * Return a new HgStoreSession instance, that is not Thread safe. - * Return null when the node is not in charge of the graph that was passed from argument. - * @return - */ - HgStoreSession openSession(String graphName); - -} \ No newline at end of file diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeBuilder.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeBuilder.java deleted file mode 100644 index 0734ddb1f..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeBuilder.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.baidu.hugegraph.store.client; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/11 - */ -public interface HgStoreNodeBuilder { - - HgStoreNodeBuilder setNodeId(Long nodeId); - - HgStoreNodeBuilder setAddress(String address); - - /** - * To build a HgStoreNode instance. - * - * @return - */ - HgStoreNode build(); - -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeCandidates.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeCandidates.java deleted file mode 100644 index 3be9048ed..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeCandidates.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.baidu.hugegraph.store.client; - -import java.util.List; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/12 - */ -public final class HgStoreNodeCandidates { - - List nodeList; - - HgStoreNodeCandidates(List nodeList){ - this.nodeList=nodeList; - } - - public int size(){ - return this.nodeList.size(); - } - - public HgStoreNode getNode(int index){ - return this.nodeList.get(index); - } - -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeNotifier.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeNotifier.java deleted file mode 100644 index 3110b42d9..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeNotifier.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.baidu.hugegraph.store.client; - -import com.baidu.hugegraph.store.client.type.HgNodeStatus; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/12 - * @version 1.0.0 - */ -public interface HgStoreNodeNotifier { - - /** - * It will be invoked by NodeManager, when some exception or issue was happened. - * @param graphName - * @param storeNotice - * @return return 0 please, for no matter what. - */ - int notice(String graphName, HgStoreNotice storeNotice); - -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeProvider.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeProvider.java deleted file mode 100644 index 9775ce6f8..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeProvider.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.baidu.hugegraph.store.client; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/27 - */ -public interface HgStoreNodeProvider { - - /** - * Applying a new HgStoreNode instance - * @param graphName - * @param nodeId - * @return - */ - HgStoreNode apply(String graphName, Long nodeId); - -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeSession.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeSession.java deleted file mode 100644 index c1565da4b..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeSession.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.baidu.hugegraph.store.client; - -import com.baidu.hugegraph.store.HgStoreSession; - -/** - * @version 0.1.0 - * @author lynn.bond@hotmail.com created on 2021/10/11 - */ -public interface HgStoreNodeSession extends HgStoreSession { - - /** - * Return the name of graph. - * @return - */ - String getGraphName(); - - /** - * Return an instance of HgStoreNode, which provided the connection of Store-Node machine. - * @return - */ - HgStoreNode getStoreNode(); - - -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreService.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreService.java deleted file mode 100644 index 0ce220655..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreService.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.baidu.hugegraph.store.client; - - -public class HgStoreService { - private static HgStoreService instance=new HgStoreService(); - private HgStoreService(){} - - static HgStoreService of(){ - return instance; - } - - -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreSessionProvider.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreSessionProvider.java deleted file mode 100644 index 0e8d2c637..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreSessionProvider.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.baidu.hugegraph.store.client; - -import com.baidu.hugegraph.store.HgSessionProvider; -import com.baidu.hugegraph.store.HgStoreSession; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/12 - */ -@ThreadSafe -public class HgStoreSessionProvider implements HgSessionProvider { - private MultiNodeSessionFactory sessionFactory=MultiNodeSessionFactory.getInstance(); - - @Override - public HgStoreSession createSession(String graphName) { - return this.sessionFactory.createStoreSession(graphName); - } -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgTkvEntryImpl.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgTkvEntryImpl.java deleted file mode 100644 index f51d37f86..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgTkvEntryImpl.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.baidu.hugegraph.store.client; - -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgTkvEntry; -import com.baidu.hugegraph.store.HgTokvEntry; - -import java.util.Arrays; -import java.util.Objects; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/14 - */ -class HgTkvEntryImpl implements HgTkvEntry { - private String table; - private byte[] key; - private byte[] value; - - HgTkvEntryImpl(String table, byte[] key, byte[] value) { - this.table = table; - this.key = key; - this.value = value; - } - - @Override - public String table() { - return this.table; - } - - @Override - public byte[] key() { - return this.key; - } - - @Override - public byte[] value() { - return this.value; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - HgTkvEntryImpl that = (HgTkvEntryImpl) o; - return Objects.equals(table, that.table) && Arrays.equals(key, that.key) && Arrays.equals(value, that.value); - } - - @Override - public int hashCode() { - int result = Objects.hash(table); - result = 31 * result + Arrays.hashCode(key); - result = 31 * result + Arrays.hashCode(value); - return result; - } - - @Override - public String toString() { - return "HgTkvEntryImpl{" + - "table='" + table + '\'' + - ", key=" + Arrays.toString(key) + - ", value=" + Arrays.toString(value) + - '}'; - } -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgTokvEntryImpl.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgTokvEntryImpl.java deleted file mode 100644 index 2fc9255b0..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgTokvEntryImpl.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.baidu.hugegraph.store.client; - -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgTokvEntry; - -import java.util.Arrays; -import java.util.Objects; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/14 - */ -class HgTokvEntryImpl implements HgTokvEntry { - - private String table; - private HgOwnerKey ownerKey; - private byte[] value; - - HgTokvEntryImpl(String table, HgOwnerKey ownerKey, byte[] value) { - this.table = table; - this.ownerKey = ownerKey; - this.value = value; - } - - @Override - public String table() { - return this.table; - } - - @Override - public HgOwnerKey ownerKey() { - return this.ownerKey; - } - - @Override - public byte[] value() { - return this.value; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - HgTokvEntryImpl that = (HgTokvEntryImpl) o; - return Objects.equals(table, that.table) && Objects.equals(ownerKey, that.ownerKey) && Arrays.equals(value, that.value); - } - - @Override - public int hashCode() { - int result = Objects.hash(table, ownerKey); - result = 31 * result + Arrays.hashCode(value); - return result; - } - - @Override - public String toString() { - return "HgTokvEntryImpl{" + - "table='" + table + '\'' + - ", okv=" + ownerKey + - ", value=" + Arrays.toString(value) + - '}'; - } -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcKvEntryImpl.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcKvEntryImpl.java deleted file mode 100644 index 0bcebc135..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcKvEntryImpl.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.baidu.hugegraph.store.client.grpc; - -import com.baidu.hugegraph.store.HgKvEntry; - -import java.util.Arrays; - -/** - * @author lynn.bond@hotmail.com - */ -class GrpcKvEntryImpl implements HgKvEntry { - private byte[] key; - private byte[] value; - private int code; - - GrpcKvEntryImpl(byte[] k, byte[] v,int code) { - this.key = k; - this.value = v; - this.code=code; - } - - @Override - public int code() { - return code; - } - - @Override - public byte[] key() { - return key; - } - - @Override - public byte[] value() { - return value; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - GrpcKvEntryImpl hgKvEntry = (GrpcKvEntryImpl) o; - return Arrays.equals(key, hgKvEntry.key) && Arrays.equals(value, hgKvEntry.value); - } - - @Override - public int hashCode() { - int result = Arrays.hashCode(key); - result = 31 * result + Arrays.hashCode(value); - return result; - } - - @Override - public String toString() { - return "HgKvEntryImpl{" + - "key=" + Arrays.toString(key) + - ", value=" + Arrays.toString(value) + - ", code=" + code + - '}'; - } -} \ No newline at end of file diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreNodeBuilder.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreNodeBuilder.java deleted file mode 100644 index d534dabba..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreNodeBuilder.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.baidu.hugegraph.store.client.grpc; - -import com.baidu.hugegraph.store.client.HgPrivate; -import com.baidu.hugegraph.store.client.HgStoreNode; -import com.baidu.hugegraph.store.client.HgStoreNodeBuilder; -import com.baidu.hugegraph.store.client.HgStoreNodeManager; -import com.baidu.hugegraph.store.client.util.HgAssert; - -import java.util.concurrent.atomic.AtomicLong; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/12 - */ -public class GrpcStoreNodeBuilder implements HgStoreNodeBuilder { - - private static GrpcStoreSessionClient sessionClient = new GrpcStoreSessionClient(); - private static GrpcStoreStreamClient streamClient = new GrpcStoreStreamClient(); - private static final AtomicLong ids = new AtomicLong(0); - - private Long nodeId; - private String address; - private HgStoreNodeManager nodeManager; - - public GrpcStoreNodeBuilder(HgStoreNodeManager nodeManager, HgPrivate hgPrivate) { - HgAssert.isArgumentNotNull(hgPrivate, "hgPrivate"); - HgAssert.isArgumentNotNull(nodeManager, "nodeManager"); - this.nodeManager = nodeManager; - } - - @Override - public GrpcStoreNodeBuilder setAddress(String address) { - HgAssert.isFalse(HgAssert.isInvalid(address), "The argument is invalid: address."); - this.address = address; - return this; - } - - @Override - public GrpcStoreNodeBuilder setNodeId(Long nodeId) { - HgAssert.isFalse(nodeId == null, "The argument is invalid: nodeId."); - this.nodeId = nodeId; - return this; - } - - @Override - public HgStoreNode build() { - // TODO: delete - if (this.nodeId == null) { - this.nodeId = ids.addAndGet(-1l); - } - - HgAssert.isFalse(this.nodeId == null, "nodeId can't to be null"); - HgAssert.isFalse(this.address == null, "address can't to be null"); - - GrpcStoreNodeImpl node = new GrpcStoreNodeImpl(this.nodeManager, sessionClient, streamClient); - node.setNodeId(this.nodeId); - node.setAddress(this.address); - - return node; - } -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvCloseableIterator.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvCloseableIterator.java deleted file mode 100644 index 370603650..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvCloseableIterator.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.baidu.hugegraph.store.client.grpc; - -import java.io.Closeable; -import java.util.Iterator; - -/** - * @author lynn.bond@hotmail.com on 2022/3/16 - */ -public interface KvCloseableIterator extends Iterator, Closeable { - public void close(); -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvListIterator.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvListIterator.java deleted file mode 100644 index 669ff1803..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvListIterator.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.baidu.hugegraph.store.client.grpc; - -import java.util.Iterator; -import java.util.List; - -/** - * @author lynn.bond@hotmail.com on 2022/4/6 - */ -class KvListIterator implements KvCloseableIterator{ - - private final Iterator iterator; - - KvListIterator(List list){ - this.iterator=list.iterator(); - } - - @Override - public void close() { - /*Nothing to do.*/ - } - - @Override - public boolean hasNext() { - return this.iterator.hasNext(); - } - - @Override - public T next() { - return this.iterator.next(); - } -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/ScanUtil.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/ScanUtil.java deleted file mode 100644 index 747440159..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/ScanUtil.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.baidu.hugegraph.store.client.grpc; - -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; - -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.client.HgStoreNodeSession; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import com.baidu.hugegraph.store.grpc.common.Header; -import com.google.protobuf.ByteString; - -/** - * @author zhangyingjie - * @date 2022/10/8 - **/ -public class ScanUtil { - - - public static Header getHeader(HgStoreNodeSession nodeSession) { - return Header.newBuilder().setGraph(nodeSession.getGraphName()).build(); - } - - public static HgOwnerKey toOk(HgOwnerKey key) { - return key == null ? HgStoreClientConst.EMPTY_OWNER_KEY : key; - } - - public static ByteString toBs(byte[] bytes) { - return ByteString.copyFrom((bytes != null) ? bytes : EMPTY_BYTES); - } - - public static ByteString getHgOwnerKey(HgOwnerKey ownerKey) { - return toBs(toOk(ownerKey).getKey()); - } - - public static byte[] getQuery(byte[] query) { - return query != null ? query : HgStoreClientConst.EMPTY_BYTES; - } - public static long getLimit(long limit) { - return limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE : limit; - } -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/SeekAbleIterator.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/SeekAbleIterator.java deleted file mode 100644 index 7bb985f60..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/SeekAbleIterator.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.baidu.hugegraph.store.client.grpc; - -import com.baidu.hugegraph.store.HgSeekAble; -import com.baidu.hugegraph.store.client.util.HgAssert; - -import java.util.Iterator; -import java.util.function.Consumer; -import java.util.function.Supplier; - -/** - * @author lynn.bond@hotmail.com on 2022/3/11 - */ -class SeekAbleIterator implements Iterator, HgSeekAble { - private Iterator iterator; - private Consumer seeker; - private Supplier positioner; - - public static SeekAbleIterator of(Iterator iterator,Supplier positioner, Consumer seeker) { - HgAssert.isArgumentNotNull(iterator, "iterator"); - HgAssert.isArgumentNotNull(positioner, "positioner"); - HgAssert.isArgumentNotNull(seeker, "seeker"); - return new SeekAbleIterator(iterator,positioner, seeker); - } - - private SeekAbleIterator(Iterator iterator, Supplier positioner,Consumer seeker) { - this.iterator = iterator; - this.positioner=positioner; - this.seeker = seeker; - } - - @Override - public byte[] position() { - return this.positioner.get(); - } - - @Override - public void seek(byte[] position) { - this.seeker.accept(position); - } - - @Override - public boolean hasNext() { - return this.iterator.hasNext(); - } - - @Override - public E next() { - return this.iterator.next(); - } -} \ No newline at end of file diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/SwitchingExecutor.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/SwitchingExecutor.java deleted file mode 100644 index b42585634..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/SwitchingExecutor.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.baidu.hugegraph.store.client.grpc; - -import javax.annotation.concurrent.ThreadSafe; -import java.util.Optional; -import java.util.function.Supplier; - -/** - * @author lynn.bond@hotmail.com on 2021/12/1 - */ -@ThreadSafe -final class SwitchingExecutor { - - static SwitchingExecutor of(){ - return new SwitchingExecutor(); - } - - private SwitchingExecutor(){} - - Optional invoke(Supplier switcher,Supplier trueSupplier,Supplier falseSupplier){ - Optional option = null; - - if(switcher.get()){ - option= Optional.of(trueSupplier.get()); - }else{ - option=Optional.of(falseSupplier.get()); - } - if (option == null) { - option = Optional.empty(); - } - - return option; - } -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/type/HgNodeStatus.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/type/HgNodeStatus.java deleted file mode 100644 index 1f68aac83..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/type/HgNodeStatus.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.baidu.hugegraph.store.client.type; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/26 - */ -public enum HgNodeStatus { - - UNKNOWN(0, "UNKNOWN"), - NOT_EXIST(100,"NOT_EXIST"), // Failed to apply for an instance via node-id from NodeManager. - NOT_ONLINE(105,"NOT_ONLINE"), // Failed to connect to Store-Node at the first time. - NOT_WORK(110,"NOT_WORK"), // When a Store-Node to be not work anymore. - - PARTITION_COMMON_FAULT(200,"PARTITION_COMMON_FAULT"), // - NOT_PARTITION_LEADER(205,"NOT_PARTITION_LEADER") // When a Store-Node is not a specific partition leader. - - ; - - private int status; - private String name; - - private HgNodeStatus(int status,String name){ - this.status=status; - this.name=name; - } - -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/type/HgStoreClientException.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/type/HgStoreClientException.java deleted file mode 100644 index 5d6545fef..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/type/HgStoreClientException.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.baidu.hugegraph.store.client.type; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/27 - */ -public class HgStoreClientException extends RuntimeException { - - public static HgStoreClientException of(String msg) { - return new HgStoreClientException(msg); - } - public static HgStoreClientException of(String msg,Throwable cause) { - return new HgStoreClientException(msg,cause); - } - - public static HgStoreClientException of(Throwable cause) { - return new HgStoreClientException(cause); - } - - public HgStoreClientException(String msg) { - super(msg); - } - - public HgStoreClientException(Throwable cause) { - super(cause); - } - - public HgStoreClientException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgStoreClientConst.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgStoreClientConst.java deleted file mode 100644 index fa9ffbc9e..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgStoreClientConst.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.baidu.hugegraph.store.client.util; - -import com.baidu.hugegraph.store.HgKvStore; -import com.baidu.hugegraph.store.HgOwnerKey; - -import java.util.Collections; -import java.util.List; - -import static com.baidu.hugegraph.store.HgKvStore.*; - -/** - * @author lynn.bond@hotmail.com - */ -public final class HgStoreClientConst { - public final static String DEFAULT_NODE_CLUSTER_ID = "default-node-cluster"; - - public final static String EMPTY_STRING = ""; - public final static String EMPTY_TABLE = ""; - public final static byte[] EMPTY_BYTES = new byte[0]; - public final static byte[] MAX_BYTES=new byte[]{(byte)0b11111111}; - public final static List EMPTY_LIST = Collections.EMPTY_LIST; - - public final static byte[] ALL_PARTITION_OWNER = new byte[0]; // means to dispatch to all partitions. - public final static HgOwnerKey EMPTY_OWNER_KEY = HgOwnerKey.of(EMPTY_BYTES, EMPTY_BYTES); - public final static HgOwnerKey ALL_PARTITION_OWNER_KEY = HgOwnerKey.of(ALL_PARTITION_OWNER, ALL_PARTITION_OWNER); - - //public final static int SCAN_GTE_BEGIN_LT_END = SCAN_GTE_BEGIN | SCAN_LT_END; - public final static int SCAN_TYPE_RANGE= SCAN_GTE_BEGIN | SCAN_LTE_END; - public final static int SCAN_TYPE_ANY = SCAN_ANY; - public final static int NO_LIMIT=0; - - public final static int TX_SESSIONS_MAP_CAPACITY=32; - public static final int NODE_MAX_RETRYING_TIMES=10; - - -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgUuid.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgUuid.java deleted file mode 100644 index 93577a94e..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgUuid.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.baidu.hugegraph.store.client.util; - -import java.nio.ByteBuffer; -import java.util.UUID; - -/** - * @author lynn.bond@hotmail.com - */ -public final class HgUuid { - - private static String encode(UUID uuid) { - ByteBuffer bb = ByteBuffer.wrap(new byte[16]); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - return Base58.encode(bb.array()); - } - - /** - * Get a UUID in Base58 FORM - * @return - */ - public static String newUUID() { - return encode(UUID.randomUUID()); - } - -} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/MetricX.java b/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/MetricX.java deleted file mode 100644 index 2e897dd41..000000000 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/MetricX.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.baidu.hugegraph.store.client.util; - -import java.util.concurrent.atomic.AtomicLong; - -/** - * @author lynn.bond@hotmail.com on 2022/1/29 - */ -public class MetricX { - public static AtomicLong iteratorSum=new AtomicLong(); - public static AtomicLong iteratorCount=new AtomicLong(); - public static AtomicLong iteratorMax=new AtomicLong(); - - public AtomicLong failureCount=new AtomicLong(); - - private long start; - private long end; - - public static MetricX ofStart(){ - return new MetricX(System.currentTimeMillis()); - } - - private MetricX(long start){ - this.start=start; - }; - - public long start(){ - return this.start=System.currentTimeMillis(); - } - - public long end(){ - return this.end=System.currentTimeMillis(); - } - - public long past(){ - return this.end-this.start; - } - - public static void plusIteratorWait(long nanoSeconds){ - iteratorSum.addAndGet(nanoSeconds); - iteratorCount.getAndIncrement(); - if(iteratorMax.get() extends Iterator, HgSeekAble, Closeable { + + byte[] key(); + + byte[] value(); + + void close(); + +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/HgKvOrderedIterator.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgKvOrderedIterator.java new file mode 100644 index 000000000..1e315d581 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgKvOrderedIterator.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +/** + * @author lynn.bond@hotmail.com created on 2022/03/10 + */ +public interface HgKvOrderedIterator extends HgKvIterator, Comparable { + long getSequence(); +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/HgKvPagingIterator.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgKvPagingIterator.java new file mode 100644 index 000000000..abf50b0d9 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgKvPagingIterator.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/24 + */ +public interface HgKvPagingIterator extends HgKvIterator, HgPageSize { +} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvStore.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgKvStore.java similarity index 64% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvStore.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/HgKvStore.java index b5b5954aa..e2c9dc91b 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgKvStore.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgKvStore.java @@ -1,12 +1,28 @@ -package com.baidu.hugegraph.store; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator; -import com.baidu.hugegraph.store.grpc.common.Kv; -import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq; +package org.apache.hugegraph.store; -import java.util.Iterator; import java.util.List; +import org.apache.hugegraph.store.client.grpc.KvCloseableIterator; + +import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq; + /** * @author lynn.bond@hotmail.com * @version 0.2.0 @@ -31,7 +47,8 @@ public interface HgKvStore { * 该版本被store内部使用。向分区写入数据， * partitionId与key.keyCode必须与pd存储的分区信息保持一致。 */ - boolean directPut(String table, int partitionId, HgOwnerKey key, byte[] value); + boolean directPut(String table, int partitionId, HgOwnerKey key, byte[] value); + byte[] get(String table, HgOwnerKey ownerKey); boolean clean(int partId); @@ -45,6 +62,7 @@ public interface HgKvStore { boolean deleteRange(String table, HgOwnerKey start, HgOwnerKey end); boolean merge(String table, HgOwnerKey key, byte[] value); + @Deprecated List batchGetOwner(String table, List keyList); @@ -60,17 +78,22 @@ public interface HgKvStore { HgKvIterator scanIterator(String table, HgOwnerKey keyPrefix, long limit); - HgKvIterator scanIterator(String table, HgOwnerKey keyPrefix, long limit, byte[] query); + HgKvIterator scanIterator(String table, HgOwnerKey keyPrefix, long limit, + byte[] query); HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey); - HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit); + HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, + long limit); - HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit, byte[] query); + HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, + long limit, byte[] query); - HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit, int scanType, byte[] query); + HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, + long limit, int scanType, byte[] query); - HgKvIterator scanIterator(String table, int codeFrom, int codeTo, int scanType, byte[] query); + HgKvIterator scanIterator(String table, int codeFrom, int codeTo, int scanType, + byte[] query); // HgKvIterator scanIterator(ScanStreamReq scanReq); @@ -89,10 +112,13 @@ public interface HgKvStore { boolean dropTable(String table); boolean deleteGraph(String graph); + List> scanBatch(HgScanQuery scanQuery); KvCloseableIterator> scanBatch2(HgScanQuery scanQuery); - KvCloseableIterator> scanBatch3(HgScanQuery scanQuery, KvCloseableIterator iterator); + + KvCloseableIterator> scanBatch3(HgScanQuery scanQuery, + KvCloseableIterator iterator); HgKvIterator batchPrefix(String table, List prefixList); } diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgOwnerKey.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgOwnerKey.java similarity index 56% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/HgOwnerKey.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/HgOwnerKey.java index db60b6c28..54495b981 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgOwnerKey.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgOwnerKey.java @@ -1,18 +1,32 @@ -package com.baidu.hugegraph.store; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.client.util.HgStoreClientUtil; +package org.apache.hugegraph.store; + +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_OWNER_KEY; import java.io.Serializable; import java.util.Arrays; -import static com.baidu.hugegraph.store.client.util.HgAssert.isArgumentNotNull; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_OWNER_KEY; +import org.apache.hugegraph.store.client.util.HgStoreClientUtil; /** * @author lynn.bond@hotmail.com created on 2021/10/14 - * @version 1.1.0 add argument assertion. - * @version 1.2.0 add newEmpty * @version 1.3.0 add canceled assert */ public class HgOwnerKey implements Serializable { @@ -21,32 +35,34 @@ public class HgOwnerKey implements Serializable { private byte[] key; private int serialNo; //顺序号，用于批量查询保证返回结果的顺序性 - public static HgOwnerKey emptyOf(){ + public static HgOwnerKey emptyOf() { return EMPTY_OWNER_KEY; } - public static HgOwnerKey newEmpty(){ - return HgOwnerKey.of(EMPTY_BYTES,EMPTY_BYTES); + public static HgOwnerKey newEmpty() { + return HgOwnerKey.of(EMPTY_BYTES, EMPTY_BYTES); } - public static HgOwnerKey ownerOf(byte[] owner){ - return new HgOwnerKey(owner,EMPTY_BYTES); + public static HgOwnerKey ownerOf(byte[] owner) { + return new HgOwnerKey(owner, EMPTY_BYTES); } - public static HgOwnerKey codeOf(int code){ + + public static HgOwnerKey codeOf(int code) { return HgOwnerKey.of(EMPTY_BYTES, EMPTY_BYTES).setKeyCode(code); } - public static HgOwnerKey of(byte[] owner, byte[] key){ - return new HgOwnerKey(owner,key); + public static HgOwnerKey of(byte[] owner, byte[] key) { + return new HgOwnerKey(owner, key); } - public static HgOwnerKey of(int keyCode, byte[] key){ + + public static HgOwnerKey of(int keyCode, byte[] key) { return new HgOwnerKey(keyCode, key); } /** - * @see HgOwnerKey:of(byte[] owner, byte[] key) * @param owner * @param key + * @see HgOwnerKey:of(byte[] owner, byte[] key) */ @Deprecated public HgOwnerKey(byte[] owner, byte[] key) { @@ -92,14 +108,15 @@ public class HgOwnerKey implements Serializable { return this; } - public HgOwnerKey setSerialNo(int serialNo){ + public HgOwnerKey setSerialNo(int serialNo) { this.serialNo = serialNo; return this; } - public int getSerialNo(){ + public int getSerialNo() { return this.serialNo; } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -118,10 +135,10 @@ public class HgOwnerKey implements Serializable { @Override public String toString() { return "HgOwnerKey{" + - "owner=" + Arrays.toString(owner) + - ", key=" + Arrays.toString(key) + - ", code=" + keyCode + - '}'; + "owner=" + Arrays.toString(owner) + + ", key=" + Arrays.toString(key) + + ", code=" + keyCode + + '}'; } } diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/HgPageSize.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgPageSize.java new file mode 100644 index 000000000..aa01b7710 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgPageSize.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +/** + * Return the amount of records returned by one query in pageable-query. + * + * @author lynn.bond@hotmail.com created on 2021/10/24 + */ +public interface HgPageSize { + long getPageSize(); + + default boolean isPageEmpty() { + return false; + } +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/HgPrivate.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgPrivate.java new file mode 100644 index 000000000..8da436912 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgPrivate.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +/** + * @author lynn.bond@hotmail.com + */ +public final class HgPrivate { + + private static final HgPrivate instance = new HgPrivate(); + + private HgPrivate() { + } + + static HgPrivate of() { + return instance; + } + +} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgScanQuery.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgScanQuery.java similarity index 81% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/HgScanQuery.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/HgScanQuery.java index da1aed0c6..f497739b7 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgScanQuery.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgScanQuery.java @@ -1,13 +1,29 @@ -package com.baidu.hugegraph.store; - -import com.baidu.hugegraph.store.client.util.HgAssert; -import com.baidu.hugegraph.store.grpc.common.ScanOrderType; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hugegraph.store; import java.util.Collections; import java.util.Iterator; import java.util.List; +import org.apache.hugegraph.store.client.util.HgAssert; +import org.apache.hugegraph.store.grpc.common.ScanOrderType; + /** * @author lynn.bond@hotmail.com on 2022/3/4 * @version 0.5.0 @@ -34,6 +50,7 @@ public interface HgScanQuery { long getSkipDegree(); int getScanType(); + ScanOrderType getOrderType(); boolean isOnlyKey(); @@ -41,11 +58,15 @@ public interface HgScanQuery { byte[] getQuery(); enum ScanMethod { - ALL, PREFIX, RANGE + ALL, + PREFIX, + RANGE } enum SortType { - UNSORTED, SORT_BY_EDGE, SORT_BY_VERTEX + UNSORTED, + SORT_BY_EDGE, + SORT_BY_VERTEX } @@ -63,7 +84,8 @@ public interface HgScanQuery { return ScanBuilder.prefixOf(table, prefixList).build(); } - static HgScanQuery prefixOf(String table, List prefixList, ScanOrderType orderType) { + static HgScanQuery prefixOf(String table, List prefixList, + ScanOrderType orderType) { return ScanBuilder.prefixOf(table, prefixList).setOrderType(orderType).build(); } @@ -71,7 +93,8 @@ public interface HgScanQuery { return ScanBuilder.prefixIteratorOf(table, prefixItr).build(); } - static HgScanQuery prefixIteratorOf(String table, Iterator prefixItr, ScanOrderType orderType) { + static HgScanQuery prefixIteratorOf(String table, Iterator prefixItr, + ScanOrderType orderType) { return ScanBuilder.prefixIteratorOf(table, prefixItr).setOrderType(orderType).build(); } @@ -93,7 +116,8 @@ public interface HgScanQuery { private List endList; private Iterator prefixItr; - public static ScanBuilder rangeOf(String table, List startList, List endList) { + public static ScanBuilder rangeOf(String table, List startList, + List endList) { HgAssert.isArgumentValid(table, "table"); HgAssert.isArgumentValid(startList, "startList"); HgAssert.isArgumentValid(endList, "endList"); @@ -158,7 +182,7 @@ public interface HgScanQuery { return this; } - public ScanBuilder setOrderType(ScanOrderType orderType){ + public ScanBuilder setOrderType(ScanOrderType orderType) { this.orderType = orderType; return this; } @@ -167,12 +191,13 @@ public interface HgScanQuery { this.query = query; return this; } - public ScanBuilder setSkipDegree(long skipDegree){ + + public ScanBuilder setSkipDegree(long skipDegree) { this.skipDegree = skipDegree; return this; } - public ScanBuilder setOnlyKey(boolean onlyKey){ + public ScanBuilder setOnlyKey(boolean onlyKey) { this.onlyKey = onlyKey; return this; } @@ -287,11 +312,13 @@ public interface HgScanQuery { sb.append(", orderType=").append(getOrderType()); sb.append(", onlyKey=").append(isOnlyKey()); sb.append(", query="); - if (query == null) sb.append("null"); - else { + if (query == null) { + sb.append("null"); + } else { sb.append('['); - for (int i = 0; i < query.length; ++i) + for (int i = 0; i < query.length; ++i) { sb.append(i == 0 ? "" : ", ").append(query[i]); + } sb.append(']'); } sb.append('}'); diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/HgSeekAble.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgSeekAble.java new file mode 100644 index 000000000..32114fd9e --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgSeekAble.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +/** + * @author lynn.bond@hotmail.com created on 2022/03/11 + */ +public interface HgSeekAble { + byte[] position(); + + void seek(byte[] position); +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/HgSessionManager.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgSessionManager.java new file mode 100644 index 000000000..ba1fccdce --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgSessionManager.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import javax.annotation.concurrent.ThreadSafe; + +import org.apache.hugegraph.store.client.HgStoreSessionProvider; + +/** + * Maintain HgStoreSession instances. + * // TODO: Holding more than one HgSessionManager is available,if you want to connect multi + * HgStore-clusters. + * + * @author lynn.bond@hotmail.com + */ + +@ThreadSafe +public final class HgSessionManager { + private final HgSessionProvider sessionProvider; + private final static HgSessionManager INSTANCE = new HgSessionManager(); + + private HgSessionManager() { + // TODO: constructed by SPI + this.sessionProvider = new HgStoreSessionProvider(); + } + + + public static HgSessionManager getInstance() { + return INSTANCE; + } + + /** + * Retrieve or create a HgStoreSession. + * + * @param graphName + * @return + */ + public HgStoreSession openSession(String graphName) { + return this.sessionProvider.createSession(graphName); + } + +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/HgSessionProvider.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgSessionProvider.java new file mode 100644 index 000000000..ccdb20066 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgSessionProvider.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import javax.annotation.concurrent.ThreadSafe; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/12 + */ +@ThreadSafe +public interface HgSessionProvider { + HgStoreSession createSession(String graphName); +} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgStoreClient.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgStoreClient.java similarity index 57% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/HgStoreClient.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/HgStoreClient.java index 131a6bac1..f34909cb3 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/HgStoreClient.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgStoreClient.java @@ -1,23 +1,41 @@ -package com.baidu.hugegraph.store; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.pd.client.PDClient; -import com.baidu.hugegraph.pd.client.PDConfig; -import com.baidu.hugegraph.store.client.HgStoreNodeManager; -import com.baidu.hugegraph.store.client.HgStoreNodePartitionerImpl; -import com.baidu.hugegraph.store.client.HgStoreSessionProvider; +package org.apache.hugegraph.store; import javax.annotation.concurrent.ThreadSafe; +import org.apache.hugegraph.pd.client.PDClient; +import org.apache.hugegraph.pd.client.PDConfig; +import org.apache.hugegraph.store.client.HgStoreNodeManager; +import org.apache.hugegraph.store.client.HgStoreNodePartitionerImpl; +import org.apache.hugegraph.store.client.HgStoreSessionProvider; + /** * Maintain HgStoreSession instances. - * // TODO: Holding more than one HgSessionManager is available,if you want to connect multi HgStore-clusters. + * // TODO: Holding more than one HgSessionManager is available,if you want to connect multi + * HgStore-clusters. * * @author lynn.bond@hotmail.com */ @ThreadSafe public final class HgStoreClient { - private HgSessionProvider sessionProvider; + private final HgSessionProvider sessionProvider; private PDClient pdClient; public HgStoreClient() { @@ -29,10 +47,12 @@ public final class HgStoreClient { pdClient = PDClient.create(config); setPdClient(pdClient); } + public HgStoreClient(PDClient pdClient) { this.sessionProvider = new HgStoreSessionProvider(); setPdClient(pdClient); } + public static HgStoreClient create(PDConfig config) { return new HgStoreClient(config); } @@ -41,7 +61,7 @@ public final class HgStoreClient { return new HgStoreClient(pdClient); } - public static HgStoreClient create(){ + public static HgStoreClient create() { return new HgStoreClient(); } @@ -71,7 +91,7 @@ public final class HgStoreClient { return this.sessionProvider.createSession(graphName); } - public PDClient getPdClient(){ + public PDClient getPdClient() { return pdClient; } diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/HgStoreSession.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgStoreSession.java new file mode 100644 index 000000000..860d58f8c --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgStoreSession.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import org.apache.hugegraph.store.client.type.HgStoreClientException; + +/** + * @author lynn.bond@hotmail.com + */ +public interface HgStoreSession extends HgKvStore { + + void beginTx(); + + /** + * @throws IllegalStateException when the tx hasn't been beginning. + * @throws HgStoreClientException when failed to commit . + */ + void commit(); + + /** + * @throws IllegalStateException when the tx hasn't been beginning. + * @throws HgStoreClientException when failed to rollback. + */ + void rollback(); + + boolean isTx(); +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/HgTkvEntry.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgTkvEntry.java new file mode 100644 index 000000000..c13d529ab --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgTkvEntry.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +/** + * @author lynn.bond@hotmail.com + */ +public interface HgTkvEntry { + String table(); + + byte[] key(); + + byte[] value(); +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/HgTokvEntry.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgTokvEntry.java new file mode 100644 index 000000000..6b51bfdbf --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgTokvEntry.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +/** + * @author lynn.bond@hotmail.com + */ +public interface HgTokvEntry { + String table(); + + HgOwnerKey ownerKey(); + + byte[] value(); +} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgNodePartition.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgNodePartition.java similarity index 52% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgNodePartition.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgNodePartition.java index 8eff82d83..0259d5654 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgNodePartition.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgNodePartition.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; import java.util.Objects; @@ -8,23 +25,25 @@ import java.util.Objects; * @author lynn.bond@hotmail.com created on 2021/10/26 */ public final class HgNodePartition { - private Long nodeId; + private final Long nodeId; //当前key的hashcode - private Integer keyCode; + private final Integer keyCode; //分区的开始结束范围 - private Integer startKey; - private Integer endKey; + private final Integer startKey; + private final Integer endKey; private int hash = -1; public static HgNodePartition of(Long nodeId, Integer keyCode) { return new HgNodePartition(nodeId, keyCode); } - public static HgNodePartition of(Long nodeId, Integer keyCode, Integer startKey, Integer endKey) { + + public static HgNodePartition of(Long nodeId, Integer keyCode, Integer startKey, + Integer endKey) { return new HgNodePartition(nodeId, keyCode, startKey, endKey); } - HgNodePartition(Long nodeId, Integer keyCode) { + HgNodePartition(Long nodeId, Integer keyCode) { this.nodeId = nodeId; this.keyCode = keyCode; this.startKey = this.endKey = keyCode; @@ -36,6 +55,7 @@ public final class HgNodePartition { this.startKey = startKey; this.endKey = endKey; } + public Long getNodeId() { return nodeId; } @@ -44,13 +64,14 @@ public final class HgNodePartition { return keyCode; } - public Integer getStartKey(){ + public Integer getStartKey() { return startKey; } - public Integer getEndKey(){ + public Integer getEndKey() { return endKey; } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -61,15 +82,15 @@ public final class HgNodePartition { @Override public int hashCode() { - if(this.hash==-1)this.hash=Objects.hash(nodeId, keyCode); + if (this.hash == -1) this.hash = Objects.hash(nodeId, keyCode); return this.hash; } @Override public String toString() { return "HgNodePartition{" + - "nodeId=" + nodeId + - ", partitionId=" + keyCode + - '}'; + "nodeId=" + nodeId + + ", partitionId=" + keyCode + + '}'; } } diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgNodePartitionerBuilder.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgNodePartitionerBuilder.java similarity index 50% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgNodePartitionerBuilder.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgNodePartitionerBuilder.java index e706ad1ce..bb4497ab7 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgNodePartitionerBuilder.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgNodePartitionerBuilder.java @@ -1,10 +1,29 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +import static org.apache.hugegraph.store.client.util.HgAssert.isFalse; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; import javax.annotation.concurrent.NotThreadSafe; -import java.net.CookieHandler; -import java.util.*; - -import static com.baidu.hugegraph.store.client.util.HgAssert.isFalse; /** * @author lynn.bond@hotmail.com created on 2021/10/26 @@ -14,8 +33,8 @@ import static com.baidu.hugegraph.store.client.util.HgAssert.isFalse; public final class HgNodePartitionerBuilder { private Set partitions = null; + /** - * * @param nodeId * @param keyCode * @return @@ -26,8 +45,8 @@ public final class HgNodePartitionerBuilder { isFalse(nodeId == null, "The argument is invalid: nodeId"); isFalse(keyCode == null, "The argument is invalid: keyCode"); - if(this.partitions==null){ - this.partitions=new HashSet<>(16,1); + if (this.partitions == null) { + this.partitions = new HashSet<>(16, 1); } this.partitions.add(HgNodePartition.of(nodeId, keyCode)); diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgPrivate.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgPrivate.java new file mode 100644 index 000000000..15b4a26f8 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgPrivate.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/26 + */ +public class HgPrivate { + private final static HgPrivate instance = new HgPrivate(); + + static HgPrivate getInstance() { + return instance; + } + + private HgPrivate() { + + } + +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNode.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNode.java new file mode 100644 index 000000000..e75a99371 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNode.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +import org.apache.hugegraph.store.HgStoreSession; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/11 + * @version 0.2.0 + */ +public interface HgStoreNode { + + /** + * Return boolean value of being online or not + * + * @return + */ + default boolean isHealthy() { + return true; + } + + /** + * Return the unique ID of store-node. + * + * @return + */ + Long getNodeId(); + + /** + * A string value concatenated by host and port: "host:port" + * + * @return + */ + String getAddress(); + + /** + * Return a new HgStoreSession instance, that is not Thread safe. + * Return null when the node is not in charge of the graph that was passed from argument. + * + * @return + */ + HgStoreSession openSession(String graphName); + +} \ No newline at end of file diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeBuilder.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeBuilder.java new file mode 100644 index 000000000..377ecf469 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeBuilder.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/11 + */ +public interface HgStoreNodeBuilder { + + HgStoreNodeBuilder setNodeId(Long nodeId); + + HgStoreNodeBuilder setAddress(String address); + + /** + * To build a HgStoreNode instance. + * + * @return + */ + HgStoreNode build(); + +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeCandidates.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeCandidates.java new file mode 100644 index 000000000..b69f53035 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeCandidates.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +import java.util.List; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/12 + */ +public final class HgStoreNodeCandidates { + + List nodeList; + + HgStoreNodeCandidates(List nodeList) { + this.nodeList = nodeList; + } + + public int size() { + return this.nodeList.size(); + } + + public HgStoreNode getNode(int index) { + return this.nodeList.get(index); + } + +} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeManager.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeManager.java similarity index 80% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeManager.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeManager.java index 6600f271b..6f3d283a0 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodeManager.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeManager.java @@ -1,15 +1,38 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.client.grpc.GrpcStoreNodeBuilder; -import com.baidu.hugegraph.store.client.type.HgNodeStatus; -import com.baidu.hugegraph.store.client.type.HgStoreClientException; -import com.baidu.hugegraph.store.client.util.HgAssert; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.client; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import javax.annotation.concurrent.ThreadSafe; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; + +import org.apache.hugegraph.store.client.grpc.GrpcStoreNodeBuilder; +import org.apache.hugegraph.store.client.type.HgNodeStatus; +import org.apache.hugegraph.store.client.type.HgStoreClientException; +import org.apache.hugegraph.store.client.util.HgAssert; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; + +import lombok.extern.slf4j.Slf4j; /** * // TODO: Mapping to Store-Node-Cluster, one to one. @@ -98,7 +121,7 @@ public final class HgStoreNodeManager { if (node == null) { log.warn("Failed to apply a HgStoreNode instance form the nodeProvider [ " - + this.nodeProvider.getClass().getName() + " ]."); + + this.nodeProvider.getClass().getName() + " ]."); notifying(graphName, nodeId, HgNodeStatus.NOT_EXIST); return null; } @@ -113,7 +136,8 @@ public final class HgStoreNodeManager { try { this.nodeNotifier.notice(graphName, HgStoreNotice.of(nodeId, status)); } catch (Throwable t) { - log.error("Failed to invoke " + this.nodeNotifier.getClass().getSimpleName() + ":notice(" + nodeId + "," + status + ")", t); + log.error("Failed to invoke " + this.nodeNotifier.getClass().getSimpleName() + + ":notice(" + nodeId + "," + status + ")", t); } } } @@ -132,7 +156,9 @@ public final class HgStoreNodeManager { try { return this.nodeNotifier.notice(graphName, notice); } catch (Throwable t) { - String msg = "Failed to invoke " + this.nodeNotifier.getClass().getSimpleName() + ", notice: [ " + notice + " ]"; + String msg = + "Failed to invoke " + this.nodeNotifier.getClass().getSimpleName() + + ", notice: [ " + notice + " ]"; log.error(msg, t); throw new HgStoreClientException(msg); } diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeNotifier.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeNotifier.java new file mode 100644 index 000000000..cae2b49e1 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeNotifier.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/12 + * @version 1.0.0 + */ +public interface HgStoreNodeNotifier { + + /** + * It will be invoked by NodeManager, when some exception or issue was happened. + * + * @param graphName + * @param storeNotice + * @return return 0 please, for no matter what. + */ + int notice(String graphName, HgStoreNotice storeNotice); + +} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodePartitioner.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodePartitioner.java similarity index 53% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodePartitioner.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodePartitioner.java index 9979a4fd0..a6c7393b6 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodePartitioner.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodePartitioner.java @@ -1,7 +1,23 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; +package org.apache.hugegraph.store.client; + +import org.apache.hugegraph.store.client.util.HgStoreClientConst; /** * @author lynn.bond@hotmail.com created on 2021/10/12 @@ -12,7 +28,8 @@ public interface HgStoreNodePartitioner { /** * The partition algorithm implementation, that specialized by user. * - * @param builder The builder of HgNodePartitionerBuilder. It's supposed to be invoked directly by user. + * @param builder The builder of HgNodePartitionerBuilder. It's supposed to be invoked + * directly by user. * e.g. builder.add(nodeId,address,partitionId); * @param graphName * @param startKey @@ -23,7 +40,8 @@ public interface HgStoreNodePartitioner { *
  • 10: The partitioner is not work.
    • * */ - int partition(HgNodePartitionerBuilder builder, String graphName, byte[] startKey, byte[] endKey); + int partition(HgNodePartitionerBuilder builder, String graphName, byte[] startKey, + byte[] endKey); /** * @param builder @@ -32,7 +50,8 @@ public interface HgStoreNodePartitioner { * @param endCode hash code * @return */ - default int partition(HgNodePartitionerBuilder builder, String graphName, int startCode, int endCode) { + default int partition(HgNodePartitionerBuilder builder, String graphName, int startCode, + int endCode) { return this.partition(builder, graphName , HgStoreClientConst.ALL_PARTITION_OWNER , HgStoreClientConst.ALL_PARTITION_OWNER); diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodePartitionerImpl.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodePartitionerImpl.java similarity index 75% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodePartitionerImpl.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodePartitionerImpl.java index 1aecdabf4..b57b1b12d 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNodePartitionerImpl.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodePartitionerImpl.java @@ -1,22 +1,41 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; + +import org.apache.hugegraph.store.client.type.HgNodeStatus; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; import com.baidu.hugegraph.pd.client.PDClient; import com.baidu.hugegraph.pd.common.KVPair; import com.baidu.hugegraph.pd.common.PDException; import com.baidu.hugegraph.pd.common.PartitionUtils; import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.client.type.HgNodeStatus; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import lombok.extern.slf4j.Slf4j; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; +import lombok.extern.slf4j.Slf4j; @Slf4j public class HgStoreNodePartitionerImpl implements HgStoreNodePartitioner, - HgStoreNodeProvider, - HgStoreNodeNotifier { + HgStoreNodeProvider, + HgStoreNodeNotifier { private PDClient pdClient; private HgStoreNodeManager nodeManager; @@ -44,15 +63,17 @@ public class HgStoreNodePartitionerImpl implements HgStoreNodePartitioner, } } else if (endKey == HgStoreClientConst.EMPTY_BYTES - || startKey == endKey || Arrays.equals(startKey, endKey)) { + || startKey == endKey || Arrays.equals(startKey, endKey)) { KVPair partShard = pdClient.getPartition(graphName, startKey); Metapb.Shard leader = partShard.getValue(); partitions = new HashSet<>(); partitions.add(HgNodePartition.of(leader.getStoreId(), - pdClient.keyToCode(graphName, startKey))); + pdClient.keyToCode(graphName, startKey))); } else { - log.warn("StartOwnerkey is not equal to endOwnerkey, which is meaningless!!, It is a error!!"); + log.warn( + "StartOwnerkey is not equal to endOwnerkey, which is meaningless!!, It is" + + " a error!!"); List stores = pdClient.getActiveStores(graphName); partitions = new HashSet<>(stores.size()); for (Metapb.Store store : stores) { @@ -74,14 +95,15 @@ public class HgStoreNodePartitionerImpl implements HgStoreNodePartitioner, HashSet partitions = new HashSet<>(); Metapb.Partition partition = null; while ((partition == null || partition.getEndKey() < endKey) - && startKey < PartitionUtils.MAX_VALUE) { + && startKey < PartitionUtils.MAX_VALUE) { KVPair partShard = pdClient.getPartitionByCode(graphName, startKey); if (partShard != null) { partition = partShard.getKey(); Metapb.Shard leader = partShard.getValue(); partitions.add(HgNodePartition.of(leader.getStoreId(), startKey, - (int) partition.getStartKey(), (int) partition.getEndKey())); + (int) partition.getStartKey(), + (int) partition.getEndKey())); startKey = (int) partition.getEndKey(); } else { break; @@ -108,7 +130,8 @@ public class HgStoreNodePartitionerImpl implements HgStoreNodePartitioner, if (partShard != null) { partition = partShard.getKey(); Metapb.Shard leader = partShard.getValue(); - partitions.add(HgNodePartition.of(leader.getStoreId(), (int) partition.getStartKey())); + partitions.add( + HgNodePartition.of(leader.getStoreId(), (int) partition.getStartKey())); } builder.setPartitions(partitions); } catch (PDException e) { @@ -117,6 +140,7 @@ public class HgStoreNodePartitionerImpl implements HgStoreNodePartitioner, } return 0; } + /** * 查询hgstore信息 * @@ -127,7 +151,7 @@ public class HgStoreNodePartitionerImpl implements HgStoreNodePartitioner, try { Metapb.Store store = pdClient.getStore(nodeId); return nodeManager.getNodeBuilder().setNodeId(store.getId()) - .setAddress(store.getAddress()).build(); + .setAddress(store.getAddress()).build(); } catch (PDException e) { throw new RuntimeException(e.getMessage(), e); } @@ -143,7 +167,7 @@ public class HgStoreNodePartitionerImpl implements HgStoreNodePartitioner, storeNotice.getPartitionLeaders().forEach((partId, leader) -> { pdClient.updatePartitionLeader(graphName, partId, leader); log.warn("updatePartitionLeader:{}-{}-{}", - graphName, partId, leader); + graphName, partId, leader); }); } if (storeNotice.getPartitionIds() != null) { @@ -153,7 +177,7 @@ public class HgStoreNodePartitionerImpl implements HgStoreNodePartitioner, } if (!storeNotice.getNodeStatus().equals( HgNodeStatus.PARTITION_COMMON_FAULT) - && !storeNotice.getNodeStatus().equals( + && !storeNotice.getNodeStatus().equals( HgNodeStatus.NOT_PARTITION_LEADER)) { pdClient.invalidPartitionCache(); log.warn("invalidPartitionCache:{} ", storeNotice.getNodeStatus()); diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeProvider.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeProvider.java new file mode 100644 index 000000000..7fc1ef86a --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeProvider.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/27 + */ +public interface HgStoreNodeProvider { + + /** + * Applying a new HgStoreNode instance + * + * @param graphName + * @param nodeId + * @return + */ + HgStoreNode apply(String graphName, Long nodeId); + +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeSession.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeSession.java new file mode 100644 index 000000000..fcc086db6 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeSession.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +import org.apache.hugegraph.store.HgStoreSession; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/11 + * @version 0.1.0 + */ +public interface HgStoreNodeSession extends HgStoreSession { + + /** + * Return the name of graph. + * + * @return + */ + String getGraphName(); + + /** + * Return an instance of HgStoreNode, which provided the connection of Store-Node machine. + * + * @return + */ + HgStoreNode getStoreNode(); + + +} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNotice.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNotice.java similarity index 57% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNotice.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNotice.java index 627b8a872..20326180e 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/HgStoreNotice.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreNotice.java @@ -1,21 +1,38 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.client.type.HgNodeStatus; -import com.baidu.hugegraph.store.client.util.HgAssert; +package org.apache.hugegraph.store.client; -import javax.annotation.concurrent.NotThreadSafe; import java.util.List; import java.util.Map; -import java.util.Objects; + +import javax.annotation.concurrent.NotThreadSafe; + +import org.apache.hugegraph.store.client.type.HgNodeStatus; +import org.apache.hugegraph.store.client.util.HgAssert; /** * @author lynn.bond@hotmail.com on 2021/11/16 */ @NotThreadSafe public class HgStoreNotice { - private Long nodeId; - private HgNodeStatus nodeStatus; - private String message; + private final Long nodeId; + private final HgNodeStatus nodeStatus; + private final String message; private Map partitionLeaders; private List partitionIds; @@ -72,11 +89,11 @@ public class HgStoreNotice { @Override public String toString() { return "HgStoreNotice{" + - "nodeId=" + nodeId + - ", nodeStatus=" + nodeStatus + - ", message='" + message + '\'' + - ", partitionLeaders=" + partitionLeaders + - ", partitionIds=" + partitionIds + - '}'; + "nodeId=" + nodeId + + ", nodeStatus=" + nodeStatus + + ", message='" + message + '\'' + + ", partitionLeaders=" + partitionLeaders + + ", partitionIds=" + partitionIds + + '}'; } } diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreService.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreService.java new file mode 100644 index 000000000..02b2d2931 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreService.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + + +public class HgStoreService { + private static final HgStoreService instance = new HgStoreService(); + + private HgStoreService() { + } + + static HgStoreService of() { + return instance; + } + + +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreSessionProvider.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreSessionProvider.java new file mode 100644 index 000000000..2b85848db --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgStoreSessionProvider.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +import javax.annotation.concurrent.ThreadSafe; + +import org.apache.hugegraph.store.HgSessionProvider; +import org.apache.hugegraph.store.HgStoreSession; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/12 + */ +@ThreadSafe +public class HgStoreSessionProvider implements HgSessionProvider { + private final MultiNodeSessionFactory sessionFactory = MultiNodeSessionFactory.getInstance(); + + @Override + public HgStoreSession createSession(String graphName) { + return this.sessionFactory.createStoreSession(graphName); + } +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgTkvEntryImpl.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgTkvEntryImpl.java new file mode 100644 index 000000000..b7c11c58e --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgTkvEntryImpl.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +import java.util.Arrays; +import java.util.Objects; + +import org.apache.hugegraph.store.HgTkvEntry; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/14 + */ +class HgTkvEntryImpl implements HgTkvEntry { + private final String table; + private final byte[] key; + private final byte[] value; + + HgTkvEntryImpl(String table, byte[] key, byte[] value) { + this.table = table; + this.key = key; + this.value = value; + } + + @Override + public String table() { + return this.table; + } + + @Override + public byte[] key() { + return this.key; + } + + @Override + public byte[] value() { + return this.value; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + HgTkvEntryImpl that = (HgTkvEntryImpl) o; + return Objects.equals(table, that.table) && Arrays.equals(key, that.key) && + Arrays.equals(value, that.value); + } + + @Override + public int hashCode() { + int result = Objects.hash(table); + result = 31 * result + Arrays.hashCode(key); + result = 31 * result + Arrays.hashCode(value); + return result; + } + + @Override + public String toString() { + return "HgTkvEntryImpl{" + + "table='" + table + '\'' + + ", key=" + Arrays.toString(key) + + ", value=" + Arrays.toString(value) + + '}'; + } +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgTokvEntryImpl.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgTokvEntryImpl.java new file mode 100644 index 000000000..d58c8c275 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/HgTokvEntryImpl.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +import java.util.Arrays; +import java.util.Objects; + +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgTokvEntry; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/14 + */ +class HgTokvEntryImpl implements HgTokvEntry { + + private final String table; + private final HgOwnerKey ownerKey; + private final byte[] value; + + HgTokvEntryImpl(String table, HgOwnerKey ownerKey, byte[] value) { + this.table = table; + this.ownerKey = ownerKey; + this.value = value; + } + + @Override + public String table() { + return this.table; + } + + @Override + public HgOwnerKey ownerKey() { + return this.ownerKey; + } + + @Override + public byte[] value() { + return this.value; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + HgTokvEntryImpl that = (HgTokvEntryImpl) o; + return Objects.equals(table, that.table) && Objects.equals(ownerKey, that.ownerKey) && + Arrays.equals(value, that.value); + } + + @Override + public int hashCode() { + int result = Objects.hash(table, ownerKey); + result = 31 * result + Arrays.hashCode(value); + return result; + } + + @Override + public String toString() { + return "HgTokvEntryImpl{" + + "table='" + table + '\'' + + ", okv=" + ownerKey + + ", value=" + Arrays.toString(value) + + '}'; + } +} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/MultiNodeSessionFactory.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/MultiNodeSessionFactory.java similarity index 52% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/MultiNodeSessionFactory.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/MultiNodeSessionFactory.java index 6fa80d8c2..e3d2507fc 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/MultiNodeSessionFactory.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/MultiNodeSessionFactory.java @@ -1,9 +1,25 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.HgStoreSession; +package org.apache.hugegraph.store.client; import javax.annotation.concurrent.ThreadSafe; -import java.util.List; + +import org.apache.hugegraph.store.HgStoreSession; /** * @author lynn.bond@hotmail.com created on 2021/10/12 @@ -13,7 +29,7 @@ public final class MultiNodeSessionFactory { // TODO multi-instance ? private final static MultiNodeSessionFactory instance = new MultiNodeSessionFactory(); // TODO multi-instance ? - private HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); + private final HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); // TODO: to be a chain assigned to each graph //private HgStoreNodeDispatcher storeNodeDispatcher; diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/NodeTkv.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTkv.java similarity index 62% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/NodeTkv.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTkv.java index a081478fb..7db52cc06 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/NodeTkv.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTkv.java @@ -1,11 +1,28 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgStoreSession; +package org.apache.hugegraph.store.client; + +import java.util.Objects; import javax.annotation.concurrent.ThreadSafe; -import java.util.Arrays; -import java.util.Objects; + +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreSession; /** * @author lynn.bond@hotmail.com created on 2021/10/26 @@ -17,6 +34,7 @@ class NodeTkv { private final HgOwnerKey key; private final HgOwnerKey endKey; private HgStoreSession session; + NodeTkv(HgNodePartition nodePartition, String table, HgOwnerKey key) { this.nodePartition = nodePartition; this.table = table; @@ -24,6 +42,7 @@ class NodeTkv { this.endKey = key; this.key.setKeyCode(this.nodePartition.getKeyCode()); } + NodeTkv(HgNodePartition nodePartition, String table, HgOwnerKey key, int keyCode) { this.nodePartition = nodePartition; this.table = table; @@ -32,6 +51,7 @@ class NodeTkv { this.key.setKeyCode(keyCode); } + NodeTkv(HgNodePartition nodePartition, String table, HgOwnerKey startKey, HgOwnerKey endKey) { this.nodePartition = nodePartition; @@ -58,7 +78,7 @@ class NodeTkv { return endKey; } - public NodeTkv setKeyCode(int code){ + public NodeTkv setKeyCode(int code) { this.key.setKeyCode(code); return this; } @@ -68,9 +88,10 @@ class NodeTkv { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; NodeTkv nptKv = (NodeTkv) o; - return Objects.equals(nodePartition, nptKv.nodePartition) && Objects.equals(table, nptKv.table) - && Objects.equals(key, nptKv.key) - && Objects.equals(endKey, nptKv.endKey); + return Objects.equals(nodePartition, nptKv.nodePartition) && + Objects.equals(table, nptKv.table) + && Objects.equals(key, nptKv.key) + && Objects.equals(endKey, nptKv.endKey); } @Override @@ -82,11 +103,11 @@ class NodeTkv { @Override public String toString() { return "NptKv{" + - "nodePartition=" + nodePartition + - ", table='" + table + '\'' + - ", key=" + key + - ", endKey=" + endKey + - '}'; + "nodePartition=" + nodePartition + + ", table='" + table + '\'' + + ", key=" + key + + ", endKey=" + endKey + + '}'; } public HgStoreSession getSession() { diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/NodeTxExecutor.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxExecutor.java similarity index 83% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/NodeTxExecutor.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxExecutor.java index 888d558a0..ca1d616ba 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/NodeTxExecutor.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxExecutor.java @@ -1,8 +1,25 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_LIST; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.NODE_MAX_RETRYING_TIMES; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.TX_SESSIONS_MAP_CAPACITY; +package org.apache.hugegraph.store.client; + +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_LIST; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.NODE_MAX_RETRYING_TIMES; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.TX_SESSIONS_MAP_CAPACITY; import java.util.Collection; import java.util.HashMap; @@ -21,13 +38,13 @@ import java.util.stream.Stream; import javax.annotation.concurrent.NotThreadSafe; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgStoreSession; -import com.baidu.hugegraph.store.client.type.HgStoreClientException; -import com.baidu.hugegraph.store.client.util.HgAssert; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import com.baidu.hugegraph.store.term.HgPair; -import com.baidu.hugegraph.store.term.HgTriple; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.client.type.HgStoreClientException; +import org.apache.hugegraph.store.client.util.HgAssert; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.term.HgPair; +import org.apache.hugegraph.store.term.HgTriple; import lombok.extern.slf4j.Slf4j; @@ -38,7 +55,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @NotThreadSafe final class NodeTxExecutor { - private String graphName; + private final String graphName; private Map sessions = new HashMap<>(TX_SESSIONS_MAP_CAPACITY, 1); private boolean isTx; NodeTxSessionProxy proxy; @@ -46,16 +63,17 @@ final class NodeTxExecutor { Function>> entries = new LinkedList<>(); Collector>> collector = Collectors.groupingBy( nkv -> nkv.getNodeId(), Collectors.mapping(NodeTkv::getKey, Collectors.toList())); - private static String maxTryMsg = "the number of retries reached the upper limit : " + NODE_MAX_RETRYING_TIMES + - ",caused by:"; - private static String msg = "Not all tx-data delivered to real-node-session successfully."; + private static final String maxTryMsg = + "the number of retries reached the upper limit : " + NODE_MAX_RETRYING_TIMES + + ",caused by:"; + private static final String msg = "Not all tx-data delivered to real-node-session successfully."; + static { System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", String.valueOf(Runtime.getRuntime().availableProcessors() * 2)); } - static NodeTxExecutor graphOf(String graphName, NodeTxSessionProxy proxy) { return new NodeTxExecutor(graphName, proxy); } @@ -103,7 +121,8 @@ final class NodeTxExecutor { return true; } AtomicBoolean allSuccess = new AtomicBoolean(true); - for (HgPair, Function> e : this.entries) { + for (HgPair, Function> e : + this.entries) { doAction(e.getKey(), e.getValue()); } if (!allSuccess.get()) { @@ -163,10 +182,12 @@ final class NodeTxExecutor { // } if ( nodeParams.getZ() instanceof Integer ){ // return this.proxy.doPartition(nodeParams.getX(), (Integer) nodeParams.getZ()) // .stream() - // .map(e -> new NodeTkv(e, nodeParams.getX(), nodeParams.getY(), nodeParams.getY() + // .map(e -> new NodeTkv(e, nodeParams.getX(), nodeParams.getY(), + // nodeParams.getY() // .getKeyCode())) // .map( - // e -> new HgPair<>(this.proxy.getStoreNode(e.getNodeId()), e) + // e -> new HgPair<>(this.proxy.getStoreNode(e.getNodeId + // ()), e) // ); // }else { // HgAssert.isTrue(nodeParams.getZ() instanceof byte[], @@ -186,18 +207,25 @@ final class NodeTxExecutor { // (HgOwnerKey) nodeParams.getZ()); // } // if (nodeParams.getZ() instanceof Integer) { - // Collection nodePartitions = this.proxy.doPartition(nodeParams.getX(), - // (Integer) nodeParams.getZ()); - // ArrayList> hgPairs = new ArrayList<>(nodePartitions.size()); + // Collection nodePartitions = this.proxy.doPartition(nodeParams + // .getX(), + // (Integer) + // nodeParams + // .getZ()); + // ArrayList> hgPairs = new ArrayList<> + // (nodePartitions.size()); // for (HgNodePartition nodePartition : nodePartitions) { - // NodeTkv nodeTkv = new NodeTkv(nodePartition, nodeParams.getX(), nodeParams.getY(), + // NodeTkv nodeTkv = new NodeTkv(nodePartition, nodeParams.getX(), nodeParams + // .getY(), // nodeParams.getY().getKeyCode()); - // hgPairs.add(new HgPair<>(this.proxy.getStoreNode(nodeTkv.getNodeId()), nodeTkv)); + // hgPairs.add(new HgPair<>(this.proxy.getStoreNode(nodeTkv.getNodeId()), + // nodeTkv)); // // } // return hgPairs; // } else { - // HgAssert.isTrue(nodeParams.getZ() instanceof byte[], "Illegal parameter to get node id"); + // HgAssert.isTrue(nodeParams.getZ() instanceof byte[], "Illegal parameter to get + // node id"); // throw new RuntimeException("not implemented"); // } // } @@ -206,7 +234,8 @@ final class NodeTxExecutor { private boolean doAction(HgTriple nodeParams, Function action) { if (nodeParams.getZ() == null) { - return this.proxy.doAction(nodeParams.getX(), nodeParams.getY(), nodeParams.getY(), action); + return this.proxy.doAction(nodeParams.getX(), nodeParams.getY(), nodeParams.getY(), + action); } else { if (nodeParams.getZ() instanceof HgOwnerKey) { boolean result = this.proxy.doAction(nodeParams.getX(), nodeParams.getY(), @@ -214,9 +243,11 @@ final class NodeTxExecutor { return result; } if (nodeParams.getZ() instanceof Integer) { - return this.proxy.doAction(nodeParams.getX(), nodeParams.getY(), (Integer) nodeParams.getZ(), action); + return this.proxy.doAction(nodeParams.getX(), nodeParams.getY(), + (Integer) nodeParams.getZ(), action); } else { - HgAssert.isTrue(nodeParams.getZ() instanceof byte[], "Illegal parameter to get node id"); + HgAssert.isTrue(nodeParams.getZ() instanceof byte[], + "Illegal parameter to get node id"); throw new RuntimeException("not implemented"); } } @@ -277,7 +308,9 @@ final class NodeTxExecutor { .map( e -> new SessionData> ( - openNodeSession(nodeFunction.apply(e.getKey())), e.getValue() + openNodeSession( + nodeFunction.apply(e.getKey())), + e.getValue() ) ) .parallel() @@ -378,11 +411,7 @@ final class NodeTxExecutor { return false; } - if (EMPTY_LIST.equals(obj)) { - return false; - } - - return true; + return !EMPTY_LIST.equals(obj); } class SessionData { diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/NodeTxSessionProxy.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java similarity index 88% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/NodeTxSessionProxy.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java index 78b3e0c1b..f1edca106 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/NodeTxSessionProxy.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/NodeTxSessionProxy.java @@ -1,12 +1,29 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; -import static com.baidu.hugegraph.store.client.util.HgAssert.isArgumentNotNull; -import static com.baidu.hugegraph.store.client.util.HgAssert.isArgumentValid; -import static com.baidu.hugegraph.store.client.util.HgAssert.isFalse; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_STRING; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.err; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toStr; import static java.util.stream.Collectors.groupingBy; +import static org.apache.hugegraph.store.client.util.HgAssert.isArgumentNotNull; +import static org.apache.hugegraph.store.client.util.HgAssert.isArgumentValid; +import static org.apache.hugegraph.store.client.util.HgAssert.isFalse; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_STRING; +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.err; +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toStr; import java.util.ArrayList; import java.util.Collection; @@ -22,20 +39,21 @@ import java.util.stream.Stream; import javax.annotation.concurrent.NotThreadSafe; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgKvOrderedIterator; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgScanQuery; -import com.baidu.hugegraph.store.HgStoreSession; -import com.baidu.hugegraph.store.client.grpc.KvBatchScanner; -import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator; -import com.baidu.hugegraph.store.client.util.HgAssert; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import com.baidu.hugegraph.store.client.util.HgStoreClientUtil; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvOrderedIterator; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.client.grpc.KvBatchScanner; +import org.apache.hugegraph.store.client.grpc.KvCloseableIterator; +import org.apache.hugegraph.store.client.util.HgAssert; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.HgStoreClientUtil; +import org.apache.hugegraph.store.term.HgPair; +import org.apache.hugegraph.store.term.HgTriple; + import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq.Builder; -import com.baidu.hugegraph.store.term.HgPair; -import com.baidu.hugegraph.store.term.HgTriple; import lombok.extern.slf4j.Slf4j; @@ -103,7 +121,7 @@ class NodeTxSessionProxy implements HgStoreSession { isArgumentNotNull(ownerKey, "ownerKey"); return this.txExecutor.prepareTx( - new HgTriple(table, ownerKey, (Integer) partitionId), + new HgTriple(table, ownerKey, partitionId), e -> e.getSession().put(table, e.getKey(), value) ); } @@ -369,7 +387,8 @@ class NodeTxSessionProxy implements HgStoreSession { } @Override - public HgKvIterator scanIterator(String table, HgOwnerKey keyPrefix, long limit, byte[] query) { + public HgKvIterator scanIterator(String table, HgOwnerKey keyPrefix, long limit, + byte[] query) { HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table"); HgAssert.isFalse(keyPrefix == null, "The argument is invalid: keyPrefix"); @@ -385,12 +404,14 @@ class NodeTxSessionProxy implements HgStoreSession { } @Override - public HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey) { + public HgKvIterator scanIterator(String table, HgOwnerKey startKey, + HgOwnerKey endKey) { return this.scanIterator(table, startKey, endKey, 0, null); } @Override - public HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit) { + public HgKvIterator scanIterator(String table, HgOwnerKey startKey, + HgOwnerKey endKey, long limit) { HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table"); HgAssert.isFalse(startKey == null, "The argument is invalid: startKey"); HgAssert.isFalse(endKey == null, "The argument is invalid: endKey"); @@ -418,7 +439,8 @@ class NodeTxSessionProxy implements HgStoreSession { .parallelStream() .map( e -> this.getStoreNode(e.getNodeId()).openSession(this.graphName) - .scanIterator(e.getTable(), e.getKey(), e.getEndKey(), limit, query) + .scanIterator(e.getTable(), e.getKey(), e.getEndKey(), limit, + query) ) .collect(Collectors.toList()) , limit); @@ -437,7 +459,8 @@ class NodeTxSessionProxy implements HgStoreSession { .parallelStream() .map( e -> this.getStoreNode(e.getNodeId()).openSession(this.graphName) - .scanIterator(e.getTable(), e.getKey(), e.getEndKey(), limit, scanType, query) + .scanIterator(e.getTable(), e.getKey(), e.getEndKey(), limit, + scanType, query) ) .collect(Collectors.toList()) , limit); @@ -445,7 +468,8 @@ class NodeTxSessionProxy implements HgStoreSession { } @Override - public HgKvIterator scanIterator(String table, int codeFrom, int codeTo, int scanType, byte[] query) { + public HgKvIterator scanIterator(String table, int codeFrom, int codeTo, + int scanType, byte[] query) { if (log.isDebugEnabled()) { log.debug("graph: {}, table: {}, codeFrom: {}, codeTo: {}, scanType: {}, query: {}" , graphName, table, codeFrom, codeTo, scanType, HgStoreClientUtil.toStr(query)); @@ -540,7 +564,8 @@ class NodeTxSessionProxy implements HgStoreSession { return notifierWrap; } - private Function>> toTkvMapFunc(HgScanQuery.ScanMethod scanMethod) { + private Function>> toTkvMapFunc( + HgScanQuery.ScanMethod scanMethod) { switch (scanMethod) { case RANGE: return scanQuery -> { @@ -549,7 +574,8 @@ class NodeTxSessionProxy implements HgStoreSession { int size = starts.size(); return IntStream.range(0, size) .boxed() - .map(i -> this.toNodeTkvList(scanQuery.getTable(), starts.get(i), ends.get(i))) + .map(i -> this.toNodeTkvList(scanQuery.getTable(), + starts.get(i), ends.get(i))) .flatMap(List::stream) .collect(groupingBy(NodeTkv::getNodeId)); }; @@ -557,7 +583,8 @@ class NodeTxSessionProxy implements HgStoreSession { return scanQuery -> scanQuery.getPrefixList() .stream() - .map(keyPrefix -> this.toNodeTkvList(scanQuery.getTable(), keyPrefix)) + .map(keyPrefix -> this.toNodeTkvList(scanQuery.getTable(), + keyPrefix)) .flatMap(List::stream) .collect(groupingBy(NodeTkv::getNodeId)); @@ -599,7 +626,8 @@ class NodeTxSessionProxy implements HgStoreSession { /*-- common --*/ private HgKvIterator toHgKvIteratorProxy(List iteratorList, long limit) { boolean isAllOrderedLimiter = iteratorList.stream() - .allMatch(e -> HgKvOrderedIterator.class.isInstance(e)); + .allMatch( + e -> e instanceof HgKvOrderedIterator); HgKvIterator iterator; if (isAllOrderedLimiter) { @@ -623,8 +651,10 @@ class NodeTxSessionProxy implements HgStoreSession { return res; } - public boolean doAction(String table, HgOwnerKey startKey, HgOwnerKey endKey, Function action) { - Collection partitions = doPartition(table, startKey.getOwner(), endKey.getOwner()); + public boolean doAction(String table, HgOwnerKey startKey, HgOwnerKey endKey, + Function action) { + Collection partitions = + doPartition(table, startKey.getOwner(), endKey.getOwner()); for (HgNodePartition partition : partitions) { HgStoreNode storeNode = this.getStoreNode(partition.getNodeId()); HgStoreSession session = this.txExecutor.openNodeSession(storeNode); @@ -637,7 +667,8 @@ class NodeTxSessionProxy implements HgStoreSession { return true; } - public boolean doAction(String table, HgOwnerKey startKey, Integer code, Function action) { + public boolean doAction(String table, HgOwnerKey startKey, Integer code, + Function action) { Collection partitions = this.doPartition(table, code); for (HgNodePartition partition : partitions) { HgStoreNode storeNode = this.getStoreNode(partition.getNodeId()); @@ -679,7 +710,8 @@ class NodeTxSessionProxy implements HgStoreSession { } private List toNodeTkvList(String table, HgOwnerKey ownerKey) { - Collection partitions = doPartition(table, ownerKey.getOwner(), ownerKey.getOwner()); + Collection partitions = + doPartition(table, ownerKey.getOwner(), ownerKey.getOwner()); ArrayList nodeTkvs = new ArrayList<>(partitions.size()); for (HgNodePartition partition : partitions) { nodeTkvs.add(new NodeTkv(partition, table, ownerKey, ownerKey)); @@ -689,7 +721,8 @@ class NodeTxSessionProxy implements HgStoreSession { } private List toNodeTkvList(String table, HgOwnerKey startKey, HgOwnerKey endKey) { - Collection partitions = doPartition(table, startKey.getOwner(), endKey.getOwner()); + Collection partitions = + doPartition(table, startKey.getOwner(), endKey.getOwner()); ArrayList nodeTkvs = new ArrayList<>(partitions.size()); for (HgNodePartition partition : partitions) { nodeTkvs.add(new NodeTkv(partition, table, startKey, endKey)); @@ -703,17 +736,20 @@ class NodeTxSessionProxy implements HgStoreSession { ArrayList nodeTkvs = new ArrayList<>(partitions.size()); for (HgNodePartition partition : partitions) { nodeTkvs.add( - new NodeTkv(partition, table, HgOwnerKey.codeOf(startCode), HgOwnerKey.codeOf(endCode))); + new NodeTkv(partition, table, HgOwnerKey.codeOf(startCode), + HgOwnerKey.codeOf(endCode))); } return nodeTkvs; } + /** * @return not null */ private Collection doPartition(String table, byte[] startKey, byte[] endKey) { HgNodePartitionerBuilder partitionerBuilder = HgNodePartitionerBuilder.resetAndGet(); - int status = this.nodePartitioner.partition(partitionerBuilder, this.graphName, startKey, endKey); + int status = this.nodePartitioner.partition(partitionerBuilder, this.graphName, startKey, + endKey); if (status != 0) { throw err("The node-partitioner is not work."); @@ -733,7 +769,8 @@ class NodeTxSessionProxy implements HgStoreSession { */ private Collection doPartition(String table, int startCode, int endCode) { HgNodePartitionerBuilder partitionerBuilder = HgNodePartitionerBuilder.resetAndGet(); - int status = this.nodePartitioner.partition(partitionerBuilder, this.graphName, startCode, endCode); + int status = this.nodePartitioner.partition(partitionerBuilder, this.graphName, startCode, + endCode); if (status != 0) { throw err("The node-partitioner is not work."); @@ -750,7 +787,8 @@ class NodeTxSessionProxy implements HgStoreSession { Collection doPartition(String table, int partitionId) { HgNodePartitionerBuilder partitionerBuilder = HgNodePartitionerBuilder.resetAndGet(); - int status = this.nodePartitioner.partition(partitionerBuilder, this.graphName, partitionId); + int status = + this.nodePartitioner.partition(partitionerBuilder, this.graphName, partitionId); if (status != 0) { throw err("The node-partitioner is not work."); @@ -764,12 +802,13 @@ class NodeTxSessionProxy implements HgStoreSession { return partitions; } + private Stream> getNodeStream(String table) { return this.toNodeTkvList(table) .stream() .map( e -> new HgPair<>(this.getStoreNode(e.getNodeId()), e) - ); + ); } Stream> getNodeStream(String table, @@ -781,7 +820,8 @@ class NodeTxSessionProxy implements HgStoreSession { ); } - Stream> getNodeStream(String table, HgOwnerKey startKey, HgOwnerKey endKey) { + Stream> getNodeStream(String table, HgOwnerKey startKey, + HgOwnerKey endKey) { return this.toNodeTkvList(table, startKey, endKey) .stream() .map( @@ -800,7 +840,8 @@ class NodeTxSessionProxy implements HgStoreSession { return nodeTkv2Node(nodeTkvList); } - List> getNode(String table, HgOwnerKey startKey, HgOwnerKey endKey) { + List> getNode(String table, HgOwnerKey startKey, + HgOwnerKey endKey) { List nodeTkvList = this.toNodeTkvList(table, startKey, endKey); return nodeTkv2Node(nodeTkvList); @@ -814,7 +855,8 @@ class NodeTxSessionProxy implements HgStoreSession { // List> getNode(String table, Integer endKey) { // .stream() - // .map(e -> new NodeTkv(e, nodeParams.getX(), nodeParams.getY(), nodeParams.getY().getKeyCode())) + // .map(e -> new NodeTkv(e, nodeParams.getX(), nodeParams.getY(), nodeParams.getY + // ().getKeyCode())) // .map( // e -> new HgPair<>(this.proxy.getStoreNode(e.getNodeId()), e) // ); diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/SequencedIterator.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/SequencedIterator.java similarity index 72% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/SequencedIterator.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/SequencedIterator.java index 9c3e0a4cc..4e8365e06 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/SequencedIterator.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/SequencedIterator.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; import java.util.Collections; import java.util.LinkedList; @@ -6,9 +23,9 @@ import java.util.List; import java.util.NoSuchElementException; import java.util.Queue; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgKvOrderedIterator; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvOrderedIterator; import lombok.extern.slf4j.Slf4j; @@ -22,9 +39,9 @@ import lombok.extern.slf4j.Slf4j; public class SequencedIterator implements HgKvIterator { private static final byte[] EMPTY_BYTES = new byte[0]; private HgKvOrderedIterator iterator; - private Queue queue; + private final Queue queue; private HgKvEntry entry; - private long limit; + private final long limit; private int count; private byte[] position = EMPTY_BYTES; private byte[] position4Seeking = EMPTY_BYTES; @@ -44,9 +61,6 @@ public class SequencedIterator implements HgKvIterator { break; } } - if (buf == null) { - return null; - } return buf; } @@ -98,11 +112,7 @@ public class SequencedIterator implements HgKvIterator { this.iterator.close(); this.iterator = this.getIterator(); } - if (this.iterator == null) { - return false; - } else { - return true; - } + return this.iterator != null; } @Override diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/ShiftWorkIteratorProxy.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/ShiftWorkIteratorProxy.java similarity index 71% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/ShiftWorkIteratorProxy.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/ShiftWorkIteratorProxy.java index 795f13e0b..47c8abbea 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/ShiftWorkIteratorProxy.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/ShiftWorkIteratorProxy.java @@ -1,13 +1,30 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; import java.util.LinkedList; import java.util.List; import java.util.NoSuchElementException; import java.util.Queue; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgKvPagingIterator; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvPagingIterator; import lombok.extern.slf4j.Slf4j; @@ -21,7 +38,7 @@ public class ShiftWorkIteratorProxy implements HgKvIterator { private HgKvPagingIterator iterator; private Queue queue = new LinkedList<>(); private HgKvEntry entry; - private int limit; + private final int limit; private int count; private int shiftCount; @@ -112,11 +129,7 @@ public class ShiftWorkIteratorProxy implements HgKvIterator { || !this.iterator.hasNext()) { this.iterator = this.getIterator(); } - if (this.iterator == null) { - return false; - } else { - return true; - } + return this.iterator != null; } @Override diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/TopWorkIteratorProxy.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/TopWorkIteratorProxy.java similarity index 60% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/TopWorkIteratorProxy.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/TopWorkIteratorProxy.java index 80b0d8a28..a101f5b6e 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/TopWorkIteratorProxy.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/TopWorkIteratorProxy.java @@ -1,10 +1,29 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; +package org.apache.hugegraph.store.client; -import java.io.IOException; -import java.util.*; +import java.util.LinkedList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Queue; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; /** * @author lynn.bond@hotmail.com created on 2021/10/21 @@ -13,9 +32,9 @@ import java.util.*; class TopWorkIteratorProxy implements HgKvIterator { private static final byte[] EMPTY_BYTES = new byte[0]; private HgKvIterator iterator; - private Queue queue; + private final Queue queue; private HgKvEntry entry; - private long limit; + private final long limit; private int count; TopWorkIteratorProxy(List iterators, long limit) { @@ -42,6 +61,7 @@ class TopWorkIteratorProxy implements HgKvIterator { return buf; } + private void closeIterators() { if (this.queue.isEmpty()) return; @@ -52,6 +72,7 @@ class TopWorkIteratorProxy implements HgKvIterator { } } + private void setIterator() { this.iterator = null; } @@ -79,7 +100,7 @@ class TopWorkIteratorProxy implements HgKvIterator { @Override public void seek(byte[] position) { - if(this.iterator!=null)this.iterator.seek(position); + if (this.iterator != null) this.iterator.seek(position); } @Override @@ -90,11 +111,7 @@ class TopWorkIteratorProxy implements HgKvIterator { if (this.iterator == null) { this.iterator = this.getIterator(); } - if (this.iterator == null) { - return false; - } else { - return true; - } + return this.iterator != null; } @@ -113,8 +130,8 @@ class TopWorkIteratorProxy implements HgKvIterator { } @Override - public void close() { - if(this.iterator!=null)this.iterator.close(); + public void close() { + if (this.iterator != null) this.iterator.close(); this.closeIterators(); } } diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/AbstractGrpcClient.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/AbstractGrpcClient.java similarity index 66% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/AbstractGrpcClient.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/AbstractGrpcClient.java index fd8090e9d..22c67162a 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/AbstractGrpcClient.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/AbstractGrpcClient.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.client.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -6,8 +23,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.IntStream; -import com.baidu.hugegraph.store.client.util.HgStoreClientConfig; -import com.baidu.hugegraph.store.term.HgPair; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.term.HgPair; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; @@ -20,14 +37,16 @@ import io.grpc.stub.AbstractBlockingStub; **/ public abstract class AbstractGrpcClient { - private static Map channels = new ConcurrentHashMap<>(); - private static int n = 5; - private static int concurrency = 1 << n; - private static AtomicLong counter = new AtomicLong(0); - private static long limit = Long.MAX_VALUE >> 1; - private Map[]> blockingStubs = new ConcurrentHashMap<>(); - private Map[]> asyncStubs = new ConcurrentHashMap<>(); - private static HgStoreClientConfig config = HgStoreClientConfig.of(); + private static final Map channels = new ConcurrentHashMap<>(); + private static final int n = 5; + private static final int concurrency = 1 << n; + private static final AtomicLong counter = new AtomicLong(0); + private static final long limit = Long.MAX_VALUE >> 1; + private final Map[]> blockingStubs = + new ConcurrentHashMap<>(); + private final Map[]> asyncStubs = + new ConcurrentHashMap<>(); + private static final HgStoreClientConfig config = HgStoreClientConfig.of(); public AbstractGrpcClient() { @@ -39,7 +58,8 @@ public abstract class AbstractGrpcClient { synchronized (channels) { if ((tc = channels.get(target)) == null) { ManagedChannel[] value = new ManagedChannel[concurrency]; - IntStream.range(0, concurrency).parallel().forEach(i -> value[i] = getManagedChannel(target)); + IntStream.range(0, concurrency).parallel() + .forEach(i -> value[i] = getManagedChannel(target)); channels.put(target, tc = value); } } @@ -72,16 +92,18 @@ public abstract class AbstractGrpcClient { }); blockingStubs.put(target, value); AbstractBlockingStub stub = value[index].getValue(); - return (AbstractBlockingStub) stub.withDeadlineAfter(config.getGrpcTimeoutSeconds(), - TimeUnit.SECONDS); + return (AbstractBlockingStub) stub.withDeadlineAfter( + config.getGrpcTimeoutSeconds(), + TimeUnit.SECONDS); } } } - return (AbstractBlockingStub) pairs[index].getValue().withDeadlineAfter(config.getGrpcTimeoutSeconds(), - TimeUnit.SECONDS); + return (AbstractBlockingStub) pairs[index].getValue() + .withDeadlineAfter(config.getGrpcTimeoutSeconds(), + TimeUnit.SECONDS); } - public AbstractAsyncStub getAsyncStub(ManagedChannel channel){ + public AbstractAsyncStub getAsyncStub(ManagedChannel channel) { return null; } @@ -117,7 +139,6 @@ public abstract class AbstractGrpcClient { } - private ManagedChannel getManagedChannel(String target) { return ManagedChannelBuilder.forTarget(target).usePlaintext().build(); } diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcKvEntryImpl.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcKvEntryImpl.java new file mode 100644 index 000000000..614336eab --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcKvEntryImpl.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.util.Arrays; + +import org.apache.hugegraph.store.HgKvEntry; + +/** + * @author lynn.bond@hotmail.com + */ +class GrpcKvEntryImpl implements HgKvEntry { + private final byte[] key; + private final byte[] value; + private final int code; + + GrpcKvEntryImpl(byte[] k, byte[] v, int code) { + this.key = k; + this.value = v; + this.code = code; + } + + @Override + public int code() { + return code; + } + + @Override + public byte[] key() { + return key; + } + + @Override + public byte[] value() { + return value; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GrpcKvEntryImpl hgKvEntry = (GrpcKvEntryImpl) o; + return Arrays.equals(key, hgKvEntry.key) && Arrays.equals(value, hgKvEntry.value); + } + + @Override + public int hashCode() { + int result = Arrays.hashCode(key); + result = 31 * result + Arrays.hashCode(value); + return result; + } + + @Override + public String toString() { + return "HgKvEntryImpl{" + + "key=" + Arrays.toString(key) + + ", value=" + Arrays.toString(value) + + ", code=" + code + + '}'; + } +} \ No newline at end of file diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcKvIteratorImpl.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcKvIteratorImpl.java similarity index 59% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcKvIteratorImpl.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcKvIteratorImpl.java index 5ec219868..e188dc28b 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcKvIteratorImpl.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcKvIteratorImpl.java @@ -1,17 +1,34 @@ -package com.baidu.hugegraph.store.client.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toIntBytes; +package org.apache.hugegraph.store.client.grpc; import java.util.List; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgKvOrderedIterator; -import com.baidu.hugegraph.store.HgKvPagingIterator; -import com.baidu.hugegraph.store.HgPageSize; -import com.baidu.hugegraph.store.HgSeekAble; -import com.baidu.hugegraph.store.client.HgStoreNodeSession; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvOrderedIterator; +import org.apache.hugegraph.store.HgKvPagingIterator; +import org.apache.hugegraph.store.HgPageSize; +import org.apache.hugegraph.store.HgSeekAble; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.HgStoreClientUtil; + import com.baidu.hugegraph.store.grpc.common.Kv; import lombok.extern.slf4j.Slf4j; @@ -23,20 +40,22 @@ import lombok.extern.slf4j.Slf4j; @Slf4j class GrpcKvIteratorImpl implements HgKvPagingIterator, HgKvOrderedIterator { - private byte[] emptyBytes = HgStoreClientConst.EMPTY_BYTES; - private KvCloseableIterator iterator; + private final byte[] emptyBytes = HgStoreClientConst.EMPTY_BYTES; + private final KvCloseableIterator iterator; private HgKvEntry element; - private HgPageSize pageLimiter; - private HgStoreNodeSession session; + private final HgPageSize pageLimiter; + private final HgStoreNodeSession session; - public static HgKvIterator of(HgStoreNodeSession nodeSession, KvCloseableIterator iterator) { - if (HgPageSize.class.isInstance(iterator)) { + public static HgKvIterator of(HgStoreNodeSession nodeSession, + KvCloseableIterator iterator) { + if (iterator instanceof HgPageSize) { return of(nodeSession, iterator, (HgPageSize) iterator); } return new GrpcKvIteratorImpl(nodeSession, iterator, () -> 1); } - public static HgKvIterator of(HgStoreNodeSession nodeSession, KvCloseableIterator iterator, + public static HgKvIterator of(HgStoreNodeSession nodeSession, + KvCloseableIterator iterator, HgPageSize pageLimiter) { return new GrpcKvIteratorImpl(nodeSession, iterator, pageLimiter); } @@ -46,7 +65,8 @@ class GrpcKvIteratorImpl implements HgKvPagingIterator, HgKvOrderedIt return new GrpcKvIteratorImpl(nodeSession, new KvListIterator(kvList), () -> pageSize); } - private GrpcKvIteratorImpl(HgStoreNodeSession session, KvCloseableIterator iterator, HgPageSize pageLimiter) { + private GrpcKvIteratorImpl(HgStoreNodeSession session, KvCloseableIterator iterator, + HgPageSize pageLimiter) { this.iterator = iterator; this.pageLimiter = pageLimiter; this.session = session; @@ -65,7 +85,8 @@ class GrpcKvIteratorImpl implements HgKvPagingIterator, HgKvOrderedIt @Override public HgKvEntry next() { Kv kv = this.iterator.next(); - this.element = new GrpcKvEntryImpl(kv.getKey().toByteArray(), kv.getValue().toByteArray(), kv.getCode()); + this.element = new GrpcKvEntryImpl(kv.getKey().toByteArray(), kv.getValue().toByteArray(), + kv.getCode()); return this.element; } @@ -94,11 +115,11 @@ class GrpcKvIteratorImpl implements HgKvPagingIterator, HgKvOrderedIt if (key == null) { return emptyBytes; } - if (!HgSeekAble.class.isInstance(this.iterator)) { + if (!(this.iterator instanceof HgSeekAble)) { return emptyBytes; } byte[] upstream = ((HgSeekAble) this.iterator).position(); - byte[] code = toIntBytes(this.element.code()); + byte[] code = HgStoreClientUtil.toIntBytes(this.element.code()); byte[] result = new byte[upstream.length + Integer.BYTES + key.length]; System.arraycopy(upstream, 0, result, 0, upstream.length); System.arraycopy(code, 0, result, upstream.length, Integer.BYTES); @@ -108,7 +129,7 @@ class GrpcKvIteratorImpl implements HgKvPagingIterator, HgKvOrderedIt @Override public void seek(byte[] position) { - if (HgSeekAble.class.isInstance(this.iterator)) { + if (this.iterator instanceof HgSeekAble) { ((HgSeekAble) this.iterator).seek(position); } } diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcNodeHealthyClient.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcNodeHealthyClient.java similarity index 69% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcNodeHealthyClient.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcNodeHealthyClient.java index a2e43f054..708d32085 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcNodeHealthyClient.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcNodeHealthyClient.java @@ -1,15 +1,34 @@ -package com.baidu.hugegraph.store.client.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import javax.annotation.concurrent.ThreadSafe; import com.baidu.hugegraph.store.grpc.HealthyGrpc; import com.baidu.hugegraph.store.grpc.HealthyOuterClass; import com.google.protobuf.Empty; + import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; -import javax.annotation.concurrent.ThreadSafe; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - /** * */ @@ -17,7 +36,8 @@ import java.util.concurrent.ConcurrentHashMap; public final class GrpcNodeHealthyClient { private final static Map CHANNEL_MAP = new ConcurrentHashMap<>(); - private final static Map STUB_MAP = new ConcurrentHashMap<>(); + private final static Map STUB_MAP = + new ConcurrentHashMap<>(); // TODO: Forbid constructing out of the package. public GrpcNodeHealthyClient() { @@ -66,11 +86,6 @@ public final class GrpcNodeHealthyClient { String res = response.getMessage(); System.out.printf("gRPC response message:%s%n", res); - if ("ok".equals(res)) { - return true; - } else { - - return false; - } + return "ok".equals(res); } } diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreNodeBuilder.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreNodeBuilder.java new file mode 100644 index 000000000..9f2707952 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreNodeBuilder.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hugegraph.store.client.HgPrivate; +import org.apache.hugegraph.store.client.HgStoreNode; +import org.apache.hugegraph.store.client.HgStoreNodeBuilder; +import org.apache.hugegraph.store.client.HgStoreNodeManager; +import org.apache.hugegraph.store.client.util.HgAssert; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/12 + */ +public class GrpcStoreNodeBuilder implements HgStoreNodeBuilder { + + private static final GrpcStoreSessionClient sessionClient = new GrpcStoreSessionClient(); + private static final GrpcStoreStreamClient streamClient = new GrpcStoreStreamClient(); + private static final AtomicLong ids = new AtomicLong(0); + + private Long nodeId; + private String address; + private final HgStoreNodeManager nodeManager; + + public GrpcStoreNodeBuilder(HgStoreNodeManager nodeManager, HgPrivate hgPrivate) { + HgAssert.isArgumentNotNull(hgPrivate, "hgPrivate"); + HgAssert.isArgumentNotNull(nodeManager, "nodeManager"); + this.nodeManager = nodeManager; + } + + @Override + public GrpcStoreNodeBuilder setAddress(String address) { + HgAssert.isFalse(HgAssert.isInvalid(address), "The argument is invalid: address."); + this.address = address; + return this; + } + + @Override + public GrpcStoreNodeBuilder setNodeId(Long nodeId) { + HgAssert.isFalse(nodeId == null, "The argument is invalid: nodeId."); + this.nodeId = nodeId; + return this; + } + + @Override + public HgStoreNode build() { + // TODO: delete + if (this.nodeId == null) { + this.nodeId = ids.addAndGet(-1L); + } + + HgAssert.isFalse(this.nodeId == null, "nodeId can't to be null"); + HgAssert.isFalse(this.address == null, "address can't to be null"); + + GrpcStoreNodeImpl node = + new GrpcStoreNodeImpl(this.nodeManager, sessionClient, streamClient); + node.setNodeId(this.nodeId); + node.setAddress(this.address); + + return node; + } +} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreNodeImpl.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreNodeImpl.java similarity index 57% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreNodeImpl.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreNodeImpl.java index e1d7854aa..4e6858ec2 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreNodeImpl.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreNodeImpl.java @@ -1,18 +1,35 @@ -package com.baidu.hugegraph.store.client.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; import java.util.Objects; -import com.baidu.hugegraph.store.HgStoreSession; -import com.baidu.hugegraph.store.client.HgStoreNode; -import com.baidu.hugegraph.store.client.HgStoreNodeManager; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.client.HgStoreNode; +import org.apache.hugegraph.store.client.HgStoreNodeManager; /** * @author lynn.bond@hotmail.com created on 2021/10/11 */ class GrpcStoreNodeImpl implements HgStoreNode { - private GrpcStoreSessionClient sessionClient; - private GrpcStoreStreamClient streamClient; - private HgStoreNodeManager nodeManager; + private final GrpcStoreSessionClient sessionClient; + private final GrpcStoreStreamClient streamClient; + private final HgStoreNodeManager nodeManager; private String address; private Long nodeId; @@ -46,9 +63,11 @@ class GrpcStoreNodeImpl implements HgStoreNode { @Override public HgStoreSession openSession(String graphName) { // HgAssert.isFalse(HgAssert.isInvalid(graphName), "the argument: graphName is invalid."); - // return new GrpcStoreNodeSessionImpl2(this, graphName,this.nodeManager, this.sessionClient, this + // return new GrpcStoreNodeSessionImpl2(this, graphName,this.nodeManager, this + // .sessionClient, this // .streamClient); - return new GrpcStoreNodeSessionImpl(this, graphName, this.nodeManager, this.sessionClient, this.streamClient); + return new GrpcStoreNodeSessionImpl(this, graphName, this.nodeManager, this.sessionClient, + this.streamClient); } @Override diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreNodeSessionImpl.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreNodeSessionImpl.java similarity index 75% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreNodeSessionImpl.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreNodeSessionImpl.java index 4a6c3203a..b63f03434 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreNodeSessionImpl.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreNodeSessionImpl.java @@ -15,14 +15,7 @@ * under the License. */ -package com.baidu.hugegraph.store.client.grpc; - -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_LIST; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_TABLE; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.SCAN_TYPE_ANY; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.SCAN_TYPE_RANGE; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.isValid; +package org.apache.hugegraph.store.client.grpc; import java.util.Collections; import java.util.LinkedList; @@ -33,20 +26,21 @@ import java.util.stream.Collectors; import javax.annotation.concurrent.NotThreadSafe; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvStore; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.client.HgStoreNode; +import org.apache.hugegraph.store.client.HgStoreNodeManager; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.client.util.HgAssert; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.HgStoreClientUtil; +import org.apache.hugegraph.store.client.util.HgUuid; import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamStub; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgKvStore; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgScanQuery; -import com.baidu.hugegraph.store.client.HgStoreNode; -import com.baidu.hugegraph.store.client.HgStoreNodeManager; -import com.baidu.hugegraph.store.client.HgStoreNodeSession; -import com.baidu.hugegraph.store.client.util.HgAssert; -import com.baidu.hugegraph.store.client.util.HgStoreClientConfig; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import com.baidu.hugegraph.store.client.util.HgUuid; import com.baidu.hugegraph.store.grpc.common.GraphMethod; import com.baidu.hugegraph.store.grpc.common.Key; import com.baidu.hugegraph.store.grpc.common.OpType; @@ -65,20 +59,20 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @NotThreadSafe class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { - private static HgStoreClientConfig hgStoreClientConfig = HgStoreClientConfig.of(); - private HgStoreNode storeNode; - private String graphName; - private GrpcStoreSessionClient storeSessionClient; - private GrpcStoreStreamClient storeStreamClient; - private HgStoreNodeManager nodeManager; - private NotifyingExecutor notifier; - private SwitchingExecutor switcher; - private BatchEntry.Builder batchEntryBuilder = BatchEntry.newBuilder(); - private Key.Builder builder = Key.newBuilder(); + private static final HgStoreClientConfig hgStoreClientConfig = HgStoreClientConfig.of(); + private final HgStoreNode storeNode; + private final String graphName; + private final GrpcStoreSessionClient storeSessionClient; + private final GrpcStoreStreamClient storeStreamClient; + private final HgStoreNodeManager nodeManager; + private final NotifyingExecutor notifier; + private final SwitchingExecutor switcher; + private final BatchEntry.Builder batchEntryBuilder = BatchEntry.newBuilder(); + private final Key.Builder builder = Key.newBuilder(); private boolean isAutoCommit = true; private String batchId; private LinkedList batchEntries = new LinkedList<>(); - private static ConcurrentHashMap tables = new ConcurrentHashMap() {{ + private static final ConcurrentHashMap tables = new ConcurrentHashMap() {{ put("unknown", 0); put("vertex", 1); put("out_edge", 2); @@ -88,8 +82,10 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { put("olap", 6); }}; - GrpcStoreNodeSessionImpl(HgStoreNode storeNode, String graphName, HgStoreNodeManager nodeManager, - GrpcStoreSessionClient sessionClient , GrpcStoreStreamClient streamClient) { + GrpcStoreNodeSessionImpl(HgStoreNode storeNode, String graphName, + HgStoreNodeManager nodeManager, + GrpcStoreSessionClient sessionClient, + GrpcStoreStreamClient streamClient) { HgAssert.isFalse(HgAssert.isInvalid(graphName), "the argument: graphName is invalid."); HgAssert.isFalse(nodeManager == null, "the argument: nodeManager is null."); @@ -187,7 +183,7 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { } @Override - public boolean directPut(String table, int partitionId, HgOwnerKey key, byte[] value) { + public boolean directPut(String table, int partitionId, HgOwnerKey key, byte[] value) { return false; } @@ -241,7 +237,7 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { return this.notifier.invoke( () -> this.storeSessionClient.doBatch(this, this.getBatchId(), entries), e -> true - ).orElse(false); + ).orElse(false); } @Override @@ -250,7 +246,7 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { () -> this.storeSessionClient.doGet(this, table, ownerKey) , e -> e.getValueResponse().getValue().toByteArray() - ).orElse(EMPTY_BYTES); + ).orElse(HgStoreClientConst.EMPTY_BYTES); } @Override @@ -260,7 +256,7 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { , e -> true - ).orElse(false); + ).orElse(false); } @Override @@ -271,24 +267,26 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { .stream() .map(kv -> (HgKvEntry) new GrpcKvEntryImpl(kv.getKey().toByteArray() , kv.getValue().toByteArray(), kv.getCode()) - ) + ) .collect(Collectors.toList())) - .orElse((List) EMPTY_LIST); + .orElse((List) HgStoreClientConst.EMPTY_LIST); } @Override public HgKvIterator batchPrefix(String table, List keyList) { return GrpcKvIteratorImpl.of(this, this.storeStreamClient.doBatchScanOneShot(this, - HgScanQuery.prefixOf(table, - keyList)) - ); + HgScanQuery.prefixOf( + table, + keyList)) + ); } @Override public boolean existsTable(String table) { return this.notifier.invoke( - () -> this.storeSessionClient.doTable(this, table, TableMethod.TABLE_METHOD_EXISTS), + () -> this.storeSessionClient.doTable(this, table, + TableMethod.TABLE_METHOD_EXISTS), e -> true) .orElse(false); } @@ -296,7 +294,8 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { @Override public boolean createTable(String table) { return this.notifier.invoke( - () -> this.storeSessionClient.doTable(this, table, TableMethod.TABLE_METHOD_CREATE), + () -> this.storeSessionClient.doTable(this, table, + TableMethod.TABLE_METHOD_CREATE), e -> true) .orElse(false); } @@ -304,7 +303,8 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { @Override public boolean deleteTable(String table) { return this.notifier.invoke( - () -> this.storeSessionClient.doTable(this, table, TableMethod.TABLE_METHOD_DELETE), + () -> this.storeSessionClient.doTable(this, table, + TableMethod.TABLE_METHOD_DELETE), e -> true) .orElse(false); } @@ -312,7 +312,8 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { @Override public boolean dropTable(String table) { return this.notifier.invoke( - () -> this.storeSessionClient.doTable(this, table, TableMethod.TABLE_METHOD_DROP), + () -> this.storeSessionClient.doTable(this, table, + TableMethod.TABLE_METHOD_DROP), e -> true) .orElse(false); } @@ -320,7 +321,8 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { @Override public boolean deleteGraph(String graph) { return this.notifier.invoke( - () -> this.storeSessionClient.doGraph(this, graph, GraphMethod.GRAPH_METHOD_DELETE), + () -> this.storeSessionClient.doGraph(this, graph, + GraphMethod.GRAPH_METHOD_DELETE), e -> true) .orElse(false); } @@ -328,7 +330,8 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { @Override public boolean truncate() { return this.notifier.invoke( - () -> this.storeSessionClient.doTable(this, EMPTY_TABLE + () -> this.storeSessionClient.doTable(this, + HgStoreClientConst.EMPTY_TABLE , TableMethod.TABLE_METHOD_TRUNCATE), e -> true) .orElse(false); @@ -343,11 +346,14 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { public HgKvIterator scanIterator(String table, long limit) { return this.switcher.invoke(getSwitcherSupplier(limit) , () -> GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScan - (this, table, limit)) + (this, table, + limit)) , () -> GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScanOneShot - (this, table, limit)) - ).get(); + (this, table, + limit)) + ).get(); } + @Override public HgKvIterator scanIterator(ScanStreamReq.Builder builder) { HgStoreStreamStub stub = getStub(); @@ -379,15 +385,18 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { public HgKvIterator scanIterator(String table, long limit, byte[] query) { return this.switcher.invoke(getSwitcherSupplier(limit) , () -> GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScan - (this, table, limit, query)) + (this, table, + limit, query)) , () -> GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScanOneShot - (this, table, limit, query)) - ).get(); + (this, table, + limit, query)) + ).get(); } @Override public HgKvIterator scanIterator(String table, HgOwnerKey keyPrefix) { - return GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScan(this, table, keyPrefix, 0)); + return GrpcKvIteratorImpl.of(this, + this.storeStreamClient.doScan(this, table, keyPrefix, 0)); } @Override @@ -399,103 +408,119 @@ class GrpcStoreNodeSessionImpl implements HgStoreNodeSession { keyPrefix, limit)), () -> GrpcKvIteratorImpl.of(this, - this.storeStreamClient.doScanOneShot(this, - table, - keyPrefix, - limit))) + this.storeStreamClient.doScanOneShot( + this, + table, + keyPrefix, + limit))) .get(); } @Override - public HgKvIterator scanIterator(String table, HgOwnerKey keyPrefix, long limit, byte[] query) { + public HgKvIterator scanIterator(String table, HgOwnerKey keyPrefix, long limit, + byte[] query) { return this.switcher.invoke(getSwitcherSupplier(limit), () -> GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScan( - this, - table, - keyPrefix, - limit, - query)), + this, + table, + keyPrefix, + limit, + query)), () -> GrpcKvIteratorImpl.of(this, - this.storeStreamClient.doScanOneShot(this, - table, - keyPrefix, - limit, - query))) + this.storeStreamClient.doScanOneShot( + this, + table, + keyPrefix, + limit, + query))) .get(); } @Override - public HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey) { + public HgKvIterator scanIterator(String table, HgOwnerKey startKey, + HgOwnerKey endKey) { return scanIterator(table, startKey, endKey, 0, HgKvStore.SCAN_ANY, null); } @Override - public HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit) { - return scanIterator(table, startKey, endKey, limit, isValid(endKey) ? SCAN_TYPE_RANGE : SCAN_TYPE_ANY, null); + public HgKvIterator scanIterator(String table, HgOwnerKey startKey, + HgOwnerKey endKey, long limit) { + return scanIterator(table, startKey, endKey, limit, + HgStoreClientUtil.isValid(endKey) ? HgStoreClientConst.SCAN_TYPE_RANGE : + HgStoreClientConst.SCAN_TYPE_ANY, null); } @Override - public HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit, byte[] query) { - return scanIterator(table, startKey, endKey, limit, isValid(endKey) ? SCAN_TYPE_RANGE : SCAN_TYPE_ANY, query); + public HgKvIterator scanIterator(String table, HgOwnerKey startKey, + HgOwnerKey endKey, long limit, byte[] query) { + return scanIterator(table, startKey, endKey, limit, + HgStoreClientUtil.isValid(endKey) ? HgStoreClientConst.SCAN_TYPE_RANGE : + HgStoreClientConst.SCAN_TYPE_ANY, query); } @Override - public HgKvIterator scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, + public HgKvIterator scanIterator(String table, HgOwnerKey startKey, + HgOwnerKey endKey, long limit, int scanType, byte[] query) { return this.switcher.invoke(getSwitcherSupplier(limit), () -> GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScan( - this, - table, - startKey, - endKey, - limit, - scanType, - query)), + this, + table, + startKey, + endKey, + limit, + scanType, + query)), () -> GrpcKvIteratorImpl.of(this, - this.storeStreamClient.doScanOneShot(this, - table, - startKey, - endKey, - limit, - scanType, - query))) + this.storeStreamClient.doScanOneShot( + this, + table, + startKey, + endKey, + limit, + scanType, + query))) .get(); } @Override - public HgKvIterator scanIterator(String table, int codeFrom, int codeTo, int scanType, byte[] query) { + public HgKvIterator scanIterator(String table, int codeFrom, int codeTo, + int scanType, byte[] query) { //TODO: Should be changed when start using hashcode as partitionId. - if(log.isDebugEnabled())log.debug("scanIterator-scanType: {}", scanType); + if (log.isDebugEnabled()) log.debug("scanIterator-scanType: {}", scanType); return GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScan(this, table , HgOwnerKey.newEmpty().codeToKey(codeFrom) , HgOwnerKey.newEmpty().codeToKey(codeTo) , HgStoreClientConst.NO_LIMIT - , SCAN_PREFIX_BEGIN | SCAN_HASHCODE | scanType + , HgKvStore.SCAN_PREFIX_BEGIN | + HgKvStore.SCAN_HASHCODE | scanType , query - ) - ); + ) + ); } @Override public List> scanBatch(HgScanQuery scanQuery) { return Collections.singletonList(GrpcKvIteratorImpl.of(this, - this.storeStreamClient.doBatchScan(this, scanQuery) - )); + this.storeStreamClient.doBatchScan( + this, scanQuery) + )); } @Override public KvCloseableIterator> scanBatch2(HgScanQuery scanQuery) { - throw new RuntimeException("not implemented"); + throw new RuntimeException("not implemented"); } @Override - public KvCloseableIterator> scanBatch3(HgScanQuery scanQuery, KvCloseableIterator iterator) { + public KvCloseableIterator> scanBatch3(HgScanQuery scanQuery, + KvCloseableIterator iterator) { return this.storeStreamClient.doBatchScan3(this, scanQuery, iterator); } diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreSessionClient.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreSessionClient.java similarity index 65% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreSessionClient.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreSessionClient.java index 205a8a4a8..5f67bee78 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreSessionClient.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreSessionClient.java @@ -1,15 +1,29 @@ -package com.baidu.hugegraph.store.client.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import static com.baidu.hugegraph.store.client.grpc.GrpcUtil.getHeader; -import static com.baidu.hugegraph.store.client.grpc.GrpcUtil.toKey; -import static com.baidu.hugegraph.store.client.grpc.GrpcUtil.toTk; +package org.apache.hugegraph.store.client.grpc; import java.util.List; import javax.annotation.concurrent.ThreadSafe; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.client.HgStoreNodeSession; + import com.baidu.hugegraph.store.grpc.common.GraphMethod; import com.baidu.hugegraph.store.grpc.common.TableMethod; import com.baidu.hugegraph.store.grpc.session.BatchEntry; @@ -44,21 +58,26 @@ class GrpcStoreSessionClient extends AbstractGrpcClient { private HgStoreSessionBlockingStub getBlockingStub(HgStoreNodeSession nodeSession) { HgStoreSessionBlockingStub stub = - (HgStoreSessionBlockingStub) getBlockingStub(nodeSession.getStoreNode().getAddress()); + (HgStoreSessionBlockingStub) getBlockingStub( + nodeSession.getStoreNode().getAddress()); return stub; } FeedbackRes doGet(HgStoreNodeSession nodeSession, String table, HgOwnerKey ownerKey) { if (log.isDebugEnabled()) { log.debug("doGet: {}-{}-{}-{}", nodeSession, table, ownerKey, GetReq.newBuilder() - .setHeader(getHeader(nodeSession)) - .setTk(toTk(table, ownerKey)) + .setHeader( + GrpcUtil.getHeader( + nodeSession)) + .setTk(GrpcUtil.toTk( + table, + ownerKey)) .build()); } return this.getBlockingStub(nodeSession) .get2(GetReq.newBuilder() - .setHeader(getHeader(nodeSession)) - .setTk(toTk(table, ownerKey)) + .setHeader(GrpcUtil.getHeader(nodeSession)) + .setTk(GrpcUtil.toTk(table, ownerKey)) .build() ); } @@ -66,7 +85,7 @@ class GrpcStoreSessionClient extends AbstractGrpcClient { FeedbackRes doClean(HgStoreNodeSession nodeSession, int partId) { return this.getBlockingStub(nodeSession) .clean(CleanReq.newBuilder() - .setHeader(getHeader(nodeSession)) + .setHeader(GrpcUtil.getHeader(nodeSession)) .setPartition(partId) .build() ); @@ -74,10 +93,10 @@ class GrpcStoreSessionClient extends AbstractGrpcClient { FeedbackRes doBatchGet(HgStoreNodeSession nodeSession, String table, List keyList) { BatchGetReq.Builder builder = BatchGetReq.newBuilder(); - builder.setHeader(getHeader(nodeSession)).setTable(table); + builder.setHeader(GrpcUtil.getHeader(nodeSession)).setTable(table); for (HgOwnerKey key : keyList) { - builder.addKey(toKey(key)); + builder.addKey(GrpcUtil.toKey(key)); } if (log.isDebugEnabled()) { @@ -92,7 +111,7 @@ class GrpcStoreSessionClient extends AbstractGrpcClient { writeReq.addAllEntry(entries); return this.getBlockingStub(nodeSession) .batch(BatchReq.newBuilder() - .setHeader(getHeader(nodeSession)) + .setHeader(GrpcUtil.getHeader(nodeSession)) .setWriteReq(writeReq) .setBatchId(batchId) .build() @@ -102,7 +121,7 @@ class GrpcStoreSessionClient extends AbstractGrpcClient { FeedbackRes doTable(HgStoreNodeSession nodeSession, String table, TableMethod method) { return this.getBlockingStub(nodeSession) .table(TableReq.newBuilder() - .setHeader(getHeader(nodeSession)) + .setHeader(GrpcUtil.getHeader(nodeSession)) .setTableName(table) .setMethod(method) .build() @@ -112,7 +131,7 @@ class GrpcStoreSessionClient extends AbstractGrpcClient { FeedbackRes doGraph(HgStoreNodeSession nodeSession, String graph, GraphMethod method) { return this.getBlockingStub(nodeSession) .graph(GraphReq.newBuilder() - .setHeader(getHeader(nodeSession)) + .setHeader(GrpcUtil.getHeader(nodeSession)) .setGraphName(graph) .setMethod(method) .build() diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreStateClient.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreStateClient.java similarity index 68% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreStateClient.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreStateClient.java index 408ea169e..79368eb8d 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreStateClient.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreStateClient.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.client.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; import java.util.List; import java.util.Set; diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreStreamClient.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreStreamClient.java similarity index 81% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreStreamClient.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreStreamClient.java index e4a0b7ed4..0a3028114 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcStoreStreamClient.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreStreamClient.java @@ -15,19 +15,19 @@ * under the License. */ -package com.baidu.hugegraph.store.client.grpc; +package org.apache.hugegraph.store.client.grpc; import javax.annotation.concurrent.ThreadSafe; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.client.HgStoreNodeSession; import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc; import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamBlockingStub; import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamStub; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgScanQuery; -import com.baidu.hugegraph.store.client.HgStoreNodeSession; import com.baidu.hugegraph.store.grpc.common.Kv; import io.grpc.ManagedChannel; @@ -48,7 +48,7 @@ public class GrpcStoreStreamClient extends AbstractGrpcClient { } @Override - public AbstractAsyncStub getAsyncStub(ManagedChannel channel){ + public AbstractAsyncStub getAsyncStub(ManagedChannel channel) { return HgStoreStreamGrpc.newStub(channel); } @@ -58,10 +58,11 @@ public class GrpcStoreStreamClient extends AbstractGrpcClient { @Override public AbstractBlockingStub getBlockingStub(ManagedChannel channel) { - return HgStoreStreamGrpc.newBlockingStub(channel); + return HgStoreStreamGrpc.newBlockingStub(channel); } - KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, long limit, byte[] query) { + KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, long limit, + byte[] query) { return KvOneShotScanner.scanAll(nodeSession , this.getBlockingStub(nodeSession) , table @@ -70,7 +71,8 @@ public class GrpcStoreStreamClient extends AbstractGrpcClient { ); } - KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, long limit) { + KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, + long limit) { return KvOneShotScanner.scanAll(nodeSession , this.getBlockingStub(nodeSession) , table @@ -79,7 +81,8 @@ public class GrpcStoreStreamClient extends AbstractGrpcClient { ); } - KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, HgOwnerKey prefix, long limit) { + KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, + HgOwnerKey prefix, long limit) { return KvOneShotScanner.scanPrefix(nodeSession , this.getBlockingStub(nodeSession) , table @@ -89,7 +92,8 @@ public class GrpcStoreStreamClient extends AbstractGrpcClient { ); } - KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, HgOwnerKey prefix, long limit, + KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, + HgOwnerKey prefix, long limit, byte[] query) { return KvOneShotScanner.scanPrefix(nodeSession , this.getBlockingStub(nodeSession) @@ -100,7 +104,8 @@ public class GrpcStoreStreamClient extends AbstractGrpcClient { ); } - KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, HgOwnerKey startKey, + KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, + HgOwnerKey startKey, HgOwnerKey endKey , long limit , int scanType @@ -198,13 +203,17 @@ public class GrpcStoreStreamClient extends AbstractGrpcClient { // 返回多个小的迭代器，允许上层并行处理 KvCloseableIterator> doBatchScan3(HgStoreNodeSession nodeSession, - HgScanQuery scanQuery, KvCloseableIterator iterator) { - KvBatchScanner.scan(this.getStub(nodeSession), nodeSession.getGraphName(), scanQuery, iterator); + HgScanQuery scanQuery, + KvCloseableIterator iterator) { + KvBatchScanner.scan(this.getStub(nodeSession), nodeSession.getGraphName(), scanQuery, + iterator); return iterator; } - KvCloseableIterator doBatchScanOneShot(HgStoreNodeSession nodeSession, HgScanQuery scanQuery) { - return KvBatchOneShotScanner.scan(nodeSession, this.getBlockingStub(nodeSession), scanQuery); + KvCloseableIterator doBatchScanOneShot(HgStoreNodeSession nodeSession, + HgScanQuery scanQuery) { + return KvBatchOneShotScanner.scan(nodeSession, this.getBlockingStub(nodeSession), + scanQuery); } } diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcUtil.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcUtil.java similarity index 53% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcUtil.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcUtil.java index d2fa13338..d93b937ac 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/GrpcUtil.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcUtil.java @@ -1,15 +1,31 @@ -package com.baidu.hugegraph.store.client.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; +package org.apache.hugegraph.store.client.grpc; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.client.HgStoreNodeSession; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; + import com.baidu.hugegraph.store.grpc.common.Header; import com.baidu.hugegraph.store.grpc.common.Key; import com.baidu.hugegraph.store.grpc.common.Kv; @@ -27,20 +43,20 @@ import io.grpc.StatusRuntimeException; */ final class GrpcUtil { - private static ThreadLocal keyBuilder = new ThreadLocal(); + private static final ThreadLocal keyBuilder = new ThreadLocal(); static Header getHeader(HgStoreNodeSession nodeSession) { return Header.newBuilder() - .setGraph(nodeSession.getGraphName()) - .build(); + .setGraph(nodeSession.getGraphName()) + .build(); } static Tk toTk(String table, HgOwnerKey ownerKey) { return Tk.newBuilder() - .setTable(table) - .setKey(ByteString.copyFrom(ownerKey.getKey())) - .setCode(ownerKey.getKeyCode()) - .build(); + .setTable(table) + .setKey(ByteString.copyFrom(ownerKey.getKey())) + .setCode(ownerKey.getKeyCode()) + .build(); } static Key.Builder getOwnerKeyBuilder() { @@ -82,27 +98,27 @@ final class GrpcUtil { static Tkv toTkv(String table, HgOwnerKey ownerKey, byte[] value) { return Tkv.newBuilder() - .setTable(table) - .setKey(ByteString.copyFrom(ownerKey.getKey())) - .setValue(ByteString.copyFrom(value)) - .setCode(ownerKey.getKeyCode()) - .build(); + .setTable(table) + .setKey(ByteString.copyFrom(ownerKey.getKey())) + .setValue(ByteString.copyFrom(value)) + .setCode(ownerKey.getKeyCode()) + .build(); } static Tp toTp(String table, HgOwnerKey ownerKey) { return Tp.newBuilder() - .setTable(table) - .setPrefix(ByteString.copyFrom(ownerKey.getKey())) - .setCode(ownerKey.getKeyCode()) - .build(); + .setTable(table) + .setPrefix(ByteString.copyFrom(ownerKey.getKey())) + .setCode(ownerKey.getKeyCode()) + .build(); } static Tse toTse(String table, HgOwnerKey startKey, HgOwnerKey endKey) { return Tse.newBuilder() - .setTable(table) - .setStart(toKey(startKey)) - .setEnd(toKey(endKey)) - .build(); + .setTable(table) + .setStart(toKey(startKey)) + .setEnd(toKey(endKey)) + .build(); } @@ -116,17 +132,18 @@ final class GrpcUtil { while (iter.hasNext()) { Kv entry = iter.next(); - resList.add(new GrpcKvEntryImpl(entry.getKey().toByteArray(), entry.getValue().toByteArray(),entry.getCode())); + resList.add(new GrpcKvEntryImpl(entry.getKey().toByteArray(), + entry.getValue().toByteArray(), entry.getCode())); } return resList; } - static StatusRuntimeException toErr(String msg){ + static StatusRuntimeException toErr(String msg) { return new StatusRuntimeException(Status.UNKNOWN.withDescription(msg)); } - static ByteString toBs(byte[] bytes){ - return ByteString.copyFrom((bytes != null) ? bytes : EMPTY_BYTES); + static ByteString toBs(byte[] bytes) { + return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES); } } diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchOneShotScanner.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchOneShotScanner.java similarity index 77% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchOneShotScanner.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchOneShotScanner.java index a4c02b5ef..aae557cfb 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchOneShotScanner.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchOneShotScanner.java @@ -15,23 +15,23 @@ * under the License. */ -package com.baidu.hugegraph.store.client.grpc; +package org.apache.hugegraph.store.client.grpc; -import static com.baidu.hugegraph.store.client.grpc.KvBatchUtil.EMPTY_POSITION; -import static com.baidu.hugegraph.store.client.grpc.KvBatchUtil.createQueryReq; -import static com.baidu.hugegraph.store.client.grpc.KvBatchUtil.getHeader; +import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.EMPTY_POSITION; +import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.createQueryReq; +import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.getHeader; import java.util.Iterator; import java.util.List; import javax.annotation.concurrent.NotThreadSafe; +import org.apache.hugegraph.store.HgPageSize; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.HgSeekAble; +import org.apache.hugegraph.store.client.HgStoreNodeSession; import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc; -import com.baidu.hugegraph.store.HgPageSize; -import com.baidu.hugegraph.store.HgScanQuery; -import com.baidu.hugegraph.store.HgSeekAble; -import com.baidu.hugegraph.store.client.HgStoreNodeSession; import com.baidu.hugegraph.store.grpc.common.Kv; import com.baidu.hugegraph.store.grpc.stream.ScanStreamBatchReq; @@ -68,9 +68,9 @@ class KvBatchOneShotScanner implements KvCloseableIterator, HgPageSize, HgSe private ScanStreamBatchReq createReq() { return ScanStreamBatchReq.newBuilder() - .setHeader(getHeader(this.nodeSession)) - .setQueryRequest(createQueryReq(this.scanQuery, Integer.MAX_VALUE)) - .build(); + .setHeader(getHeader(this.nodeSession)) + .setQueryRequest(createQueryReq(this.scanQuery, Integer.MAX_VALUE)) + .build(); } private Iterator createIterator() { @@ -82,7 +82,7 @@ class KvBatchOneShotScanner implements KvCloseableIterator, HgPageSize, HgSe @Override public boolean hasNext() { if (this.iterator == null) { - this.iterator =this.createIterator(); + this.iterator = this.createIterator(); } return this.iterator.hasNext(); } @@ -90,7 +90,7 @@ class KvBatchOneShotScanner implements KvCloseableIterator, HgPageSize, HgSe @Override public Kv next() { if (this.iterator == null) { - this.iterator =this.createIterator(); + this.iterator = this.createIterator(); } return this.iterator.next(); } diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchScanner.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchScanner.java similarity index 79% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchScanner.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchScanner.java index ad898a381..a6c31cba0 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchScanner.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchScanner.java @@ -15,9 +15,9 @@ * under the License. */ -package com.baidu.hugegraph.store.client.grpc; +package org.apache.hugegraph.store.client.grpc; -import static com.baidu.hugegraph.store.client.grpc.KvBatchUtil.createQueryReq; +import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.createQueryReq; import java.io.Closeable; import java.nio.ByteBuffer; @@ -29,16 +29,16 @@ import java.util.function.Supplier; import javax.annotation.concurrent.NotThreadSafe; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvOrderedIterator; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.buffer.KVByteBuffer; +import org.apache.hugegraph.store.client.util.PropertyUtil; import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc; import org.apache.hugegraph.store.grpc.stream.KvStream; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgKvOrderedIterator; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgScanQuery; -import com.baidu.hugegraph.store.buffer.KVByteBuffer; -import com.baidu.hugegraph.store.client.util.PropertyUtil; import com.baidu.hugegraph.store.grpc.common.Header; import com.baidu.hugegraph.store.grpc.common.ScanOrderType; import com.baidu.hugegraph.store.grpc.stream.ScanReceiptRequest; @@ -66,20 +66,21 @@ public class KvBatchScanner implements Closeable { private final HgScanQuery scanQuery; static final Supplier> NO_DATA = () -> null; private final ScanReceiptRequest.Builder responseBuilder = ScanReceiptRequest.newBuilder(); - private KvBatchReceiver receiver; + private final KvBatchReceiver receiver; public KvBatchScanner( - HgStoreStreamGrpc.HgStoreStreamStub stub, - String graphName, - HgScanQuery scanQuery, - KvCloseableIterator iterator) { + HgStoreStreamGrpc.HgStoreStreamStub stub, + String graphName, + HgScanQuery scanQuery, + KvCloseableIterator iterator) { this.graphName = graphName; this.notifier = (KvBatchScannerMerger) iterator; this.notifier.registerScanner(this); this.running = true; this.scanQuery = scanQuery; - receiver = new KvBatchReceiver(this, scanQuery.getOrderType() == ScanOrderType.ORDER_STRICT); + receiver = + new KvBatchReceiver(this, scanQuery.getOrderType() == ScanOrderType.ORDER_STRICT); sender = stub.scanBatch2(receiver); sendQuery(this.scanQuery); // 发送查询请求 } @@ -93,10 +94,11 @@ public class KvBatchScanner implements Closeable { * @return data merger iterator */ public static KvCloseableIterator ofMerger( - HgScanQuery scanQuery, BiFunction handler) { + HgScanQuery scanQuery, BiFunction handler) { KvBatchScannerMerger merger; if (scanQuery.getOrderType() == ScanOrderType.ORDER_STRICT) { - merger = new KvBatchScannerMerger.SortedScannerMerger(new TaskSplitter(scanQuery, handler)); + merger = new KvBatchScannerMerger.SortedScannerMerger( + new TaskSplitter(scanQuery, handler)); } else { merger = new KvBatchScannerMerger(new TaskSplitter(scanQuery, handler)); } @@ -105,10 +107,10 @@ public class KvBatchScanner implements Closeable { } public static void scan( - HgStoreStreamGrpc.HgStoreStreamStub stub, - String graphName, - HgScanQuery scanQuery, - KvCloseableIterator iterator) { + HgStoreStreamGrpc.HgStoreStreamStub stub, + String graphName, + HgScanQuery scanQuery, + KvCloseableIterator iterator) { new KvBatchScanner(stub, graphName, scanQuery, iterator); } @@ -121,10 +123,11 @@ public class KvBatchScanner implements Closeable { synchronized (this.sender) { if (running) { this.sender.onNext( - ScanStreamBatchReq.newBuilder() - .setHeader(Header.newBuilder().setGraph(graphName).build()) - .setQueryRequest(createQueryReq(query, 0)) - .build()); + ScanStreamBatchReq.newBuilder() + .setHeader( + Header.newBuilder().setGraph(graphName).build()) + .setQueryRequest(createQueryReq(query, 0)) + .build()); } } } @@ -145,15 +148,18 @@ public class KvBatchScanner implements Closeable { synchronized (this.sender) { if (running) { this.sender.onNext( - ScanStreamBatchReq.newBuilder() - .setHeader(Header.newBuilder().setGraph(graphName).build()) - .setReceiptRequest(responseBuilder.setTimes(seqNo).build()) - .build()); + ScanStreamBatchReq.newBuilder() + .setHeader( + Header.newBuilder().setGraph(graphName).build()) + .setReceiptRequest( + responseBuilder.setTimes(seqNo).build()) + .build()); } } } - public void dataArrived(Supplier> supplier) throws InterruptedException { + public void dataArrived(Supplier> supplier) throws + InterruptedException { notifier.dataArrived(this, supplier); } @@ -194,12 +200,14 @@ public class KvBatchScanner implements Closeable { private KvBatchScannerMerger notifier; private Iterator prefixItr; private int maxTaskSize = 0; // 最大并行任务数 - private int maxBatchSize = PropertyUtil.getInt("net.kv.scanner.batch.size", 1000); // 每批次最大点数量 + private int maxBatchSize = PropertyUtil.getInt("net.kv.scanner.batch.size", 1000); + // 每批次最大点数量 private volatile boolean finished = false; private volatile boolean splitting = false; private volatile int nextKeySerialNo = 1; - public TaskSplitter(HgScanQuery scanQuery, BiFunction handler) { + public TaskSplitter(HgScanQuery scanQuery, + BiFunction handler) { this.scanQuery = scanQuery; this.taskHandler = handler; if (scanQuery.getScanMethod() == HgScanQuery.ScanMethod.PREFIX) { @@ -253,13 +261,15 @@ public class KvBatchScanner implements Closeable { } this.splitting = true; if (scanQuery.getScanMethod() == HgScanQuery.ScanMethod.PREFIX) { - if (prefixItr.hasNext() && (maxTaskSize == 0 || notifier.getScannerCount() < maxTaskSize)) { + if (prefixItr.hasNext() && + (maxTaskSize == 0 || notifier.getScannerCount() < maxTaskSize)) { List keys = new ArrayList<>(maxBatchSize); for (int i = 0; i < maxBatchSize && prefixItr.hasNext(); i++) { keys.add(prefixItr.next().setSerialNo(nextKeySerialNo++)); } taskHandler.apply( - HgScanQuery.prefixOf(scanQuery.getTable(), keys, scanQuery.getOrderType()), this.notifier); + HgScanQuery.prefixOf(scanQuery.getTable(), keys, + scanQuery.getOrderType()), this.notifier); // 评估最大任务数 evaluateMaxTaskSize(); if (this.notifier.getScannerCount() < this.maxTaskSize) { @@ -299,13 +309,13 @@ public class KvBatchScanner implements Closeable { int seqNo = value.getSeqNo(); boolean isOver = value.getOver(); scanner.dataArrived( - () -> { - scanner.sendResponse(seqNo); - if (isOver) { - scanner.dataComplete(); - } - return new KVBytesIterator(buffer, sortByVertex, scanner); - }); + () -> { + scanner.sendResponse(seqNo); + if (isOver) { + scanner.dataComplete(); + } + return new KVBytesIterator(buffer, sortByVertex, scanner); + }); } catch (InterruptedException e) { close(); log.error("exception ", e); diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchScanner5.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchScanner5.java similarity index 82% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchScanner5.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchScanner5.java index 7491601a4..57ef6a76e 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchScanner5.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchScanner5.java @@ -15,10 +15,7 @@ * under the License. */ -package com.baidu.hugegraph.store.client.grpc; - -import static com.baidu.hugegraph.store.client.grpc.KvBatchUtil.createQueryReq; -import static com.baidu.hugegraph.store.client.grpc.KvBatchUtil.getRequestBuilder; +package org.apache.hugegraph.store.client.grpc; import java.util.Iterator; import java.util.List; @@ -34,14 +31,14 @@ import java.util.function.Supplier; import javax.annotation.concurrent.NotThreadSafe; +import org.apache.hugegraph.store.HgPageSize; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.client.type.HgStoreClientException; +import org.apache.hugegraph.store.client.util.Base58; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc; -import com.baidu.hugegraph.store.HgPageSize; -import com.baidu.hugegraph.store.HgScanQuery; -import com.baidu.hugegraph.store.client.HgStoreNodeSession; -import com.baidu.hugegraph.store.client.type.HgStoreClientException; -import com.baidu.hugegraph.store.client.util.Base58; -import com.baidu.hugegraph.store.client.util.HgStoreClientConfig; import com.baidu.hugegraph.store.grpc.common.Kv; import com.baidu.hugegraph.store.grpc.stream.*; @@ -56,7 +53,8 @@ import lombok.extern.slf4j.Slf4j; @NotThreadSafe class KvBatchScanner5 { private final static HgStoreClientConfig storeClientConfig = HgStoreClientConfig.of(); - //private final static int HAVE_NEXT_TIMEOUT_SECONDS = storeClientConfig.getNetKvScannerHaveNextTimeout(); + //private final static int HAVE_NEXT_TIMEOUT_SECONDS = storeClientConfig + // .getNetKvScannerHaveNextTimeout(); private final static int HAVE_NEXT_TIMEOUT_SECONDS = 60; private final static long PAGE_SIZE = storeClientConfig.getNetKvScannerPageSize(); @@ -87,7 +85,8 @@ class KvBatchScanner5 { private final AtomicBoolean serverFinished = new AtomicBoolean(); private final AtomicBoolean clientFinished = new AtomicBoolean(); - private final ScanReceiptRequest.Builder receiptReqBuilder = ScanReceiptRequest.newBuilder(); + private final ScanReceiptRequest.Builder receiptReqBuilder = + ScanReceiptRequest.newBuilder(); private final ScanCancelRequest cancelReq = ScanCancelRequest.newBuilder().build(); private final HgStoreNodeSession nodeSession; private final OrderAgent agent; @@ -105,20 +104,22 @@ class KvBatchScanner5 { HgScanQuery scanQuery, HgStoreNodeSession nodeSession) { - if (log.isDebugEnabled()) + if (log.isDebugEnabled()) { if (scanQuery.getPrefixList() != null && scanQuery.getPrefixList().size() > 0) { brokerId = Base58.encode(scanQuery.getPrefixList().get(0).getKey()); - log.debug("[ANALYSIS START] [{}] firstKey: {}, keyLength: {}, table: {}, node: {}" + log.debug( + "[ANALYSIS START] [{}] firstKey: {}, keyLength: {}, table: {}, node: {}" , brokerId , scanQuery.getPrefixList().get(0) , scanQuery.getPrefixList().size() , scanQuery.getTable() , nodeSession.getStoreNode().getAddress()); } + } this.scanQuery = scanQuery; - this.reqBuilder = getRequestBuilder(nodeSession); + this.reqBuilder = KvBatchUtil.getRequestBuilder(nodeSession); this.nodeSession = nodeSession; this.agent = new OrderAgent(brokerId); this.requestObserver = stub.scanBatch(agent); @@ -157,33 +158,39 @@ class KvBatchScanner5 { if (!this.clientFinished.get()) { this.send(() -> - getReqBuilder().setReceiptRequest(this.receiptReqBuilder.setTimes(timesBuf.get()).build()).build() + getReqBuilder().setReceiptRequest( + this.receiptReqBuilder.setTimes(timesBuf.get()).build()) + .build() ); } } private void makeADeal() { this.send(() -> getReqBuilder() - .setQueryRequest(createQueryReq(scanQuery, PAGE_SIZE)).build() + .setQueryRequest(KvBatchUtil.createQueryReq(scanQuery, PAGE_SIZE)).build() ); } private void finish(long tookAmt) { this.clientFinished.set(true); - if (log.isDebugEnabled()) log.debug("[ANALYSIS END] [{}] times: {}, received: {}, took: {}" - , this.brokerId - , this.receivedLastTimes.get() - , this.receivedCount.get() - , tookAmt - ); - if (this.receivedCount.get() != tookAmt) { - if (log.isDebugEnabled()) log.debug("[ANALYSIS END] [{}] times: {}, received: {}, took: {}" + if (log.isDebugEnabled()) { + log.debug("[ANALYSIS END] [{}] times: {}, received: {}, took: {}" , this.brokerId , this.receivedLastTimes.get() , this.receivedCount.get() , tookAmt ); } + if (this.receivedCount.get() != tookAmt) { + if (log.isDebugEnabled()) { + log.debug("[ANALYSIS END] [{}] times: {}, received: {}, took: {}" + , this.brokerId + , this.receivedLastTimes.get() + , this.receivedCount.get() + , tookAmt + ); + } + } synchronized (this.state) { if (this.state.value < OrderState.COMPLETED.value) { this.send(() -> getReqBuilder().setCancelRequest(this.cancelReq).build()); @@ -199,7 +206,7 @@ class KvBatchScanner5 { private void send(Supplier supplier) { this.senderLock.lock(); try { - if (this.serverFinished.get() == false) { + if (!this.serverFinished.get()) { this.requestObserver.onNext(supplier.get()); } Thread.yield(); @@ -210,7 +217,7 @@ class KvBatchScanner5 { private class OrderAgent implements StreamObserver { private final AtomicInteger count = new AtomicInteger(0); - private AtomicBoolean over = new AtomicBoolean(false); + private final AtomicBoolean over = new AtomicBoolean(false); private final String agentId; OrderAgent(String agentId) { @@ -221,7 +228,8 @@ class KvBatchScanner5 { public void onNext(KvPageRes value) { if (log.isDebugEnabled()) { log.debug("Scan [ {} ] [ {} ] times, received: [ {} ]" - , nodeSession.getStoreNode().getAddress(), value.getTimes(), value.getDataList().size()); + , nodeSession.getStoreNode().getAddress(), value.getTimes(), + value.getDataList().size()); } serverFinished.set(value.getOver()); @@ -298,7 +306,9 @@ class KvBatchScanner5 { if (this.done.get()) { res = () -> null; } else { - throw HgStoreClientException.of("Timeout, max time: " + HAVE_NEXT_TIMEOUT_SECONDS + " seconds" + + throw HgStoreClientException.of( + "Timeout, max time: " + HAVE_NEXT_TIMEOUT_SECONDS + + " seconds" + ", isOver: " + this.done.get() + ", isStop: " + this.stop.get() + ", last-times: " + this.lastTimes + @@ -308,12 +318,15 @@ class KvBatchScanner5 { } } catch (InterruptedException e) { - log.error("Failed to receive List from queue because of interruption of current thread [" + log.error( + "Failed to receive List from queue because of interruption of " + + "current thread [" + Thread.currentThread().getName() + "]"); Thread.currentThread().interrupt(); - throw HgStoreClientException.of("Failed to receive List from queue, cause by:", e); + throw HgStoreClientException.of( + "Failed to receive List from queue, cause by:", e); } finally { pickUpLock.unlock(); } @@ -347,7 +360,7 @@ class KvBatchScanner5 { /* iterator */ private static class OrderConsumer implements KvCloseableIterator, HgPageSize { - private OrderBroker broker; + private final OrderBroker broker; private Iterator dataIterator; private long tookCount = 0; private final String consumerId; @@ -398,13 +411,15 @@ class KvBatchScanner5 { if (this.dataIterator == null) { if (log.isDebugEnabled()) { - log.debug("[ANALYSIS NULL -> FALSE] [{}] , tookCount: {}", consumerId, tookCount); + log.debug("[ANALYSIS NULL -> FALSE] [{}] , tookCount: {}", consumerId, + tookCount); } return false; } else { if (log.isDebugEnabled()) { if (!this.dataIterator.hasNext()) { - log.debug("[ANALYSIS hasNext -> FALSE] [{}] , tookCount: {}", consumerId, tookCount); + log.debug("[ANALYSIS hasNext -> FALSE] [{}] , tookCount: {}", consumerId, + tookCount); } } return this.dataIterator.hasNext(); diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchScannerMerger.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchScannerMerger.java similarity index 82% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchScannerMerger.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchScannerMerger.java index 812ebecfd..e7e62b2a6 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchScannerMerger.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchScannerMerger.java @@ -1,6 +1,21 @@ -package com.baidu.hugegraph.store.client.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import static com.baidu.hugegraph.store.client.grpc.KvBatchScanner.NO_DATA; +package org.apache.hugegraph.store.client.grpc; import java.util.Iterator; import java.util.List; @@ -12,11 +27,11 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgKvOrderedIterator; -import com.baidu.hugegraph.store.HgPageSize; -import com.baidu.hugegraph.store.client.util.PropertyUtil; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvOrderedIterator; +import org.apache.hugegraph.store.HgPageSize; +import org.apache.hugegraph.store.client.util.PropertyUtil; import lombok.extern.slf4j.Slf4j; @@ -25,13 +40,15 @@ import lombok.extern.slf4j.Slf4j; * 对请求任务的拆分,创建多个请求队列 */ @Slf4j -public class KvBatchScannerMerger implements KvCloseableIterator>, HgPageSize { +public class KvBatchScannerMerger implements KvCloseableIterator>, + HgPageSize { static int maxWaitCount = PropertyUtil.getInt("net.kv.scanner.wait.timeout", 60); - protected final BlockingQueue>> queue = new LinkedBlockingQueue<>(); + protected final BlockingQueue>> queue = + new LinkedBlockingQueue<>(); private Supplier> current = null; private final KvBatchScanner.TaskSplitter taskSplitter; - private List scanners = new CopyOnWriteArrayList<>(); + private final List scanners = new CopyOnWriteArrayList<>(); public KvBatchScannerMerger(KvBatchScanner.TaskSplitter splitter) { this.taskSplitter = splitter; @@ -62,8 +79,9 @@ public class KvBatchScannerMerger implements KvCloseableIterator maxWaitCount) { - log.error("KvBatchScanner wait data timeout {}, closeables is {}, task is {}", - waitTime, scanners.size(), taskSplitter.isFinished()); + log.error( + "KvBatchScanner wait data timeout {}, closeables is {}, task is {}", + waitTime, scanners.size(), taskSplitter.isFinished()); break; } } @@ -72,7 +90,7 @@ public class KvBatchScannerMerger implements KvCloseableIterator maxWaitCount) { break; } - } else if (current == NO_DATA) { + } else if (current == KvBatchScanner.NO_DATA) { queue = null; break; } else { @@ -207,7 +225,7 @@ public class KvBatchScannerMerger implements KvCloseableIterator scannerQueues = new ConcurrentHashMap<>(); + private final Map scannerQueues = new ConcurrentHashMap<>(); public SortedScannerMerger(KvBatchScanner.TaskSplitter splitter) { super(splitter); @@ -298,11 +316,12 @@ public class KvBatchScannerMerger implements KvCloseableIterator> supplier) { + public void dataArrived(KvBatchScanner scanner, + Supplier> supplier) { scannerQueues.putIfAbsent(scanner, new ScannerDataQueue()); scannerQueues.get(scanner).add(supplier); } diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchUtil.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchUtil.java similarity index 58% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchUtil.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchUtil.java index d8bbeca76..55b62856a 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvBatchUtil.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchUtil.java @@ -1,20 +1,38 @@ -package com.baidu.hugegraph.store.client.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.util.List; + +import javax.annotation.concurrent.NotThreadSafe; + +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgScanQuery; -import com.baidu.hugegraph.store.client.HgStoreNodeSession; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; import com.baidu.hugegraph.store.grpc.common.Header; import com.baidu.hugegraph.store.grpc.common.ScanMethod; import com.baidu.hugegraph.store.grpc.stream.*; import com.google.protobuf.ByteString; + import lombok.extern.slf4j.Slf4j; -import javax.annotation.concurrent.NotThreadSafe; -import java.util.List; - -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; - /** * @author lynn.bond@hotmail.com created on 2022/04/23 */ @@ -70,34 +88,41 @@ class KvBatchUtil { return Header.newBuilder().setGraph(nodeSession.getGraphName()).build(); } - static void addPrefixCondition(HgScanQuery scanQuery, ScanQueryRequest.Builder qb, ScanCondition.Builder cb) { + static void addPrefixCondition(HgScanQuery scanQuery, ScanQueryRequest.Builder qb, + ScanCondition.Builder cb) { List prefixList = scanQuery.getPrefixList(); if (prefixList == null || prefixList.isEmpty()) { - throw new RuntimeException("The start-list of ScanQuery shouldn't to be invalid in ScanMethod.PREFIX mode."); + throw new RuntimeException( + "The start-list of ScanQuery shouldn't to be invalid in ScanMethod.PREFIX " + + "mode."); } prefixList.forEach((e) -> { qb.addCondition(cb.clear() - .setPrefix(toBs(e.getKey())) - .setCode(e.getKeyCode()) - .setSerialNo(e.getSerialNo()) - .build() + .setPrefix(toBs(e.getKey())) + .setCode(e.getKeyCode()) + .setSerialNo(e.getSerialNo()) + .build() ); }); } - static void addRangeCondition(HgScanQuery scanQuery, ScanQueryRequest.Builder qb, ScanCondition.Builder cb) { + static void addRangeCondition(HgScanQuery scanQuery, ScanQueryRequest.Builder qb, + ScanCondition.Builder cb) { List startList = scanQuery.getStartList(); List endList = scanQuery.getEndList(); if (startList == null || startList.isEmpty()) { - throw new RuntimeException("The start-list of ScanQuery shouldn't to be invalid in ScanMethod.RANGE mode."); + throw new RuntimeException( + "The start-list of ScanQuery shouldn't to be invalid in ScanMethod.RANGE mode" + + "."); } if (endList == null || endList.isEmpty()) { - throw new RuntimeException("The end-list of ScanQuery shouldn't to be invalid in ScanMethod.RANGE mode."); + throw new RuntimeException( + "The end-list of ScanQuery shouldn't to be invalid in ScanMethod.RANGE mode."); } if (startList.size() != endList.size()) { @@ -108,10 +133,10 @@ class KvBatchUtil { HgOwnerKey start = startList.get(i); HgOwnerKey end = endList.get(i); qb.addCondition(cb.clear().setCode(start.getKeyCode()) - .setStart(toBs(start.getKey())) - .setEnd(toBs(end.getKey())) - .setSerialNo(start.getSerialNo()) - .build() + .setStart(toBs(start.getKey())) + .setEnd(toBs(end.getKey())) + .setSerialNo(start.getSerialNo()) + .build() ); } @@ -122,7 +147,7 @@ class KvBatchUtil { } static ByteString toBs(byte[] bytes) { - return ByteString.copyFrom((bytes != null) ? bytes : EMPTY_BYTES); + return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES); } } \ No newline at end of file diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvCloseableIterator.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvCloseableIterator.java new file mode 100644 index 000000000..d70879831 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvCloseableIterator.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.io.Closeable; +import java.util.Iterator; + +/** + * @author lynn.bond@hotmail.com on 2022/3/16 + */ +public interface KvCloseableIterator extends Iterator, Closeable { + void close(); +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvListIterator.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvListIterator.java new file mode 100644 index 000000000..3ef8be0f7 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvListIterator.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.util.Iterator; +import java.util.List; + +/** + * @author lynn.bond@hotmail.com on 2022/4/6 + */ +class KvListIterator implements KvCloseableIterator { + + private final Iterator iterator; + + KvListIterator(List list) { + this.iterator = list.iterator(); + } + + @Override + public void close() { + /*Nothing to do.*/ + } + + @Override + public boolean hasNext() { + return this.iterator.hasNext(); + } + + @Override + public T next() { + return this.iterator.next(); + } +} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvOneShotScanner.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvOneShotScanner.java similarity index 75% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvOneShotScanner.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvOneShotScanner.java index a307ff6f3..5cc703a68 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvOneShotScanner.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvOneShotScanner.java @@ -15,27 +15,23 @@ * under the License. */ -package com.baidu.hugegraph.store.client.grpc; - -import static com.baidu.hugegraph.store.HgKvStore.SCAN_ANY; -import static com.baidu.hugegraph.store.HgKvStore.SCAN_PREFIX_BEGIN; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toBytes; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toLong; +package org.apache.hugegraph.store.client.grpc; import java.util.Iterator; import java.util.List; import javax.annotation.concurrent.NotThreadSafe; +import org.apache.hugegraph.store.HgKvStore; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgPageSize; +import org.apache.hugegraph.store.HgSeekAble; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.HgStoreClientUtil; import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamBlockingStub; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgPageSize; -import com.baidu.hugegraph.store.HgSeekAble; -import com.baidu.hugegraph.store.client.HgStoreNodeSession; -import com.baidu.hugegraph.store.client.util.HgStoreClientConfig; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; import com.baidu.hugegraph.store.grpc.common.Header; import com.baidu.hugegraph.store.grpc.common.Kv; import com.baidu.hugegraph.store.grpc.common.ScanMethod; @@ -53,7 +49,7 @@ class KvOneShotScanner implements KvCloseableIterator, HgPageSize, HgSeekAbl private static final HgStoreClientConfig storeClientConfig = HgStoreClientConfig.of(); private final HgStoreNodeSession session; private final HgStoreStreamBlockingStub stub; - private ScanStreamReq.Builder reqBuilder = ScanStreamReq.newBuilder(); + private final ScanStreamReq.Builder reqBuilder = ScanStreamReq.newBuilder(); private ScanStreamReq req; private Iterator iterator; private List list = null; @@ -70,28 +66,38 @@ class KvOneShotScanner implements KvCloseableIterator, HgPageSize, HgSeekAbl private boolean in = true; private byte[] nodePosition = HgStoreClientConst.EMPTY_BYTES; - public static KvCloseableIterator scanAll(HgStoreNodeSession session, HgStoreStreamBlockingStub stub, + public static KvCloseableIterator scanAll(HgStoreNodeSession session, + HgStoreStreamBlockingStub stub, String table, long limit, byte[] query) { - return new KvOneShotScanner(ScanMethod.ALL, session, stub, table, null, null, null, limit, -1, SCAN_ANY, + return new KvOneShotScanner(ScanMethod.ALL, session, stub, table, null, null, null, limit, + -1, HgKvStore.SCAN_ANY, query); } - public static KvCloseableIterator scanPrefix(HgStoreNodeSession session, HgStoreStreamBlockingStub stub, - String table, HgOwnerKey prefix, long limit, byte[] query) { - return new KvOneShotScanner(ScanMethod.PREFIX, session, stub, table, prefix, null, null, limit, - prefix.getKeyCode(), SCAN_PREFIX_BEGIN, query); + public static KvCloseableIterator scanPrefix(HgStoreNodeSession session, + HgStoreStreamBlockingStub stub, + String table, HgOwnerKey prefix, long limit, + byte[] query) { + return new KvOneShotScanner(ScanMethod.PREFIX, session, stub, table, prefix, null, null, + limit, + prefix.getKeyCode(), HgKvStore.SCAN_PREFIX_BEGIN, query); } - public static KvCloseableIterator scanRange(HgStoreNodeSession nodeSession, HgStoreStreamBlockingStub stub, - String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit, + public static KvCloseableIterator scanRange(HgStoreNodeSession nodeSession, + HgStoreStreamBlockingStub stub, + String table, HgOwnerKey startKey, + HgOwnerKey endKey, long limit, int scanType, byte[] query) { - return new KvOneShotScanner(ScanMethod.RANGE, nodeSession, stub, table, null, startKey, endKey, limit, + return new KvOneShotScanner(ScanMethod.RANGE, nodeSession, stub, table, null, startKey, + endKey, limit, startKey.getKeyCode(), scanType, query); } - private KvOneShotScanner(ScanMethod scanMethod, HgStoreNodeSession session, HgStoreStreamBlockingStub stub, - String table, HgOwnerKey prefix, HgOwnerKey startKey, HgOwnerKey endKey, long limit, + private KvOneShotScanner(ScanMethod scanMethod, HgStoreNodeSession session, + HgStoreStreamBlockingStub stub, + String table, HgOwnerKey prefix, HgOwnerKey startKey, + HgOwnerKey endKey, long limit, int partition, int scanType, byte[] query) { this.scanMethod = scanMethod; this.session = session; @@ -103,7 +109,8 @@ class KvOneShotScanner implements KvCloseableIterator, HgPageSize, HgSeekAbl this.partition = partition; this.scanType = scanType; this.query = query != null ? query : HgStoreClientConst.EMPTY_BYTES; - this.limit = limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE : limit; // <=0 means no limit + this.limit = limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE : + limit; // <=0 means no limit this.pageSize = storeClientConfig.getNetKvScannerPageSize(); } @@ -170,7 +177,7 @@ class KvOneShotScanner implements KvCloseableIterator, HgPageSize, HgSeekAbl @Override public byte[] position() { - return toBytes(this.session.getStoreNode().getNodeId().longValue()); + return HgStoreClientUtil.toBytes(this.session.getStoreNode().getNodeId().longValue()); } @Override @@ -181,7 +188,7 @@ class KvOneShotScanner implements KvCloseableIterator, HgPageSize, HgSeekAbl byte[] nodeIdBytes = new byte[Long.BYTES]; System.arraycopy(position, 0, nodeIdBytes, 0, Long.BYTES); long nodeId = this.session.getStoreNode().getNodeId().longValue(); - long pId = toLong(nodeIdBytes); + long pId = HgStoreClientUtil.toLong(nodeIdBytes); this.in = nodeId >= pId; if (this.in && nodeId == pId) { this.nodePosition = new byte[position.length - Long.BYTES]; @@ -196,7 +203,7 @@ class KvOneShotScanner implements KvCloseableIterator, HgPageSize, HgSeekAbl } static ByteString toBs(byte[] bytes) { - return ByteString.copyFrom((bytes != null) ? bytes : EMPTY_BYTES); + return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES); } @Override diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvPageScanner.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvPageScanner.java similarity index 80% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvPageScanner.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvPageScanner.java index bf5f24f85..71a9ecab6 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/KvPageScanner.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvPageScanner.java @@ -15,13 +15,7 @@ * under the License. */ -package com.baidu.hugegraph.store.client.grpc; - -import static com.baidu.hugegraph.store.HgKvStore.SCAN_ANY; -import static com.baidu.hugegraph.store.HgKvStore.SCAN_PREFIX_BEGIN; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toBytes; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toLong; +package org.apache.hugegraph.store.client.grpc; import java.util.Collections; import java.util.Iterator; @@ -33,16 +27,18 @@ import java.util.concurrent.atomic.AtomicBoolean; import javax.annotation.concurrent.NotThreadSafe; +import org.apache.hugegraph.store.HgKvStore; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgPageSize; +import org.apache.hugegraph.store.HgSeekAble; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.client.util.HgBufferProxy; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.HgStoreClientUtil; +import org.apache.hugegraph.store.client.util.MetricX; import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamStub; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgPageSize; -import com.baidu.hugegraph.store.HgSeekAble; -import com.baidu.hugegraph.store.client.HgStoreNodeSession; -import com.baidu.hugegraph.store.client.util.HgBufferProxy; -import com.baidu.hugegraph.store.client.util.HgStoreClientConfig; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import com.baidu.hugegraph.store.client.util.MetricX; import com.baidu.hugegraph.store.grpc.common.Header; import com.baidu.hugegraph.store.grpc.common.Kv; import com.baidu.hugegraph.store.grpc.common.ScanMethod; @@ -62,8 +58,8 @@ import lombok.extern.slf4j.Slf4j; @NotThreadSafe class KvPageScanner implements KvCloseableIterator, HgPageSize, HgSeekAble { - private static HgStoreClientConfig clientConfig = HgStoreClientConfig.of(); - private static int nextTimeout = clientConfig.getNetKvScannerHaveNextTimeout(); + private static final HgStoreClientConfig clientConfig = HgStoreClientConfig.of(); + private static final int nextTimeout = clientConfig.getNetKvScannerHaveNextTimeout(); private final HgStoreNodeSession session; private final HgStoreStreamStub stub; private int pageSize = clientConfig.getNetKvScannerPageSize(); @@ -72,13 +68,15 @@ class KvPageScanner implements KvCloseableIterator, HgPageSize, HgSeekAble { private Iterator iterator; private StreamObserver observer; private ScanStreamReq.Builder reqBuilder = ScanStreamReq.newBuilder(); - private SelectParam.Builder selectBuilder = SelectParam.newBuilder(); + private final SelectParam.Builder selectBuilder = SelectParam.newBuilder(); private final BlockingQueue reqQueue = new LinkedBlockingQueue<>(); private boolean in = true; private byte[] nodePosition = HgStoreClientConst.EMPTY_BYTES; - private KvPageScanner(ScanMethod scanMethod, HgStoreNodeSession session, HgStoreStreamStub stub, String table, - HgOwnerKey prefix, HgOwnerKey startKey, HgOwnerKey endKey, long limit, int partition, + private KvPageScanner(ScanMethod scanMethod, HgStoreNodeSession session, HgStoreStreamStub stub, + String table, + HgOwnerKey prefix, HgOwnerKey startKey, HgOwnerKey endKey, long limit, + int partition, int scanType, byte[] query) { this.session = session; this.stub = stub; @@ -98,7 +96,8 @@ class KvPageScanner implements KvCloseableIterator, HgPageSize, HgSeekAble { this.init(); } - public KvPageScanner(HgStoreNodeSession session, HgStoreStreamStub stub, ScanStreamReq.Builder reqBuilder) { + public KvPageScanner(HgStoreNodeSession session, HgStoreStreamStub stub, + ScanStreamReq.Builder reqBuilder) { this.session = session; this.stub = stub; reqBuilder.setPageSize(pageSize); @@ -107,22 +106,29 @@ class KvPageScanner implements KvCloseableIterator, HgPageSize, HgSeekAble { this.init(); } - public static KvCloseableIterator scanAll(HgStoreNodeSession nodeSession, HgStoreStreamStub stub, String table, + public static KvCloseableIterator scanAll(HgStoreNodeSession nodeSession, + HgStoreStreamStub stub, String table, long limit, byte[] query) { return new KvPageScanner(ScanMethod.ALL, nodeSession, stub, table, null, null, null, limit, - -1, SCAN_ANY, query); + -1, HgKvStore.SCAN_ANY, query); } - public static KvCloseableIterator scanPrefix(HgStoreNodeSession nodeSession, HgStoreStreamStub stub, - String table, HgOwnerKey prefix, long limit, byte[] query) { - return new KvPageScanner(ScanMethod.PREFIX, nodeSession, stub, table, prefix, null, null, limit, - prefix.getKeyCode(), SCAN_PREFIX_BEGIN, query); + public static KvCloseableIterator scanPrefix(HgStoreNodeSession nodeSession, + HgStoreStreamStub stub, + String table, HgOwnerKey prefix, long limit, + byte[] query) { + return new KvPageScanner(ScanMethod.PREFIX, nodeSession, stub, table, prefix, null, null, + limit, + prefix.getKeyCode(), HgKvStore.SCAN_PREFIX_BEGIN, query); } - public static KvCloseableIterator scanRange(HgStoreNodeSession nodeSession, HgStoreStreamStub stub, - String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit, + public static KvCloseableIterator scanRange(HgStoreNodeSession nodeSession, + HgStoreStreamStub stub, + String table, HgOwnerKey startKey, + HgOwnerKey endKey, long limit, int scanType, byte[] query) { - return new KvPageScanner(ScanMethod.RANGE, nodeSession, stub, table, null, startKey, endKey, limit, + return new KvPageScanner(ScanMethod.RANGE, nodeSession, stub, table, null, startKey, endKey, + limit, startKey.getKeyCode(), scanType, query); } @@ -223,7 +229,8 @@ class KvPageScanner implements KvCloseableIterator, HgPageSize, HgSeekAble { start = System.nanoTime(); } List data = this.proxy.receive(nextTimeout, (sec) -> { - String msg = "failed to receive data from net scanning, because of timeout [ " + sec + " ] sec."; + String msg = "failed to receive data from net scanning, because of timeout [ " + sec + + " ] sec."; log.error(msg); this.clientError(msg); throw new RuntimeException(msg); @@ -259,7 +266,7 @@ class KvPageScanner implements KvCloseableIterator, HgPageSize, HgSeekAble { @Override public byte[] position() { - return toBytes(this.session.getStoreNode().getNodeId().longValue()); + return HgStoreClientUtil.toBytes(this.session.getStoreNode().getNodeId().longValue()); } @Override @@ -270,7 +277,7 @@ class KvPageScanner implements KvCloseableIterator, HgPageSize, HgSeekAble { byte[] nodeIdBytes = new byte[Long.BYTES]; System.arraycopy(position, 0, nodeIdBytes, 0, Long.BYTES); long nodeId = this.session.getStoreNode().getNodeId().longValue(); - long pId = toLong(nodeIdBytes); + long pId = HgStoreClientUtil.toLong(nodeIdBytes); this.in = nodeId >= pId; if (this.in && nodeId == pId) { this.nodePosition = new byte[position.length - Long.BYTES]; @@ -295,7 +302,7 @@ class KvPageScanner implements KvCloseableIterator, HgPageSize, HgSeekAble { } static ByteString toBs(byte[] bytes) { - return ByteString.copyFrom((bytes != null) ? bytes : EMPTY_BYTES); + return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES); } } diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/NotifyingExecutor.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/NotifyingExecutor.java similarity index 76% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/NotifyingExecutor.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/NotifyingExecutor.java index 8ba2f6494..718bd880d 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/grpc/NotifyingExecutor.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/NotifyingExecutor.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.client.grpc; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; import java.util.HashMap; import java.util.Map; @@ -8,11 +25,12 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; -import com.baidu.hugegraph.store.client.HgStoreNodeManager; -import com.baidu.hugegraph.store.client.HgStoreNodeSession; -import com.baidu.hugegraph.store.client.HgStoreNotice; -import com.baidu.hugegraph.store.client.type.HgNodeStatus; -import com.baidu.hugegraph.store.client.type.HgStoreClientException; +import org.apache.hugegraph.store.client.HgStoreNodeManager; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.client.HgStoreNotice; +import org.apache.hugegraph.store.client.type.HgNodeStatus; +import org.apache.hugegraph.store.client.type.HgStoreClientException; + import com.baidu.hugegraph.store.grpc.common.ResStatus; import com.baidu.hugegraph.store.grpc.session.FeedbackRes; import com.baidu.hugegraph.store.grpc.session.PartitionFaultResponse; @@ -28,13 +46,14 @@ import lombok.extern.slf4j.Slf4j; */ @Slf4j final class NotifyingExecutor { - private String graphName; - private HgStoreNodeManager nodeManager; - private HgStoreNodeSession nodeSession; + private final String graphName; + private final HgStoreNodeManager nodeManager; + private final HgStoreNodeSession nodeSession; private Map> partitionFaultHandlers; - NotifyingExecutor(String graphName, HgStoreNodeManager nodeManager, HgStoreNodeSession nodeSession) { + NotifyingExecutor(String graphName, HgStoreNodeManager nodeManager, + HgStoreNodeSession nodeSession) { this.graphName = graphName; this.nodeManager = nodeManager; this.nodeSession = nodeSession; @@ -61,8 +80,10 @@ final class NotifyingExecutor { throw err(t); } - if (log.isDebugEnabled()) log.debug("gRPC [{}] status: {}" - , this.nodeSession.getStoreNode().getAddress(), res.getStatus().getCode()); + if (log.isDebugEnabled()) { + log.debug("gRPC [{}] status: {}" + , this.nodeSession.getStoreNode().getAddress(), res.getStatus().getCode()); + } Optional option = null; @@ -182,13 +203,15 @@ final class NotifyingExecutor { log.info("partitions' leader have changed: [partitionId - leaderId] "); nodeManager.notifying( this.graphName, - HgStoreNotice.of(this.nodeSession.getStoreNode().getNodeId(), HgNodeStatus.NOT_PARTITION_LEADER) - .setPartitionLeaders( - res.getPartitionLeadersList() + HgStoreNotice.of(this.nodeSession.getStoreNode().getNodeId(), + HgNodeStatus.NOT_PARTITION_LEADER) + .setPartitionLeaders( + res.getPartitionLeadersList() .stream() .peek((e) -> { - log.info("[{} - {}]", e.getPartitionId(), e.getLeaderId()); - } + log.info("[{} - {}]", e.getPartitionId(), + e.getLeaderId()); + } ) .collect( Collectors.toMap( @@ -196,7 +219,7 @@ final class NotifyingExecutor { PartitionLeader::getLeaderId ) ) - ) + ) ); }; } @@ -210,7 +233,7 @@ final class NotifyingExecutor { nodeManager.notifying( this.graphName, HgStoreNotice.of(this.nodeSession.getStoreNode().getNodeId(), status) - .setPartitionIds(res.getPartitionIdsList()) + .setPartitionIds(res.getPartitionIdsList()) ); }; } @@ -219,7 +242,8 @@ final class NotifyingExecutor { return t -> { nodeManager.notifying( this.graphName, - HgStoreNotice.of(this.nodeSession.getStoreNode().getNodeId(), status, t.getMessage()) + HgStoreNotice.of(this.nodeSession.getStoreNode().getNodeId(), status, + t.getMessage()) ); }; } diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/ScanUtil.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/ScanUtil.java new file mode 100644 index 000000000..66189c848 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/ScanUtil.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; + +import com.baidu.hugegraph.store.grpc.common.Header; +import com.google.protobuf.ByteString; + +/** + * @author zhangyingjie + * @date 2022/10/8 + **/ +public class ScanUtil { + + + public static Header getHeader(HgStoreNodeSession nodeSession) { + return Header.newBuilder().setGraph(nodeSession.getGraphName()).build(); + } + + public static HgOwnerKey toOk(HgOwnerKey key) { + return key == null ? HgStoreClientConst.EMPTY_OWNER_KEY : key; + } + + public static ByteString toBs(byte[] bytes) { + return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES); + } + + public static ByteString getHgOwnerKey(HgOwnerKey ownerKey) { + return toBs(toOk(ownerKey).getKey()); + } + + public static byte[] getQuery(byte[] query) { + return query != null ? query : HgStoreClientConst.EMPTY_BYTES; + } + + public static long getLimit(long limit) { + return limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE : limit; + } +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/SeekAbleIterator.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/SeekAbleIterator.java new file mode 100644 index 000000000..39b5b7c21 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/SeekAbleIterator.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.util.Iterator; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import org.apache.hugegraph.store.HgSeekAble; +import org.apache.hugegraph.store.client.util.HgAssert; + +/** + * @author lynn.bond@hotmail.com on 2022/3/11 + */ +class SeekAbleIterator implements Iterator, HgSeekAble { + private final Iterator iterator; + private final Consumer seeker; + private final Supplier positioner; + + public static SeekAbleIterator of(Iterator iterator, Supplier positioner, + Consumer seeker) { + HgAssert.isArgumentNotNull(iterator, "iterator"); + HgAssert.isArgumentNotNull(positioner, "positioner"); + HgAssert.isArgumentNotNull(seeker, "seeker"); + return new SeekAbleIterator(iterator, positioner, seeker); + } + + private SeekAbleIterator(Iterator iterator, Supplier positioner, + Consumer seeker) { + this.iterator = iterator; + this.positioner = positioner; + this.seeker = seeker; + } + + @Override + public byte[] position() { + return this.positioner.get(); + } + + @Override + public void seek(byte[] position) { + this.seeker.accept(position); + } + + @Override + public boolean hasNext() { + return this.iterator.hasNext(); + } + + @Override + public E next() { + return this.iterator.next(); + } +} \ No newline at end of file diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/SwitchingExecutor.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/SwitchingExecutor.java new file mode 100644 index 000000000..b91fe8c5c --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/SwitchingExecutor.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.util.Optional; +import java.util.function.Supplier; + +import javax.annotation.concurrent.ThreadSafe; + +/** + * @author lynn.bond@hotmail.com on 2021/12/1 + */ +@ThreadSafe +final class SwitchingExecutor { + + static SwitchingExecutor of() { + return new SwitchingExecutor(); + } + + private SwitchingExecutor() { + } + + Optional invoke(Supplier switcher, Supplier trueSupplier, + Supplier falseSupplier) { + Optional option = null; + + if (switcher.get()) { + option = Optional.of(trueSupplier.get()); + } else { + option = Optional.of(falseSupplier.get()); + } + if (option == null) { + option = Optional.empty(); + } + + return option; + } +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/type/HgNodeStatus.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/type/HgNodeStatus.java new file mode 100644 index 000000000..61626dadb --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/type/HgNodeStatus.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.type; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/26 + */ +public enum HgNodeStatus { + + UNKNOWN(0, "UNKNOWN"), + NOT_EXIST(100, "NOT_EXIST"), // Failed to apply for an instance via node-id from NodeManager. + NOT_ONLINE(105, "NOT_ONLINE"), // Failed to connect to Store-Node at the first time. + NOT_WORK(110, "NOT_WORK"), // When a Store-Node to be not work anymore. + + PARTITION_COMMON_FAULT(200, "PARTITION_COMMON_FAULT"), // + NOT_PARTITION_LEADER(205, + "NOT_PARTITION_LEADER") // When a Store-Node is not a specific partition + // leader. + + ; + + private final int status; + private final String name; + + HgNodeStatus(int status, String name) { + this.status = status; + this.name = name; + } + +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/type/HgStoreClientException.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/type/HgStoreClientException.java new file mode 100644 index 000000000..5d0089b80 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/type/HgStoreClientException.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.type; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/27 + */ +public class HgStoreClientException extends RuntimeException { + + public static HgStoreClientException of(String msg) { + return new HgStoreClientException(msg); + } + + public static HgStoreClientException of(String msg, Throwable cause) { + return new HgStoreClientException(msg, cause); + } + + public static HgStoreClientException of(Throwable cause) { + return new HgStoreClientException(cause); + } + + public HgStoreClientException(String msg) { + super(msg); + } + + public HgStoreClientException(Throwable cause) { + super(cause); + } + + public HgStoreClientException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/Base58.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/Base58.java similarity index 77% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/Base58.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/Base58.java index f9801c1b8..c9ec738bb 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/Base58.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/Base58.java @@ -1,7 +1,25 @@ -package com.baidu.hugegraph.store.client.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.util; import java.io.UnsupportedEncodingException; import java.math.BigInteger; +import java.nio.charset.StandardCharsets; /** * Created with IntelliJ IDEA. @@ -12,7 +30,8 @@ import java.math.BigInteger; */ public class Base58 { - public static final char[] ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray(); + public static final char[] ALPHABET = + "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray(); private static final int[] INDEXES = new int[128]; static { @@ -60,11 +79,7 @@ public class Base58 { } byte[] output = copyOfRange(temp, j, temp.length); - try { - return new String(output, "US-ASCII"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); // Cannot happen. - } + return new String(output, StandardCharsets.US_ASCII); } public static byte[] decode(String input) throws IllegalArgumentException { diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/ExecutorPool.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/ExecutorPool.java similarity index 74% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/ExecutorPool.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/ExecutorPool.java index e548b9871..2c67224c0 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/ExecutorPool.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/ExecutorPool.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.client.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.util; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; @@ -24,21 +41,27 @@ public final class ExecutorPool { return new HgDefaultThreadFactory(namePrefix); } - public static ThreadPoolExecutor createExecutor(String name, long keepAliveTime, int coreThreads, int maxThreads) { + public static ThreadPoolExecutor createExecutor(String name, long keepAliveTime, + int coreThreads, int maxThreads) { return new ThreadPoolExecutor(coreThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<>(), - newThreadFactory(name), new ThreadPoolExecutor.CallerRunsPolicy() + newThreadFactory(name), + new ThreadPoolExecutor.CallerRunsPolicy() ); } // // private final static ExecutorService executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, // 10L, TimeUnit.SECONDS, - // new LinkedBlockingQueue<>(), - // new HgDefaultThreadFactory + // new + // LinkedBlockingQueue<>(), + // new + // HgDefaultThreadFactory // ("store-common")); - // private final static ExecutorService grpcExecutor = createExecutor("store-grpc", 60L, 600, 10240); - //// public final static ExecutorService scannerExecutor = createExecutor("scanner", 10l, 200, 1024); + // private final static ExecutorService grpcExecutor = createExecutor("store-grpc", 60L, 600, + // 10240); + //// public final static ExecutorService scannerExecutor = createExecutor("scanner", 10l, + // 200, 1024); // // static { // Thread hook = new Thread(() -> { @@ -101,7 +124,7 @@ public final class ExecutorPool { * The default thread factory, which added threadNamePrefix in construction method. */ static class HgDefaultThreadFactory implements ThreadFactory { - private static AtomicInteger poolNumber = new AtomicInteger(1); + private static final AtomicInteger poolNumber = new AtomicInteger(1); private final AtomicInteger threadNumber = new AtomicInteger(1); private final String namePrefix; diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgAssert.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgAssert.java similarity index 75% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgAssert.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgAssert.java index 723ed86c7..3c8d07234 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgAssert.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgAssert.java @@ -1,7 +1,23 @@ -package com.baidu.hugegraph.store.client.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.util; import java.util.Collection; -import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -80,18 +96,15 @@ public final class HgAssert { } public static boolean isInvalid(byte[] bytes) { - if (bytes == null || bytes.length == 0) return true; - return false; + return bytes == null || bytes.length == 0; } public static boolean isInvalid(Map map) { - if (map == null || map.isEmpty()) return true; - return false; + return map == null || map.isEmpty(); } public static boolean isInvalid(Collection list) { - if (list == null || list.isEmpty()) return true; - return false; + return list == null || list.isEmpty(); } public static boolean isContains(Collection list, T item) { diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgBufferProxy.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgBufferProxy.java similarity index 77% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgBufferProxy.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgBufferProxy.java index f2b114e31..7231f0548 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgBufferProxy.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgBufferProxy.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.client.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.util; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -10,7 +27,7 @@ import java.util.function.Supplier; import javax.annotation.CheckForNull; -import com.baidu.hugegraph.store.client.type.HgStoreClientException; +import org.apache.hugegraph.store.client.type.HgStoreClientException; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgStoreClientConfig.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgStoreClientConfig.java similarity index 78% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgStoreClientConfig.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgStoreClientConfig.java index 43492a90e..c82952e2b 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgStoreClientConfig.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgStoreClientConfig.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.client.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.util; import java.util.PropertyResourceBundle; @@ -25,7 +42,7 @@ public final class HgStoreClientConfig { private Integer netKvScannerHaveNextTimeout = NET_KV_SCANNER_HAVE_NEXT_TIMEOUT; private static PropertyResourceBundle prb = null; - private static String fileName = "hg-store-client"; + private static final String fileName = "hg-store-client"; private static HgStoreClientConfig defaultInstance; @@ -54,17 +71,17 @@ public final class HgStoreClientConfig { PropertiesWrapper wrapper = new PropertiesWrapper(prb); log.info("grpc.timeout.seconds = " - + (config.grpcTimeoutSeconds = wrapper.getInt("grpc.timeout.seconds" + + (config.grpcTimeoutSeconds = wrapper.getInt("grpc.timeout.seconds" , config.grpcTimeoutSeconds)) ); log.info("grpc.max.inbound.message.size = " - + (config.grpcMaxInboundMessageSize = GRPC_DEFAULT_MAX_INBOUND_MESSAGE_SIZE) + + (config.grpcMaxInboundMessageSize = GRPC_DEFAULT_MAX_INBOUND_MESSAGE_SIZE) ); log.info("grpc.max.outbound.message.size = " - + (config.grpcMaxOutboundMessageSize = GRPC_DEFAULT_MAX_OUTBOUND_MESSAGE_SIZE) + + (config.grpcMaxOutboundMessageSize = GRPC_DEFAULT_MAX_OUTBOUND_MESSAGE_SIZE) ); log.info("net.kv.scanner.page.size = " - + (config.netKvScannerPageSize = wrapper.getInt("net.kv.scanner.page.size" + + (config.netKvScannerPageSize = wrapper.getInt("net.kv.scanner.page.size" , config.netKvScannerPageSize)) ); log.info("net.kv.scanner.have.next.timeout = {}", config.netKvScannerHaveNextTimeout); @@ -119,7 +136,7 @@ public final class HgStoreClientConfig { } private static class PropertiesWrapper { - private PropertyResourceBundle prb; + private final PropertyResourceBundle prb; PropertiesWrapper(PropertyResourceBundle prb) { this.prb = prb; @@ -134,7 +151,8 @@ public final class HgStoreClientConfig { try { res = Integer.valueOf(buf); } catch (Throwable t) { - log.error("Failed to parse a int value[ " + buf + " ] of the key[ " + key + " ].", t); + log.error("Failed to parse a int value[ " + buf + " ] of the key[ " + key + " ].", + t); } return res; diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgStoreClientConst.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgStoreClientConst.java new file mode 100644 index 000000000..0a32d50dd --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgStoreClientConst.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.util; + +import java.util.Collections; +import java.util.List; + +import org.apache.hugegraph.store.HgKvStore; +import org.apache.hugegraph.store.HgOwnerKey; + +/** + * @author lynn.bond@hotmail.com + */ +public final class HgStoreClientConst { + public final static String DEFAULT_NODE_CLUSTER_ID = "default-node-cluster"; + + public final static String EMPTY_STRING = ""; + public final static String EMPTY_TABLE = ""; + public final static byte[] EMPTY_BYTES = new byte[0]; + public final static byte[] MAX_BYTES = new byte[]{(byte) 0b11111111}; + public final static List EMPTY_LIST = Collections.EMPTY_LIST; + + public final static byte[] ALL_PARTITION_OWNER = new byte[0]; + // means to dispatch to all partitions. + public final static HgOwnerKey EMPTY_OWNER_KEY = HgOwnerKey.of(EMPTY_BYTES, EMPTY_BYTES); + public final static HgOwnerKey ALL_PARTITION_OWNER_KEY = + HgOwnerKey.of(ALL_PARTITION_OWNER, ALL_PARTITION_OWNER); + + //public final static int SCAN_GTE_BEGIN_LT_END = SCAN_GTE_BEGIN | SCAN_LT_END; + public final static int SCAN_TYPE_RANGE = HgKvStore.SCAN_GTE_BEGIN | HgKvStore.SCAN_LTE_END; + public final static int SCAN_TYPE_ANY = HgKvStore.SCAN_ANY; + public final static int NO_LIMIT = 0; + + public final static int TX_SESSIONS_MAP_CAPACITY = 32; + public static final int NODE_MAX_RETRYING_TIMES = 10; + + +} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgStoreClientUtil.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgStoreClientUtil.java similarity index 70% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgStoreClientUtil.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgStoreClientUtil.java index f5be87330..c53a2a537 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/HgStoreClientUtil.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgStoreClientUtil.java @@ -1,8 +1,21 @@ -package com.baidu.hugegraph.store.client.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.client.type.HgStoreClientException; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.client.util; import java.net.InetAddress; import java.net.UnknownHostException; @@ -10,6 +23,11 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.client.type.HgStoreClientException; + +import lombok.extern.slf4j.Slf4j; + /** * @author lynn.bond@hotmail.com created on 2021/10/14 */ @@ -39,8 +57,7 @@ public final class HgStoreClientUtil { public static boolean isValid(HgOwnerKey key) { if (key == null) return false; if (key.getKey() == null) return false; - if (key.getKey().length == 0) return false; - return true; + return key.getKey().length != 0; } public static String toStr(byte[] b) { @@ -57,11 +74,10 @@ public final class HgStoreClientUtil { public static String toStr(HgOwnerKey ownerKey) { if (ownerKey == null) return ""; - return new StringBuilder() - .append("{ ") - .append("owner: ").append(Arrays.toString(ownerKey.getOwner())) - .append(", key: ").append(toStr(ownerKey.getKey())) - .append(" }").toString(); + return "{ " + + "owner: " + Arrays.toString(ownerKey.getOwner()) + + ", key: " + toStr(ownerKey.getKey()) + + " }"; } public static byte[] toBytes(String str) { @@ -109,12 +125,12 @@ public final class HgStoreClientUtil { } public static byte[] combine(byte[] first, byte[] second) { - if (first == null){ - first=HgStoreClientConst.EMPTY_BYTES; + if (first == null) { + first = HgStoreClientConst.EMPTY_BYTES; } - if(second == null){ - second=HgStoreClientConst.EMPTY_BYTES; + if (second == null) { + second = HgStoreClientConst.EMPTY_BYTES; } byte[] result = new byte[first.length + second.length]; @@ -130,11 +146,12 @@ public final class HgStoreClientUtil { if (stackElements != null) { for (int i = 0; i < stackElements.length; i++) { sb.append(stackElements[i].getClassName()).append(" : ") - .append(stackElements[i].getMethodName()).append(" [ "); + .append(stackElements[i].getMethodName()).append(" [ "); sb.append(stackElements[i].getLineNumber()).append(" ]\n"); } - sb.append("--------------------------------------------------------------------------------------\n"); + sb.append( + "--------------------------------------------------------------------------------------\n"); } log.error(sb.toString()); } diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgUuid.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgUuid.java new file mode 100644 index 000000000..76c77ca53 --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgUuid.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.util; + +import java.nio.ByteBuffer; +import java.util.UUID; + +/** + * @author lynn.bond@hotmail.com + */ +public final class HgUuid { + + private static String encode(UUID uuid) { + ByteBuffer bb = ByteBuffer.wrap(new byte[16]); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + return Base58.encode(bb.array()); + } + + /** + * Get a UUID in Base58 FORM + * + * @return + */ + public static String newUUID() { + return encode(UUID.randomUUID()); + } + +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/MetricX.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/MetricX.java new file mode 100644 index 000000000..8f8c6809e --- /dev/null +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/MetricX.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.util; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author lynn.bond@hotmail.com on 2022/1/29 + */ +public class MetricX { + public static AtomicLong iteratorSum = new AtomicLong(); + public static AtomicLong iteratorCount = new AtomicLong(); + public static AtomicLong iteratorMax = new AtomicLong(); + + public AtomicLong failureCount = new AtomicLong(); + + private long start; + private long end; + + public static MetricX ofStart() { + return new MetricX(System.currentTimeMillis()); + } + + private MetricX(long start) { + this.start = start; + } + + public long start() { + return this.start = System.currentTimeMillis(); + } + + public long end() { + return this.end = System.currentTimeMillis(); + } + + public long past() { + return this.end - this.start; + } + + public static void plusIteratorWait(long nanoSeconds) { + iteratorSum.addAndGet(nanoSeconds); + iteratorCount.getAndIncrement(); + if (iteratorMax.get() < nanoSeconds) { + iteratorMax.set(nanoSeconds); + } + } + + /** + * amount of waiting + * + * @return millisecond + */ + public static long getIteratorWait() { + return iteratorSum.get() / 1_000_000; + } + + /** + * average of waiting + * + * @return millisecond + */ + public static long getIteratorWaitAvg() { + if (iteratorCount.get() == 0) return -1; + return getIteratorWait() / iteratorCount.get(); + } + + /** + * maximum of waiting + * + * @return millisecond + */ + public static long getIteratorWaitMax() { + return iteratorMax.get() / 1_000_000; + } + + public static long getIteratorCount() { + return iteratorCount.get(); + } + + public void countFail() { + this.failureCount.getAndIncrement(); + } + + public long getFailureCount() { + return this.failureCount.get(); + } + +} diff --git a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/PropertyUtil.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/PropertyUtil.java similarity index 66% rename from hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/PropertyUtil.java rename to hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/PropertyUtil.java index 248c46ee3..f6f297a58 100644 --- a/hg-store-client/src/main/java/com/baidu/hugegraph/store/client/util/PropertyUtil.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/PropertyUtil.java @@ -1,14 +1,32 @@ -package com.baidu.hugegraph.store.client.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +package org.apache.hugegraph.store.client.util; import java.security.AccessController; import java.security.PrivilegedAction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class PropertyUtil { private static final Logger LOG = LoggerFactory.getLogger(PropertyUtil.class); + public static String get(String key) { return get(key, null); } @@ -26,7 +44,8 @@ public class PropertyUtil { if (System.getSecurityManager() == null) { value = System.getProperty(key); } else { - value = AccessController.doPrivileged((PrivilegedAction) () -> System.getProperty(key)); + value = AccessController.doPrivileged( + (PrivilegedAction) () -> System.getProperty(key)); } } catch (Exception e) { LOG.error("exception {}", e); diff --git a/hg-store-client/src/main/resources/hg-store-client.properties b/hg-store-client/src/main/resources/hg-store-client.properties index 32b420aaa..aee00cf90 100644 --- a/hg-store-client/src/main/resources/hg-store-client.properties +++ b/hg-store-client/src/main/resources/hg-store-client.properties @@ -1,3 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# #grpc.timeout.seconds=10 #grpc.max.inbound.message.size= #grpc.max.outbound.message.size= diff --git a/hg-store-client/src/main/resources/log4j2.xml b/hg-store-client/src/main/resources/log4j2.xml index f5f852e5a..fab1b79c1 100644 --- a/hg-store-client/src/main/resources/log4j2.xml +++ b/hg-store-client/src/main/resources/log4j2.xml @@ -1,5 +1,22 @@ + + @@ -28,7 +45,7 @@ - + @@ -36,8 +53,8 @@ - - + + @@ -56,7 +73,7 @@ - + @@ -64,8 +81,8 @@ - - + + diff --git a/hg-store-client/src/test/java/com/baidu/hugegraph/store/HgSessionManagerOneRaftFakePDTest.java b/hg-store-client/src/test/java/com/baidu/hugegraph/store/HgSessionManagerOneRaftFakePDTest.java deleted file mode 100644 index d39aa16bc..000000000 --- a/hg-store-client/src/test/java/com/baidu/hugegraph/store/HgSessionManagerOneRaftFakePDTest.java +++ /dev/null @@ -1,439 +0,0 @@ -package com.baidu.hugegraph.store; - -import com.baidu.hugegraph.pd.common.PartitionUtils; -import com.baidu.hugegraph.store.client.*; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import org.junit.Assert; -// import org.junit.BeforeClass; -// import org.junit.Test; - -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -import static com.baidu.hugegraph.store.client.util.HgAssert.isInvalid; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; -import static com.baidu.hugegraph.store.util.HgStoreTestUtil.*; - -/** - * 使用fake-pd，支持raft的单元测试 - */ -public class HgSessionManagerOneRaftFakePDTest { - private static Map leaderMap = new ConcurrentHashMap<>(); - private static Map storeMap = new ConcurrentHashMap<>(); - - private static int partitionCount = 3; // 需要与store的application.yml的fake-pd.partition-count保持一致 - private static String[] storeAddress = { - "127.0.0.1:8500" - }; - - // @BeforeClass - public static void init() { - for (String address : storeAddress) { - storeMap.put((long) address.hashCode(), address); - } - for (int i = 0; i < partitionCount; i++) - leaderMap.put(i, (long) storeMap.keySet().iterator().next()); - - HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); - nodeManager.setNodePartitioner(new HgStoreNodePartitioner() { - @Override - public int partition(HgNodePartitionerBuilder builder, String graphName, byte[] startKey, byte[] endKey) { - int startCode = PartitionUtils.calcHashcode(startKey); - int endCode = PartitionUtils.calcHashcode(endKey); - if (ALL_PARTITION_OWNER == startKey) { - storeMap.forEach((k,v)->{ - builder.add(k, -1); - }); - } else if (endKey == HgStoreClientConst.EMPTY_BYTES || startKey == endKey || Arrays.equals(startKey, endKey)) { - builder.add(leaderMap.get(startCode % partitionCount), startCode); - } else { - Assert.fail("OwnerKey转成HashCode后已经无序了， 按照OwnerKey范围查询没意义"); - builder.add(leaderMap.get(startCode % partitionCount), startCode); - builder.add(leaderMap.get(endCode % partitionCount), endCode); - } - return 0; - } - }); - nodeManager.setNodeProvider(new HgStoreNodeProvider() { - @Override - public HgStoreNode apply(String graphName, Long nodeId) { - System.out.println("HgStoreNodeProvider apply " + graphName + " " + nodeId + " " + storeMap.get(nodeId)); - return nodeManager.getNodeBuilder().setNodeId(nodeId) - .setAddress(storeMap.get(nodeId)).build(); - } - }); - nodeManager.setNodeNotifier(new HgStoreNodeNotifier() { - @Override - public int notice(String graphName, HgStoreNotice storeNotice) { - System.out.println("recv node notifier " + storeNotice); - if ( storeNotice.getPartitionLeaders().size() > 0) { - leaderMap.putAll(storeNotice.getPartitionLeaders()); - System.out.println("leader changed "); - leaderMap.forEach((k, v)->{ - System.out.print(" " + k + " " + v + ","); - }); - System.out.println(); - } - return 0; - } - }); - } - - private static HgStoreSession getStoreSession() { - return HgSessionManager.getInstance().openSession(GRAPH_NAME); - } - private static HgStoreSession getStoreSession(String graph) { - return HgSessionManager.getInstance().openSession(graph); - } - // @Test - public void put_get() { - System.out.println("--- test put & get ---"); - - HgStoreSession session = getStoreSession(); - - HgOwnerKey key = toOwnerKey("FOR-PUT-KEY"); - //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); - byte[] value = toBytes("FOR-PUT-VALUE"); - - Assert.assertTrue(session.put(TABLE_NAME, key, value)); - - Assert.assertEquals(toStr(value), toStr(session.get(TABLE_NAME, key))); - } - - // @Test - public void put_get2() { - System.out.println("--- test put & get 2---"); - - String GRAPH_NAME = "default/hugegraph/g2"; - String TABLE_NAME = "put_get2"; - - HgStoreSession session = getStoreSession(GRAPH_NAME); - - HgOwnerKey key = toOwnerKey("FOR-PUT-KEY"); - //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); - byte[] value = toBytes("FOR-PUT-VALUE"); - - Assert.assertTrue(session.put(TABLE_NAME, key, value)); - - Assert.assertEquals(toStr(value), toStr(session.get(TABLE_NAME, key))); - - HgKvIterator iterator = session.scanIterator(TABLE_NAME); - while (iterator.hasNext()){ - HgKvEntry entry = iterator.next(); - - System.out.println(new String(entry.key())); - Assert.assertEquals(toStr(key.getKey()), toStr(entry.key())); - } - - } - - - // @Test - public void batchGet() { - System.out.println("--- test batchGet ---"); - HgStoreSession session = getStoreSession(); - String keyPrefix = "BATCH-GET-UNIT"; - - Map map = batchPut(session, keyPrefix); - List keyList = map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); - - //printOwner(keyList); - List resList = session.batchGetOwner(TABLE_NAME, keyList); - - Assert.assertFalse(isInvalid(resList)); - Assert.assertEquals(resList.size(), keyList.size()); - - // println(list); - println("--- batch-get result ---"); - Assert.assertTrue((resList.stream() - .map(e -> map.containsKey(toOwnerKey(e.key()))).allMatch(Boolean::booleanValue)) - ); - - } - - // @Test - //CAUTION: ONLY FOR LONG！ - //注意：目前只可以对long类型value进行Merge操作。 - public void merge() { - System.out.println("--- test merge (1+1=2)---"); - HgStoreSession session = getStoreSession(); - String mergeKey = "merge-key"; - HgOwnerKey key = toOwnerKey(mergeKey); - byte[] value = toBytes(1l); - - System.out.println("- put " + mergeKey + ":1 -"); - session.put(TABLE_NAME, key, value); - System.out.println("- merge " + mergeKey + ":1 -"); - session.merge(TABLE_NAME, key, value); - long res = toLong(session.get(TABLE_NAME, key)); - System.out.printf("after merge " + mergeKey + "=%s%n", res); - Assert.assertEquals(2l, res); - - String putStr = "19"; - session.put(TABLE_NAME, key, toBytes(putStr)); - byte[] b1 = session.get(TABLE_NAME, key); - Assert.assertEquals(putStr, toStr(b1)); - } - - // @Test - public void delete() { - System.out.println("--- test delete ---"); - HgStoreSession session = getStoreSession(); - - String delKey = "del-key"; - String delValue = "del-value"; - HgOwnerKey key = toOwnerKey(delKey); - byte[] value = toBytes(delValue); - - println("- put " + delKey + ": " + delValue); - session.put(TABLE_NAME, key, value); - println("- delete " + delKey); - session.delete(TABLE_NAME, key); - value = session.get(TABLE_NAME, key); - println("- get " + delKey + ": " + toStr(value)); - Assert.assertTrue(EMPTY_BYTES.equals(value)); - } - - // @Test - public void deleteSingle() { - System.out.println("--- test deleteSingle ---"); - HgStoreSession session = getStoreSession(); - - String delKey = "del-single-key"; - String delValue = "del-single-value"; - HgOwnerKey key = toOwnerKey(delKey); - byte[] value = toBytes(delValue); - - println("- put [" + delKey + "] = " + delValue); - session.put(TABLE_NAME, key, value); - - value = session.get(TABLE_NAME, key); - println("- before del, get [" + delKey + "] = " + toStr(value)); - Assert.assertEquals(delValue, toStr(value)); - - println("- delete-single : [" + delKey + "]"); - session.deleteSingle(TABLE_NAME, key); - value = session.get(TABLE_NAME, key); - println("- after del, get [" + delKey + "] = " + toStr(value)); - Assert.assertEquals("", toStr(value)); - - } - - // @Test - public void deleteRange() { - println("--- test deleteRange ---"); - HgStoreSession session = getStoreSession(); - - String rangePrefix = "DEL-RANGE-KEY"; - String owner="batch-delete-owner"; - Map map = batchPut(session,TABLE_NAME, rangePrefix, 10,key->{ - return toOwnerKey(owner,key); - }); - - HgOwnerKey startKey = toOwnerKey(owner,rangePrefix + "-00"); - HgOwnerKey endKey = toOwnerKey(owner,rangePrefix + "-05"); - - Assert.assertTrue(session.deleteRange(TABLE_NAME, startKey, endKey)); - - println("- after delete range from [" - + toStr(startKey.getKey()) - + "] to [" - + toStr(endKey.getKey()) + "]"); - - for (int i = 0; i < 10; i++) { - HgOwnerKey key = toOwnerKey(owner,rangePrefix + "-" + padLeftZeros(String.valueOf(i), 2)); - String value = toStr(session.get(TABLE_NAME, key)); - println("- get [" + toStr(key.getKey()) + "] = " + value); - - // TODO: [start,end)? - if (i < 5) { - Assert.assertEquals("", value); - } else { - //println(value); - Assert.assertNotEquals("", value); - } - } - - } - - // @Test - public void deletePrefix() { - System.out.println("--- test deletePrefix ---"); - HgStoreSession session = getStoreSession(); - - String prefixStr = "DEL-PREFIX-KEY"; - String owner="batch-delete-owner"; - Map map = batchPut(session,TABLE_NAME, prefixStr, 10,key->{ - return toOwnerKey(owner,key); - }); - - //printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList())); - - HgOwnerKey prefix = toOwnerKey(owner,prefixStr); - - Assert.assertEquals(10, amountOf(session.scanIterator(TABLE_NAME, prefix))); - session.deletePrefix(TABLE_NAME, prefix); - Assert.assertEquals(0, amountOf(session.scanIterator(TABLE_NAME, prefix))); - - println("- after delete by prefix:[" + prefixStr + "]"); - - for (int i = 0; i < 10; i++) { - HgOwnerKey key = toOwnerKey(owner,prefixStr + toSuffix(i, 2)); - String value = toStr(session.get(TABLE_NAME, key)); - System.out.println("- get [" + toStr(key.getKey()) + "] = " + value); - Assert.assertEquals("", value); - } - - } - - // @Test - public void scanIterator() { - println("--- test scanIterator ---"); - String tableName = "UNIT_SCAN"; - String keyName = "SCAN-ITER"; - HgStoreSession session = getStoreSession(); - batchPut(session, tableName, keyName, 10000); - int count = 0; - int limit = 0; - int max = 99999; - HgKvIterator iterator = null; - - println("-- test 0 element --"); - iterator = session.scanIterator(tableName, toAllPartitionKey("__SCAN-001"), toAllPartitionKey("__SCAN-100"), 0); - Assert.assertFalse(iterator.hasNext()); - try { - iterator.next(); - Assert.assertTrue(false); - } catch (Throwable t) { - println("-- test NoSuchElementException --"); - Assert.assertTrue(NoSuchElementException.class.isInstance(t)); - } - - println("-- test limit 1 to 10 --"); - for (int i = 1; i <= 10; i++) { - println("- limit " + i + " -"); - limit = i; - iterator = session.scanIterator(tableName, toAllPartitionKey(keyName + "-0"), toAllPartitionKey(keyName + "-1"), limit); - count = 0; - while (iterator.hasNext()) { - count++; - HgKvEntry entry = iterator.next(); - println(entry); - } - Assert.assertEquals(limit, count); - } - - println("-- test limit 1 to 10 not enough --"); - for (int i = 1; i <= 10; i++) { - println("- limit " + i + " -"); - limit = i; - iterator = session.scanIterator(tableName, - toAllPartitionKey(keyName + "-00001"), toAllPartitionKey(keyName + "-00005"), limit); - count = 0; - while (iterator.hasNext()) { - count++; - HgKvEntry entry = iterator.next(); - println(entry); - } - if (i <= 5) { - Assert.assertEquals(limit, count); - } else { - Assert.assertEquals(5, count); - } - - } - - println("-- test limit 0 (no limit) --"); - limit = 0; - iterator = session.scanIterator(tableName, toAllPartitionKey(keyName + "-0"), toAllPartitionKey(keyName + "-1"), limit); - - count = 0; - while (iterator.hasNext()) { - count++; - HgKvEntry entry = iterator.next(); - if (count % 1000 == 0) { - println(entry); - } - if (count >= max) break; - } - Assert.assertEquals(10000, count); - - println("-- test scan all --"); - iterator = session.scanIterator(tableName); - count = 0; - while (iterator.hasNext()) { - count++; - HgKvEntry entry = iterator.next(); - if (count % 1000 == 0) { - println(entry); - } - if (count >= max) break; - } - Assert.assertEquals(10000, count); - - println("-- test scan prefix --"); - iterator = session.scanIterator(tableName, toAllPartitionKey(keyName + "-01")); - count = 0; - while (iterator.hasNext()) { - count++; - HgKvEntry entry = iterator.next(); - if (count % 100 == 0) { - println(entry); - } - if (count >= max) break; - } - Assert.assertEquals(1000, count); - } - - - - // @Test - public void truncate() { - println("--- test truncate ---"); - String graph="graph_truncate"; - String tableName = "UNIT_TRUNCATE_1"; - String keyName = "KEY_TRUNCATE"; - - HgStoreSession session = getStoreSession(graph); - batchPut(session, tableName, keyName, 100); - Assert.assertEquals(100, amountOf(session.scanIterator(tableName))); - - String tableName2 = "UNIT_TRUNCATE_2"; - batchPut(session, tableName2, keyName, 100); - Assert.assertEquals(100, amountOf(session.scanIterator(tableName2))); - - - session.truncate(); - Assert.assertEquals(0, amountOf(session.scanIterator(tableName))); - Assert.assertEquals(0, amountOf(session.scanIterator(tableName2))); - } - - //// @Test - public void scanIteratorHuge() { - - /*************** test no limit, with 10 millions **************/ - String tableName = "UNIT_HUGE"; - String keyName = "SCAN-HUGE"; - int amount = 10_000_000; - HgStoreSession session = getStoreSession(); - - if (amountOf(session.scanIterator(tableName, toAllPartitionKey(keyName), 10)) < 10) { - batchPut(session, tableName, keyName, amount); - } - - int count = 0; - HgKvIterator iterator = session.scanIterator(tableName); - - while (iterator.hasNext()) { - count++; - HgKvEntry entry = iterator.next(); - if (count % (amount / 10) == 0) { - println(entry); - } - } - - Assert.assertEquals(amount, count); - } -} \ No newline at end of file diff --git a/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreClientBase.java b/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreClientBase.java deleted file mode 100644 index c145eec15..000000000 --- a/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreClientBase.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.baidu.hugegraph.store.client; - -import com.baidu.hugegraph.pd.client.PDClient; -import com.baidu.hugegraph.pd.client.PDConfig; -import com.baidu.hugegraph.store.HgStoreClient; -import com.baidu.hugegraph.store.HgStoreSession; -import org.junit.Before; - -public class HgStoreClientBase { - - protected HgStoreClient storeClient; - protected PDClient pdClient; - - protected static String Graph_Name = "testGraphName"; - protected static String Table_Name = "testTableName"; - @Before - public void init(){ - storeClient = HgStoreClient.create(PDConfig.of("127.0.0.1:8686") - .setEnableCache(true)); - pdClient = storeClient.getPdClient(); - - HgStoreSession session = storeClient.openSession(Graph_Name); - session.dropTable(Table_Name); - } -} diff --git a/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreNodeSessionTest.java b/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreNodeSessionTest.java deleted file mode 100644 index 652041924..000000000 --- a/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreNodeSessionTest.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.baidu.hugegraph.store.client; - -import com.baidu.hugegraph.store.HgStoreSession; -import org.junit.Assert; -// import org.junit.BeforeClass; -// import org.junit.Test; - -import static com.baidu.hugegraph.store.util.HgStoreTestUtil.*; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/12 - */ -public class HgStoreNodeSessionTest { - private static HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); - private static HgStoreNode node; - - // @BeforeClass - public static void init() { - node = nodeManager.addNode(GRAPH_NAME, nodeManager.getNodeBuilder().setAddress("localhost:9080").build()); - } - - private HgStoreNode getOneNode() { - return node; - } - - private static HgStoreSession getStoreSession() { - return node.openSession(GRAPH_NAME); - } - - // @Test - public void truncate() { - println("--- test truncate ---"); - String tableName = "UNIT_TRUNCATE_1"; - String keyName = "KEY_TRUNCATE"; - - HgStoreSession session = getStoreSession(); - batchPut(session,tableName, keyName, 100); - Assert.assertEquals(100, amountOf(session.scanIterator(tableName))); - - String tableName2 = "UNIT_TRUNCATE_2"; - batchPut(session,tableName2, keyName, 100); - Assert.assertEquals(100, amountOf(session.scanIterator(tableName2))); - - - session.truncate(); - Assert.assertEquals(0, amountOf(session.scanIterator(tableName))); - Assert.assertEquals(0, amountOf(session.scanIterator(tableName2))); - } - -} \ No newline at end of file diff --git a/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreNodeStateTest.java b/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreNodeStateTest.java deleted file mode 100644 index 9854ea89f..000000000 --- a/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreNodeStateTest.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.baidu.hugegraph.store.client; - -// import org.junit.Test; - -import java.util.concurrent.atomic.AtomicInteger; - -import static com.baidu.hugegraph.store.util.HgStoreTestUtil.GRAPH_NAME; -import static com.baidu.hugegraph.store.util.HgStoreTestUtil.println; - - -public class HgStoreNodeStateTest { - private final static HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); - static int nodeNumber = 0; - - static { - registerNode(GRAPH_NAME, Long.valueOf(nodeNumber++), "localhost:9180"); - registerNode(GRAPH_NAME, Long.valueOf(nodeNumber++), "localhost:9280"); - registerNode(GRAPH_NAME, Long.valueOf(nodeNumber++), "localhost:9380"); - } - - private static void registerNode(String graphName, Long nodeId, String address) { - nodeManager.addNode(graphName, nodeManager.getNodeBuilder().setNodeId(nodeId).setAddress(address).build()); - } - - - // @Test - public void isNodeHealthy() { - AtomicInteger count = new AtomicInteger(0); - - for (int i = 0; i < Integer.MAX_VALUE; i++) { - nodeManager.getStoreNodes(GRAPH_NAME) - .stream().map( - node -> { - println(node.getNodeId() + " " + node.getAddress() + "is healthy: " + node.isHealthy()); - return node.isHealthy(); - } - ).count(); - - Thread.yield(); - - try { - Thread.sleep(10000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - -} \ No newline at end of file diff --git a/hg-store-client/src/test/java/com/baidu/hugegraph/store/common/BitsTest.java b/hg-store-client/src/test/java/com/baidu/hugegraph/store/common/BitsTest.java deleted file mode 100644 index 83ed75244..000000000 --- a/hg-store-client/src/test/java/com/baidu/hugegraph/store/common/BitsTest.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.baidu.hugegraph.store.common; - -import com.baidu.hugegraph.store.term.Bits; -import org.junit.Assert; -// import org.junit.Test; - -public class BitsTest { - // @Test - public void test(){ - for(int i = 0; i < Integer.MAX_VALUE; i = i + 10){ - byte[] val = new byte[4]; - Bits.putInt(val, 0, i); - int n = Bits.getInt(val, 0); - Assert.assertEquals(i, n); - } - } -} diff --git a/hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerOneRaftFakePDTest.java b/hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerOneRaftFakePDTest.java new file mode 100644 index 000000000..1c169e413 --- /dev/null +++ b/hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerOneRaftFakePDTest.java @@ -0,0 +1,498 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import static org.apache.hugegraph.store.client.util.HgAssert.isInvalid; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import org.apache.hugegraph.store.client.HgNodePartitionerBuilder; +import org.apache.hugegraph.store.client.HgStoreNode; +import org.apache.hugegraph.store.client.HgStoreNodeManager; +import org.apache.hugegraph.store.client.HgStoreNodeNotifier; +import org.apache.hugegraph.store.client.HgStoreNodePartitioner; +import org.apache.hugegraph.store.client.HgStoreNodeProvider; +import org.apache.hugegraph.store.client.HgStoreNotice; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; + +import com.baidu.hugegraph.pd.common.PartitionUtils; +import com.baidu.hugegraph.store.client.*; + +/** + * 使用fake-pd，支持raft的单元测试 + */ +public class HgSessionManagerOneRaftFakePDTest { + private static final Map leaderMap = new ConcurrentHashMap<>(); + private static final Map storeMap = new ConcurrentHashMap<>(); + + private static final int partitionCount = 3; // 需要与store的application.yml的fake-pd.partition-count保持一致 + private static final String[] storeAddress = { + "127.0.0.1:8500" + }; + + // @BeforeClass + public static void init() { + for (String address : storeAddress) { + storeMap.put((long) address.hashCode(), address); + } + for (int i = 0; i < partitionCount; i++) { + leaderMap.put(i, storeMap.keySet().iterator().next()); + } + + HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); + nodeManager.setNodePartitioner(new HgStoreNodePartitioner() { + @Override + public int partition(HgNodePartitionerBuilder builder, String graphName, + byte[] startKey, byte[] endKey) { + int startCode = PartitionUtils.calcHashcode(startKey); + int endCode = PartitionUtils.calcHashcode(endKey); + if (ALL_PARTITION_OWNER == startKey) { + storeMap.forEach((k, v) -> { + builder.add(k, -1); + }); + } else if (endKey == HgStoreClientConst.EMPTY_BYTES || startKey == endKey || + Arrays.equals(startKey, endKey)) { + builder.add(leaderMap.get(startCode % partitionCount), startCode); + } else { + Assert.fail("OwnerKey转成HashCode后已经无序了， 按照OwnerKey范围查询没意义"); + builder.add(leaderMap.get(startCode % partitionCount), startCode); + builder.add(leaderMap.get(endCode % partitionCount), endCode); + } + return 0; + } + }); + nodeManager.setNodeProvider(new HgStoreNodeProvider() { + @Override + public HgStoreNode apply(String graphName, Long nodeId) { + System.out.println("HgStoreNodeProvider apply " + graphName + " " + nodeId + " " + + storeMap.get(nodeId)); + return nodeManager.getNodeBuilder().setNodeId(nodeId) + .setAddress(storeMap.get(nodeId)).build(); + } + }); + nodeManager.setNodeNotifier(new HgStoreNodeNotifier() { + @Override + public int notice(String graphName, HgStoreNotice storeNotice) { + System.out.println("recv node notifier " + storeNotice); + if (storeNotice.getPartitionLeaders().size() > 0) { + leaderMap.putAll(storeNotice.getPartitionLeaders()); + System.out.println("leader changed "); + leaderMap.forEach((k, v) -> { + System.out.print(" " + k + " " + v + ","); + }); + System.out.println(); + } + return 0; + } + }); + } + + private static HgStoreSession getStoreSession() { + return HgSessionManager.getInstance().openSession(HgStoreTestUtil.GRAPH_NAME); + } + + private static HgStoreSession getStoreSession(String graph) { + return HgSessionManager.getInstance().openSession(graph); + } + + // @Test + public void put_get() { + System.out.println("--- test put & get ---"); + + HgStoreSession session = getStoreSession(); + + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + + Assert.assertTrue(session.put(HgStoreTestUtil.TABLE_NAME, key, value)); + + Assert.assertEquals(HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get( + HgStoreTestUtil.TABLE_NAME, key))); + } + + // @Test + public void put_get2() { + System.out.println("--- test put & get 2---"); + + String GRAPH_NAME = "default/hugegraph/g2"; + String TABLE_NAME = "put_get2"; + + HgStoreSession session = getStoreSession(GRAPH_NAME); + + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + + Assert.assertTrue(session.put(TABLE_NAME, key, value)); + + Assert.assertEquals(HgStoreTestUtil.toStr(value), + HgStoreTestUtil.toStr(session.get(TABLE_NAME, key))); + + HgKvIterator iterator = session.scanIterator(TABLE_NAME); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); + + System.out.println(new String(entry.key())); + Assert.assertEquals(HgStoreTestUtil.toStr(key.getKey()), + HgStoreTestUtil.toStr(entry.key())); + } + + } + + + // @Test + public void batchGet() { + System.out.println("--- test batchGet ---"); + HgStoreSession session = getStoreSession(); + String keyPrefix = "BATCH-GET-UNIT"; + + Map map = HgStoreTestUtil.batchPut(session, keyPrefix); + List keyList = + map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); + + //printOwner(keyList); + List resList = session.batchGetOwner(HgStoreTestUtil.TABLE_NAME, keyList); + + Assert.assertFalse(isInvalid(resList)); + Assert.assertEquals(resList.size(), keyList.size()); + + // println(list); + HgStoreTestUtil.println("--- batch-get result ---"); + Assert.assertTrue((resList.stream() + .map(e -> map.containsKey(HgStoreTestUtil.toOwnerKey(e.key()))) + .allMatch(Boolean::booleanValue)) + ); + + } + + // @Test + //CAUTION: ONLY FOR LONG！ + //注意：目前只可以对long类型value进行Merge操作。 + public void merge() { + System.out.println("--- test merge (1+1=2)---"); + HgStoreSession session = getStoreSession(); + String mergeKey = "merge-key"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey); + byte[] value = HgStoreTestUtil.toBytes(1L); + + System.out.println("- put " + mergeKey + ":1 -"); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + System.out.println("- merge " + mergeKey + ":1 -"); + session.merge(HgStoreTestUtil.TABLE_NAME, key, value); + long res = HgStoreTestUtil.toLong(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.printf("after merge " + mergeKey + "=%s%n", res); + Assert.assertEquals(2L, res); + + String putStr = "19"; + session.put(HgStoreTestUtil.TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr)); + byte[] b1 = session.get(HgStoreTestUtil.TABLE_NAME, key); + Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1)); + } + + // @Test + public void delete() { + System.out.println("--- test delete ---"); + HgStoreSession session = getStoreSession(); + + String delKey = "del-key"; + String delValue = "del-value"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); + + HgStoreTestUtil.println("- put " + delKey + ": " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + HgStoreTestUtil.println("- delete " + delKey); + session.delete(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(EMPTY_BYTES, value); + } + + // @Test + public void deleteSingle() { + System.out.println("--- test deleteSingle ---"); + HgStoreSession session = getStoreSession(); + + String delKey = "del-single-key"; + String delValue = "del-single-value"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); + + HgStoreTestUtil.println("- put [" + delKey + "] = " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- before del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value)); + + HgStoreTestUtil.println("- delete-single : [" + delKey + "]"); + session.deleteSingle(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals("", HgStoreTestUtil.toStr(value)); + + } + + // @Test + public void deleteRange() { + HgStoreTestUtil.println("--- test deleteRange ---"); + HgStoreSession session = getStoreSession(); + + String rangePrefix = "DEL-RANGE-KEY"; + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, rangePrefix, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + + HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00"); + HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05"); + + Assert.assertTrue(session.deleteRange(HgStoreTestUtil.TABLE_NAME, startKey, endKey)); + + HgStoreTestUtil.println("- after delete range from [" + + HgStoreTestUtil.toStr(startKey.getKey()) + + "] to [" + + HgStoreTestUtil.toStr(endKey.getKey()) + "]"); + + for (int i = 0; i < 10; i++) { + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" + + HgStoreTestUtil.padLeftZeros( + String.valueOf(i), 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + HgStoreTestUtil.println( + "- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); + + // TODO: [start,end)? + if (i < 5) { + Assert.assertEquals("", value); + } else { + //println(value); + Assert.assertNotEquals("", value); + } + } + + } + + // @Test + public void deletePrefix() { + System.out.println("--- test deletePrefix ---"); + HgStoreSession session = getStoreSession(); + + String prefixStr = "DEL-PREFIX-KEY"; + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, prefixStr, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + + //printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList())); + + HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr); + + Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + session.deletePrefix(HgStoreTestUtil.TABLE_NAME, prefix); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + + HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]"); + + for (int i = 0; i < 10; i++) { + HgOwnerKey key = + HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.println("- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); + Assert.assertEquals("", value); + } + + } + + // @Test + public void scanIterator() { + HgStoreTestUtil.println("--- test scanIterator ---"); + String tableName = "UNIT_SCAN"; + String keyName = "SCAN-ITER"; + HgStoreSession session = getStoreSession(); + HgStoreTestUtil.batchPut(session, tableName, keyName, 10000); + int count = 0; + int limit = 0; + int max = 99999; + HgKvIterator iterator = null; + + HgStoreTestUtil.println("-- test 0 element --"); + iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"), + HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0); + Assert.assertFalse(iterator.hasNext()); + try { + iterator.next(); + Assert.fail(); + } catch (Throwable t) { + HgStoreTestUtil.println("-- test NoSuchElementException --"); + Assert.assertTrue(t instanceof NoSuchElementException); + } + + HgStoreTestUtil.println("-- test limit 1 to 10 --"); + for (int i = 1; i <= 10; i++) { + HgStoreTestUtil.println("- limit " + i + " -"); + limit = i; + iterator = session.scanIterator(tableName, + HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), + limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.println(entry); + } + Assert.assertEquals(limit, count); + } + + HgStoreTestUtil.println("-- test limit 1 to 10 not enough --"); + for (int i = 1; i <= 10; i++) { + HgStoreTestUtil.println("- limit " + i + " -"); + limit = i; + iterator = session.scanIterator(tableName, + HgStoreTestUtil.toAllPartitionKey(keyName + "-00001"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-00005"), + limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.println(entry); + } + if (i <= 5) { + Assert.assertEquals(limit, count); + } else { + Assert.assertEquals(5, count); + } + + } + + HgStoreTestUtil.println("-- test limit 0 (no limit) --"); + limit = 0; + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), limit); + + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 1000 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(10000, count); + + HgStoreTestUtil.println("-- test scan all --"); + iterator = session.scanIterator(tableName); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 1000 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(10000, count); + + HgStoreTestUtil.println("-- test scan prefix --"); + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-01")); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 100 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(1000, count); + } + + + // @Test + public void truncate() { + HgStoreTestUtil.println("--- test truncate ---"); + String graph = "graph_truncate"; + String tableName = "UNIT_TRUNCATE_1"; + String keyName = "KEY_TRUNCATE"; + + HgStoreSession session = getStoreSession(graph); + HgStoreTestUtil.batchPut(session, tableName, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + String tableName2 = "UNIT_TRUNCATE_2"; + HgStoreTestUtil.batchPut(session, tableName2, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); + + + session.truncate(); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); + } + + //// @Test + public void scanIteratorHuge() { + + /*************** test no limit, with 10 millions **************/ + String tableName = "UNIT_HUGE"; + String keyName = "SCAN-HUGE"; + int amount = 10_000_000; + HgStoreSession session = getStoreSession(); + + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) < + 10) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); + } + + int count = 0; + HgKvIterator iterator = session.scanIterator(tableName); + + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % (amount / 10) == 0) { + HgStoreTestUtil.println(entry); + } + } + + Assert.assertEquals(amount, count); + } +} \ No newline at end of file diff --git a/hg-store-client/src/test/java/com/baidu/hugegraph/store/HgSessionManagerRaftFakePDTest.java b/hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerRaftFakePDTest.java similarity index 56% rename from hg-store-client/src/test/java/com/baidu/hugegraph/store/HgSessionManagerRaftFakePDTest.java rename to hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerRaftFakePDTest.java index 51c9418bb..988a9a9ac 100644 --- a/hg-store-client/src/test/java/com/baidu/hugegraph/store/HgSessionManagerRaftFakePDTest.java +++ b/hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerRaftFakePDTest.java @@ -1,32 +1,59 @@ -package com.baidu.hugegraph.store; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.pd.common.PartitionUtils; -import com.baidu.hugegraph.store.client.*; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import org.junit.Assert; -// import org.junit.BeforeClass; -// import org.junit.Test; +package org.apache.hugegraph.store; -import java.util.*; +import static org.apache.hugegraph.store.client.util.HgAssert.isInvalid; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; -import static com.baidu.hugegraph.store.client.util.HgAssert.isInvalid; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; -import static com.baidu.hugegraph.store.util.HgStoreTestUtil.*; +import org.apache.hugegraph.store.client.HgNodePartitionerBuilder; +import org.apache.hugegraph.store.client.HgStoreNode; +import org.apache.hugegraph.store.client.HgStoreNodeManager; +import org.apache.hugegraph.store.client.HgStoreNodeNotifier; +import org.apache.hugegraph.store.client.HgStoreNodePartitioner; +import org.apache.hugegraph.store.client.HgStoreNodeProvider; +import org.apache.hugegraph.store.client.HgStoreNotice; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; + +import com.baidu.hugegraph.pd.common.PartitionUtils; +import com.baidu.hugegraph.store.client.*; /** * 使用fake-pd，支持raft的单元测试 */ public class HgSessionManagerRaftFakePDTest { - private static Map leaderMap = new ConcurrentHashMap<>(); - private static Map storeMap = new ConcurrentHashMap<>(); + private static final Map leaderMap = new ConcurrentHashMap<>(); + private static final Map storeMap = new ConcurrentHashMap<>(); - private static int partitionCount = 3; // 需要与store的application.yml的fake-pd.partition-count保持一致 - private static String[] storeAddress = { // 需要与store的application.yml的fake-pd.store-list保持一致 - "127.0.0.1:8501","127.0.0.1:8502","127.0.0.1:8503" + private static final int partitionCount = 3; // 需要与store的application.yml的fake-pd.partition-count保持一致 + private static final String[] storeAddress = { // 需要与store的application.yml的fake-pd.store-list保持一致 + "127.0.0.1:8501", "127.0.0.1:8502", "127.0.0.1:8503" }; + /*private static String[] storeAddress = { "127.0.0.1:9080" };*/ @@ -35,20 +62,23 @@ public class HgSessionManagerRaftFakePDTest { for (String address : storeAddress) { storeMap.put((long) address.hashCode(), address); } - for (int i = 0; i < partitionCount; i++) - leaderMap.put(i, (long) storeMap.keySet().iterator().next()); + for (int i = 0; i < partitionCount; i++) { + leaderMap.put(i, storeMap.keySet().iterator().next()); + } HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); nodeManager.setNodePartitioner(new HgStoreNodePartitioner() { @Override - public int partition(HgNodePartitionerBuilder builder, String graphName, byte[] startKey, byte[] endKey) { + public int partition(HgNodePartitionerBuilder builder, String graphName, + byte[] startKey, byte[] endKey) { int startCode = PartitionUtils.calcHashcode(startKey); int endCode = PartitionUtils.calcHashcode(endKey); if (ALL_PARTITION_OWNER == startKey) { - storeMap.forEach((k,v)->{ + storeMap.forEach((k, v) -> { builder.add(k, -1); }); - } else if (endKey == HgStoreClientConst.EMPTY_BYTES || startKey == endKey || Arrays.equals(startKey, endKey)) { + } else if (endKey == HgStoreClientConst.EMPTY_BYTES || startKey == endKey || + Arrays.equals(startKey, endKey)) { builder.add(leaderMap.get(startCode % partitionCount), startCode); } else { Assert.fail("OwnerKey转成HashCode后已经无序了， 按照OwnerKey范围查询没意义"); @@ -61,19 +91,20 @@ public class HgSessionManagerRaftFakePDTest { nodeManager.setNodeProvider(new HgStoreNodeProvider() { @Override public HgStoreNode apply(String graphName, Long nodeId) { - System.out.println("HgStoreNodeProvider apply " + graphName + " " + nodeId + " " + storeMap.get(nodeId)); + System.out.println("HgStoreNodeProvider apply " + graphName + " " + nodeId + " " + + storeMap.get(nodeId)); return nodeManager.getNodeBuilder().setNodeId(nodeId) - .setAddress(storeMap.get(nodeId)).build(); + .setAddress(storeMap.get(nodeId)).build(); } }); nodeManager.setNodeNotifier(new HgStoreNodeNotifier() { @Override public int notice(String graphName, HgStoreNotice storeNotice) { System.out.println("recv node notifier " + storeNotice); - if ( storeNotice.getPartitionLeaders().size() > 0) { + if (storeNotice.getPartitionLeaders().size() > 0) { leaderMap.putAll(storeNotice.getPartitionLeaders()); System.out.println("leader changed "); - leaderMap.forEach((k, v)->{ + leaderMap.forEach((k, v) -> { System.out.print(" " + k + " " + v + ","); }); System.out.println(); @@ -84,24 +115,27 @@ public class HgSessionManagerRaftFakePDTest { } private static HgStoreSession getStoreSession() { - return HgSessionManager.getInstance().openSession(GRAPH_NAME); + return HgSessionManager.getInstance().openSession(HgStoreTestUtil.GRAPH_NAME); } + private static HgStoreSession getStoreSession(String graph) { return HgSessionManager.getInstance().openSession(graph); } + // @Test public void put_get() { System.out.println("--- test put & get ---"); HgStoreSession session = getStoreSession(); - HgOwnerKey key = toOwnerKey("FOR-PUT-KEY"); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); - byte[] value = toBytes("FOR-PUT-VALUE"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); - Assert.assertTrue(session.put(TABLE_NAME, key, value)); + Assert.assertTrue(session.put(HgStoreTestUtil.TABLE_NAME, key, value)); - Assert.assertEquals(toStr(value), toStr(session.get(TABLE_NAME, key))); + Assert.assertEquals(HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get( + HgStoreTestUtil.TABLE_NAME, key))); } // // @Test @@ -129,11 +163,12 @@ public class HgSessionManagerRaftFakePDTest { // HgStoreSession session = getStoreSession(); // Assert.assertTrue(session.batchPutOwner(entries)); // System.out.println("put ok"); -// Assert.assertEquals(amount, amountOf(session.scanIterator(TABLE_NAME, toAllPartitionKey(keyPrefix)))); +// Assert.assertEquals(amount, amountOf(session.scanIterator(TABLE_NAME, toAllPartitionKey +// (keyPrefix)))); // // } - // // @Test + // // @Test // // public void batchPutWithoutStop() { // final int loops = 6 * 10; @@ -177,19 +212,21 @@ public class HgSessionManagerRaftFakePDTest { HgStoreSession session = getStoreSession(); String keyPrefix = "BATCH-GET-UNIT"; - Map map = batchPut(session, keyPrefix); - List keyList = map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); + Map map = HgStoreTestUtil.batchPut(session, keyPrefix); + List keyList = + map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); //printOwner(keyList); - List resList = session.batchGetOwner(TABLE_NAME, keyList); + List resList = session.batchGetOwner(HgStoreTestUtil.TABLE_NAME, keyList); Assert.assertFalse(isInvalid(resList)); Assert.assertEquals(resList.size(), keyList.size()); - // println(list); - println("--- batch-get result ---"); + // println(list); + HgStoreTestUtil.println("--- batch-get result ---"); Assert.assertTrue((resList.stream() - .map(e -> map.containsKey(toOwnerKey(e.key()))).allMatch(Boolean::booleanValue)) + .map(e -> map.containsKey(HgStoreTestUtil.toOwnerKey(e.key()))) + .allMatch(Boolean::booleanValue)) ); } @@ -201,21 +238,21 @@ public class HgSessionManagerRaftFakePDTest { System.out.println("--- test merge (1+1=2)---"); HgStoreSession session = getStoreSession(); String mergeKey = "merge-key"; - HgOwnerKey key = toOwnerKey(mergeKey); - byte[] value = toBytes(1l); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey); + byte[] value = HgStoreTestUtil.toBytes(1L); System.out.println("- put " + mergeKey + ":1 -"); - session.put(TABLE_NAME, key, value); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); System.out.println("- merge " + mergeKey + ":1 -"); - session.merge(TABLE_NAME, key, value); - long res = toLong(session.get(TABLE_NAME, key)); + session.merge(HgStoreTestUtil.TABLE_NAME, key, value); + long res = HgStoreTestUtil.toLong(session.get(HgStoreTestUtil.TABLE_NAME, key)); System.out.printf("after merge " + mergeKey + "=%s%n", res); - Assert.assertEquals(2l, res); + Assert.assertEquals(2L, res); String putStr = "19"; - session.put(TABLE_NAME, key, toBytes(putStr)); - byte[] b1 = session.get(TABLE_NAME, key); - Assert.assertEquals(putStr, toStr(b1)); + session.put(HgStoreTestUtil.TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr)); + byte[] b1 = session.get(HgStoreTestUtil.TABLE_NAME, key); + Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1)); } // @Test @@ -225,16 +262,16 @@ public class HgSessionManagerRaftFakePDTest { String delKey = "del-key"; String delValue = "del-value"; - HgOwnerKey key = toOwnerKey(delKey); - byte[] value = toBytes(delValue); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); - println("- put " + delKey + ": " + delValue); - session.put(TABLE_NAME, key, value); - println("- delete " + delKey); - session.delete(TABLE_NAME, key); - value = session.get(TABLE_NAME, key); - println("- get " + delKey + ": " + toStr(value)); - Assert.assertTrue(EMPTY_BYTES.equals(value)); + HgStoreTestUtil.println("- put " + delKey + ": " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + HgStoreTestUtil.println("- delete " + delKey); + session.delete(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(EMPTY_BYTES, value); } // @Test @@ -244,49 +281,56 @@ public class HgSessionManagerRaftFakePDTest { String delKey = "del-single-key"; String delValue = "del-single-value"; - HgOwnerKey key = toOwnerKey(delKey); - byte[] value = toBytes(delValue); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); - println("- put [" + delKey + "] = " + delValue); - session.put(TABLE_NAME, key, value); + HgStoreTestUtil.println("- put [" + delKey + "] = " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); - value = session.get(TABLE_NAME, key); - println("- before del, get [" + delKey + "] = " + toStr(value)); - Assert.assertEquals(delValue, toStr(value)); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- before del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value)); - println("- delete-single : [" + delKey + "]"); - session.deleteSingle(TABLE_NAME, key); - value = session.get(TABLE_NAME, key); - println("- after del, get [" + delKey + "] = " + toStr(value)); - Assert.assertEquals("", toStr(value)); + HgStoreTestUtil.println("- delete-single : [" + delKey + "]"); + session.deleteSingle(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals("", HgStoreTestUtil.toStr(value)); } // @Test public void deleteRange() { - println("--- test deleteRange ---"); + HgStoreTestUtil.println("--- test deleteRange ---"); HgStoreSession session = getStoreSession(); String rangePrefix = "DEL-RANGE-KEY"; - String owner="batch-delete-owner"; - Map map = batchPut(session,TABLE_NAME, rangePrefix, 10,key->{ - return toOwnerKey(owner,key); - }); + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, rangePrefix, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); - HgOwnerKey startKey = toOwnerKey(owner,rangePrefix + "-00"); - HgOwnerKey endKey = toOwnerKey(owner,rangePrefix + "-05"); + HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00"); + HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05"); - Assert.assertTrue(session.deleteRange(TABLE_NAME, startKey, endKey)); + Assert.assertTrue(session.deleteRange(HgStoreTestUtil.TABLE_NAME, startKey, endKey)); - println("- after delete range from [" - + toStr(startKey.getKey()) - + "] to [" - + toStr(endKey.getKey()) + "]"); + HgStoreTestUtil.println("- after delete range from [" + + HgStoreTestUtil.toStr(startKey.getKey()) + + "] to [" + + HgStoreTestUtil.toStr(endKey.getKey()) + "]"); for (int i = 0; i < 10; i++) { - HgOwnerKey key = toOwnerKey(owner,rangePrefix + "-" + padLeftZeros(String.valueOf(i), 2)); - String value = toStr(session.get(TABLE_NAME, key)); - println("- get [" + toStr(key.getKey()) + "] = " + value); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" + + HgStoreTestUtil.padLeftZeros( + String.valueOf(i), 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + HgStoreTestUtil.println( + "- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); // TODO: [start,end)? if (i < 5) { @@ -305,25 +349,30 @@ public class HgSessionManagerRaftFakePDTest { HgStoreSession session = getStoreSession(); String prefixStr = "DEL-PREFIX-KEY"; - String owner="batch-delete-owner"; - Map map = batchPut(session,TABLE_NAME, prefixStr, 10,key->{ - return toOwnerKey(owner,key); - }); + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, prefixStr, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); //printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList())); - HgOwnerKey prefix = toOwnerKey(owner,prefixStr); + HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr); - Assert.assertEquals(10, amountOf(session.scanIterator(TABLE_NAME, prefix))); - session.deletePrefix(TABLE_NAME, prefix); - Assert.assertEquals(0, amountOf(session.scanIterator(TABLE_NAME, prefix))); + Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + session.deletePrefix(HgStoreTestUtil.TABLE_NAME, prefix); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); - println("- after delete by prefix:[" + prefixStr + "]"); + HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]"); for (int i = 0; i < 10; i++) { - HgOwnerKey key = toOwnerKey(owner,prefixStr + toSuffix(i, 2)); - String value = toStr(session.get(TABLE_NAME, key)); - System.out.println("- get [" + toStr(key.getKey()) + "] = " + value); + HgOwnerKey key = + HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.println("- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); Assert.assertEquals("", value); } @@ -456,52 +505,58 @@ public class HgSessionManagerRaftFakePDTest { // @Test public void scanIterator() { - println("--- test scanIterator ---"); + HgStoreTestUtil.println("--- test scanIterator ---"); String tableName = "UNIT_SCAN"; String keyName = "SCAN-ITER"; HgStoreSession session = getStoreSession(); - batchPut(session, tableName, keyName, 10000); + HgStoreTestUtil.batchPut(session, tableName, keyName, 10000); int count = 0; int limit = 0; int max = 99999; HgKvIterator iterator = null; - println("-- test 0 element --"); - iterator = session.scanIterator(tableName, toAllPartitionKey("__SCAN-001"), toAllPartitionKey("__SCAN-100"), 0); + HgStoreTestUtil.println("-- test 0 element --"); + iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"), + HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0); Assert.assertFalse(iterator.hasNext()); try { iterator.next(); - Assert.assertTrue(false); + Assert.fail(); } catch (Throwable t) { - println("-- test NoSuchElementException --"); - Assert.assertTrue(NoSuchElementException.class.isInstance(t)); + HgStoreTestUtil.println("-- test NoSuchElementException --"); + Assert.assertTrue(t instanceof NoSuchElementException); } - println("-- test limit 1 to 10 --"); + HgStoreTestUtil.println("-- test limit 1 to 10 --"); for (int i = 1; i <= 10; i++) { - println("- limit " + i + " -"); + HgStoreTestUtil.println("- limit " + i + " -"); limit = i; - iterator = session.scanIterator(tableName, toAllPartitionKey(keyName + "-0"), toAllPartitionKey(keyName + "-1"), limit); + iterator = session.scanIterator(tableName, + HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), + limit); count = 0; while (iterator.hasNext()) { count++; HgKvEntry entry = iterator.next(); - println(entry); + HgStoreTestUtil.println(entry); } Assert.assertEquals(limit, count); } - println("-- test limit 1 to 10 not enough --"); + HgStoreTestUtil.println("-- test limit 1 to 10 not enough --"); for (int i = 1; i <= 10; i++) { - println("- limit " + i + " -"); + HgStoreTestUtil.println("- limit " + i + " -"); limit = i; iterator = session.scanIterator(tableName, - toAllPartitionKey(keyName + "-00001"), toAllPartitionKey(keyName + "-00005"), limit); + HgStoreTestUtil.toAllPartitionKey(keyName + "-00001"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-00005"), + limit); count = 0; while (iterator.hasNext()) { count++; HgKvEntry entry = iterator.next(); - println(entry); + HgStoreTestUtil.println(entry); } if (i <= 5) { Assert.assertEquals(limit, count); @@ -511,42 +566,45 @@ public class HgSessionManagerRaftFakePDTest { } - println("-- test limit 0 (no limit) --"); + HgStoreTestUtil.println("-- test limit 0 (no limit) --"); limit = 0; - iterator = session.scanIterator(tableName, toAllPartitionKey(keyName + "-0"), toAllPartitionKey(keyName + "-1"), limit); + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), limit); count = 0; while (iterator.hasNext()) { count++; HgKvEntry entry = iterator.next(); if (count % 1000 == 0) { - println(entry); + HgStoreTestUtil.println(entry); } if (count >= max) break; } Assert.assertEquals(10000, count); - println("-- test scan all --"); + HgStoreTestUtil.println("-- test scan all --"); iterator = session.scanIterator(tableName); count = 0; while (iterator.hasNext()) { count++; HgKvEntry entry = iterator.next(); if (count % 1000 == 0) { - println(entry); + HgStoreTestUtil.println(entry); } if (count >= max) break; } Assert.assertEquals(10000, count); - println("-- test scan prefix --"); - iterator = session.scanIterator(tableName, toAllPartitionKey(keyName + "-01")); + HgStoreTestUtil.println("-- test scan prefix --"); + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-01")); count = 0; while (iterator.hasNext()) { count++; HgKvEntry entry = iterator.next(); if (count % 100 == 0) { - println(entry); + HgStoreTestUtil.println(entry); } if (count >= max) break; } @@ -554,26 +612,25 @@ public class HgSessionManagerRaftFakePDTest { } - // @Test public void truncate() { - println("--- test truncate ---"); - String graph="graph_truncate"; + HgStoreTestUtil.println("--- test truncate ---"); + String graph = "graph_truncate"; String tableName = "UNIT_TRUNCATE_1"; String keyName = "KEY_TRUNCATE"; HgStoreSession session = getStoreSession(graph); - batchPut(session, tableName, keyName, 100); - Assert.assertEquals(100, amountOf(session.scanIterator(tableName))); + HgStoreTestUtil.batchPut(session, tableName, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); String tableName2 = "UNIT_TRUNCATE_2"; - batchPut(session, tableName2, keyName, 100); - Assert.assertEquals(100, amountOf(session.scanIterator(tableName2))); + HgStoreTestUtil.batchPut(session, tableName2, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); session.truncate(); - Assert.assertEquals(0, amountOf(session.scanIterator(tableName))); - Assert.assertEquals(0, amountOf(session.scanIterator(tableName2))); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); } //// @Test @@ -585,8 +642,10 @@ public class HgSessionManagerRaftFakePDTest { int amount = 10_000_000; HgStoreSession session = getStoreSession(); - if (amountOf(session.scanIterator(tableName, toAllPartitionKey(keyName), 10)) < 10) { - batchPut(session, tableName, keyName, amount); + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) < + 10) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); } int count = 0; @@ -596,7 +655,7 @@ public class HgSessionManagerRaftFakePDTest { count++; HgKvEntry entry = iterator.next(); if (count % (amount / 10) == 0) { - println(entry); + HgStoreTestUtil.println(entry); } } diff --git a/hg-store-client/src/test/java/com/baidu/hugegraph/store/HgSessionManagerRaftPDTest.java b/hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerRaftPDTest.java similarity index 59% rename from hg-store-client/src/test/java/com/baidu/hugegraph/store/HgSessionManagerRaftPDTest.java rename to hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerRaftPDTest.java index 497a13465..7a8c6c766 100644 --- a/hg-store-client/src/test/java/com/baidu/hugegraph/store/HgSessionManagerRaftPDTest.java +++ b/hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerRaftPDTest.java @@ -1,30 +1,55 @@ -package com.baidu.hugegraph.store; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.stream.Collectors; + +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.store.client.grpc.KvCloseableIterator; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.MetricX; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; +import org.junit.Test; import com.baidu.hugegraph.pd.client.PDClient; import com.baidu.hugegraph.pd.client.PDConfig; -import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator; -import com.baidu.hugegraph.store.client.util.HgStoreClientConfig; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import com.baidu.hugegraph.store.client.util.MetricX; -import com.baidu.hugegraph.store.grpc.common.Kv; import com.baidu.hugegraph.store.grpc.common.ScanOrderType; -import com.baidu.hugegraph.store.util.HgStoreTestUtil; + import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -// import org.junit.BeforeClass; -// import org.junit.Test; - -import java.io.*; -import java.text.SimpleDateFormat; -import java.util.*; -import java.util.stream.Collectors; - -import static com.baidu.hugegraph.store.client.util.HgAssert.isInvalid; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; -import static com.baidu.hugegraph.store.util.HgStoreTestUtil.*; /** @@ -42,6 +67,7 @@ public class HgSessionManagerRaftPDTest { public final static String TABLE_NAME_3 = "table3"; public static HgStoreClient storeClient; + // @BeforeClass public static void init() { pdClient = PDClient.create(PDConfig.of(pdAddress).setEnableCache(true)); @@ -49,8 +75,9 @@ public class HgSessionManagerRaftPDTest { } private static HgStoreSession getStoreSession() { - return storeClient.openSession(GRAPH_NAME); + return storeClient.openSession(HgStoreTestUtil.GRAPH_NAME); } + private static HgStoreSession getStoreSession(String graph) { return storeClient.openSession(graph); } @@ -59,18 +86,18 @@ public class HgSessionManagerRaftPDTest { public void put_get() { System.out.println("--- test put & get ---"); - HgStoreSession session = getStoreSession(GRAPH_NAME); + HgStoreSession session = getStoreSession(HgStoreTestUtil.GRAPH_NAME); - HgOwnerKey key = toOwnerKey("FOR-PUT-KEY"); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); - byte[] value = toBytes("FOR-PUT-VALUE"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); // Assert.assertTrue(session.put(TABLE_NAME, key, value)); // Assert.assertEquals(toStr(value), toStr(session.get(TABLE_NAME, key))); long stime = System.currentTimeMillis(); - batchPut(session, TABLE_NAME, "BATCH-PUT-TEST", 30000); + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, "BATCH-PUT-TEST", 30000); System.out.println("Time is " + (System.currentTimeMillis() - stime)); } @@ -78,23 +105,24 @@ public class HgSessionManagerRaftPDTest { System.out.println("--- test put & get ---"); HgStoreSession session = getStoreSession("testGraph"); - HgOwnerKey key = toOwnerKey("FOR-PUT-KEY"); - byte[] value = toBytes("FOR-PUT-VALUE"); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); long stime = System.currentTimeMillis(); - batchPut(session, "testTable", "BATCH-PUT-TEST", 30000); + HgStoreTestUtil.batchPut(session, "testTable", "BATCH-PUT-TEST", 30000); System.out.println("Time is " + (System.currentTimeMillis() - stime)); } + // @Test public void scanPartition() { - println("--- test scanPartition ---"); + HgStoreTestUtil.println("--- test scanPartition ---"); HgStoreSession session = getStoreSession(); // System.out.println(amountOf(session.scanIterator(TABLE_NAME))); - Iterator iterator = session.scanIterator(TABLE_NAME, 0, 65535, - HgKvStore.SCAN_HASHCODE, EMPTY_BYTES); - System.out.println(amountOf(iterator)); + Iterator iterator = session.scanIterator(HgStoreTestUtil.TABLE_NAME, 0, 65535, + HgKvStore.SCAN_HASHCODE, EMPTY_BYTES); + System.out.println(HgStoreTestUtil.amountOf(iterator)); } // @Test @@ -102,7 +130,7 @@ public class HgSessionManagerRaftPDTest { System.out.println("--- test check ---"); HgStoreSession session = getStoreSession(); - HgKvIterator iterator = session.scanIterator(TABLE_NAME); + HgKvIterator iterator = session.scanIterator(HgStoreTestUtil.TABLE_NAME); int count = 0; while (iterator.hasNext()) { iterator.next(); @@ -110,6 +138,7 @@ public class HgSessionManagerRaftPDTest { } System.out.println(count); } + // @Test public void put_get_unique() { System.out.println("--- test put & get ---"); @@ -121,13 +150,14 @@ public class HgSessionManagerRaftPDTest { SimpleDateFormat formatter = new SimpleDateFormat("HHmmss"); String timestamp = formatter.format(date); - HgOwnerKey key = toOwnerKey("FOR-PUT-KEY-" + timestamp); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY-" + timestamp); //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); - byte[] value = toBytes("FOR-PUT-VALUE"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); - Assert.assertTrue(session.put(TABLE_NAME, key, value)); + Assert.assertTrue(session.put(HgStoreTestUtil.TABLE_NAME, key, value)); - Assert.assertEquals(toStr(value), toStr(session.get(TABLE_NAME, key))); + Assert.assertEquals(HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get( + HgStoreTestUtil.TABLE_NAME, key))); } @@ -137,7 +167,7 @@ public class HgSessionManagerRaftPDTest { HgStoreSession session = getStoreSession(); String keyPrefix = "BATCH-GET-UNIT"; - Map map = batchPut(session, keyPrefix, 1000); + Map map = HgStoreTestUtil.batchPut(session, keyPrefix, 1000); File outputFile = new File("tmp/batch_put_list"); if (outputFile.exists()) { @@ -148,12 +178,13 @@ public class HgSessionManagerRaftPDTest { ObjectOutputStream oo = new ObjectOutputStream(new FileOutputStream(outputFile)); oo.writeObject(map); oo.close(); - System.out.printf("%d entries have be put into graph %s\n", map.size(), GRAPH_NAME); + System.out.printf("%d entries have be put into graph %s\n", map.size(), + HgStoreTestUtil.GRAPH_NAME); int count = 0; HgKvIterator iterator = null; - iterator = session.scanIterator(TABLE_NAME); + iterator = session.scanIterator(HgStoreTestUtil.TABLE_NAME); while (iterator.hasNext()) { iterator.next(); count++; @@ -172,11 +203,13 @@ public class HgSessionManagerRaftPDTest { System.out.printf("%d entries get from %s\n", map.size(), outputFile.getPath()); HgStoreSession session = getStoreSession(); - List keyList = map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); - List resList = session.batchGetOwner(TABLE_NAME, keyList); + List keyList = + map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); + List resList = session.batchGetOwner(HgStoreTestUtil.TABLE_NAME, keyList); Assert.assertTrue((resList.stream() - .map(e -> map.containsKey(toOwnerKey(e.key()))).allMatch(Boolean::booleanValue)) + .map(e -> map.containsKey(HgStoreTestUtil.toOwnerKey(e.key()))) + .allMatch(Boolean::booleanValue)) ); } @@ -192,7 +225,7 @@ public class HgSessionManagerRaftPDTest { String keyPrefix = "BATCH-GET-UNIT-" + timestamp; - Map map = batchPut(session, keyPrefix, 300000); + Map map = HgStoreTestUtil.batchPut(session, keyPrefix, 300000); File outputFile = new File("tmp/batch_put_list"); if (outputFile.exists()) { @@ -203,7 +236,8 @@ public class HgSessionManagerRaftPDTest { ObjectOutputStream oo = new ObjectOutputStream(new FileOutputStream(outputFile)); oo.writeObject(map); oo.close(); - System.out.printf("%d entries have be put into graph %s\n", map.size(), GRAPH_NAME); + System.out.printf("%d entries have be put into graph %s\n", map.size(), + HgStoreTestUtil.GRAPH_NAME); } // @Test @@ -218,15 +252,15 @@ public class HgSessionManagerRaftPDTest { String keyPrefix1 = "BATCH-PUT-UNIT-X-" + timestamp; String keyPrefix2 = "BATCH-PUT-UNIT-Y-" + timestamp; String keyPrefix3 = "BATCH-PUT-UNIT-Z-" + timestamp; - batchPut(session1, TABLE_NAME_1, keyPrefix1, 1000); - batchPut(session1, TABLE_NAME_2, keyPrefix1, 1000); - batchPut(session1, TABLE_NAME_3, keyPrefix1, 1000); - batchPut(session2, TABLE_NAME_1, keyPrefix2, 1000); - batchPut(session2, TABLE_NAME_2, keyPrefix2, 1000); - batchPut(session2, TABLE_NAME_3, keyPrefix2, 1000); - batchPut(session3, TABLE_NAME_1, keyPrefix3, 1000); - batchPut(session3, TABLE_NAME_2, keyPrefix3, 1000); - batchPut(session3, TABLE_NAME_3, keyPrefix3, 1000); + HgStoreTestUtil.batchPut(session1, TABLE_NAME_1, keyPrefix1, 1000); + HgStoreTestUtil.batchPut(session1, TABLE_NAME_2, keyPrefix1, 1000); + HgStoreTestUtil.batchPut(session1, TABLE_NAME_3, keyPrefix1, 1000); + HgStoreTestUtil.batchPut(session2, TABLE_NAME_1, keyPrefix2, 1000); + HgStoreTestUtil.batchPut(session2, TABLE_NAME_2, keyPrefix2, 1000); + HgStoreTestUtil.batchPut(session2, TABLE_NAME_3, keyPrefix2, 1000); + HgStoreTestUtil.batchPut(session3, TABLE_NAME_1, keyPrefix3, 1000); + HgStoreTestUtil.batchPut(session3, TABLE_NAME_2, keyPrefix3, 1000); + HgStoreTestUtil.batchPut(session3, TABLE_NAME_3, keyPrefix3, 1000); } // @Test @@ -236,21 +270,21 @@ public class HgSessionManagerRaftPDTest { System.out.println("--- test merge (1+1=2)---"); HgStoreSession session = getStoreSession(); String mergeKey = "merge-key"; - HgOwnerKey key = toOwnerKey(mergeKey); - byte[] value = toBytes(1l); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey); + byte[] value = HgStoreTestUtil.toBytes(1L); System.out.println("- put " + mergeKey + ":1 -"); - session.put(TABLE_NAME, key, value); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); System.out.println("- merge " + mergeKey + ":1 -"); - session.merge(TABLE_NAME, key, value); - long res = toLong(session.get(TABLE_NAME, key)); + session.merge(HgStoreTestUtil.TABLE_NAME, key, value); + long res = HgStoreTestUtil.toLong(session.get(HgStoreTestUtil.TABLE_NAME, key)); System.out.printf("after merge " + mergeKey + "=%s%n", res); - Assert.assertEquals(2l, res); + Assert.assertEquals(2L, res); String putStr = "19"; - session.put(TABLE_NAME, key, toBytes(putStr)); - byte[] b1 = session.get(TABLE_NAME, key); - Assert.assertEquals(putStr, toStr(b1)); + session.put(HgStoreTestUtil.TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr)); + byte[] b1 = session.get(HgStoreTestUtil.TABLE_NAME, key); + Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1)); } // @Test @@ -260,16 +294,16 @@ public class HgSessionManagerRaftPDTest { String delKey = "del-key"; String delValue = "del-value"; - HgOwnerKey key = toOwnerKey(delKey); - byte[] value = toBytes(delValue); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); - println("- put " + delKey + ": " + delValue); - session.put(TABLE_NAME, key, value); - println("- delete " + delKey); - session.delete(TABLE_NAME, key); - value = session.get(TABLE_NAME, key); - println("- get " + delKey + ": " + toStr(value)); - Assert.assertTrue(EMPTY_BYTES.equals(value)); + HgStoreTestUtil.println("- put " + delKey + ": " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + HgStoreTestUtil.println("- delete " + delKey); + session.delete(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(EMPTY_BYTES, value); } // @Test @@ -279,49 +313,56 @@ public class HgSessionManagerRaftPDTest { String delKey = "del-single-key"; String delValue = "del-single-value"; - HgOwnerKey key = toOwnerKey(delKey); - byte[] value = toBytes(delValue); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); - println("- put [" + delKey + "] = " + delValue); - session.put(TABLE_NAME, key, value); + HgStoreTestUtil.println("- put [" + delKey + "] = " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); - value = session.get(TABLE_NAME, key); - println("- before del, get [" + delKey + "] = " + toStr(value)); - Assert.assertEquals(delValue, toStr(value)); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- before del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value)); - println("- delete-single : [" + delKey + "]"); - session.deleteSingle(TABLE_NAME, key); - value = session.get(TABLE_NAME, key); - println("- after del, get [" + delKey + "] = " + toStr(value)); - Assert.assertEquals("", toStr(value)); + HgStoreTestUtil.println("- delete-single : [" + delKey + "]"); + session.deleteSingle(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals("", HgStoreTestUtil.toStr(value)); } // @Test public void deleteRange() { - println("--- test deleteRange ---"); + HgStoreTestUtil.println("--- test deleteRange ---"); HgStoreSession session = getStoreSession(); String rangePrefix = "DEL-RANGE-KEY"; String owner = "batch-delete-owner"; - Map map = batchPut(session, TABLE_NAME, rangePrefix, 10, key -> { - return toOwnerKey(owner, key); - }); + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, rangePrefix, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); - HgOwnerKey startKey = toOwnerKey(owner, rangePrefix + "-00"); - HgOwnerKey endKey = toOwnerKey(owner, rangePrefix + "-05"); + HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00"); + HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05"); - Assert.assertTrue(session.deleteRange(TABLE_NAME, startKey, endKey)); + Assert.assertTrue(session.deleteRange(HgStoreTestUtil.TABLE_NAME, startKey, endKey)); - println("- after delete range from [" - + toStr(startKey.getKey()) - + "] to [" - + toStr(endKey.getKey()) + "]"); + HgStoreTestUtil.println("- after delete range from [" + + HgStoreTestUtil.toStr(startKey.getKey()) + + "] to [" + + HgStoreTestUtil.toStr(endKey.getKey()) + "]"); for (int i = 0; i < 10; i++) { - HgOwnerKey key = toOwnerKey(owner, rangePrefix + "-" + padLeftZeros(String.valueOf(i), 2)); - String value = toStr(session.get(TABLE_NAME, key)); - println("- get [" + toStr(key.getKey()) + "] = " + value); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" + + HgStoreTestUtil.padLeftZeros( + String.valueOf(i), 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + HgStoreTestUtil.println( + "- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); // TODO: [start,end)? if (i < 5) { @@ -341,24 +382,29 @@ public class HgSessionManagerRaftPDTest { String prefixStr = "DEL-PREFIX-KEY"; String owner = "batch-delete-owner"; - Map map = batchPut(session, TABLE_NAME, prefixStr, 10, key -> { - return toOwnerKey(owner, key); - }); + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, prefixStr, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); //printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList())); - HgOwnerKey prefix = toOwnerKey(owner, prefixStr); + HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr); - Assert.assertEquals(10, amountOf(session.scanIterator(TABLE_NAME, prefix))); - session.deletePrefix(TABLE_NAME, prefix); - Assert.assertEquals(0, amountOf(session.scanIterator(TABLE_NAME, prefix))); + Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + session.deletePrefix(HgStoreTestUtil.TABLE_NAME, prefix); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); - println("- after delete by prefix:[" + prefixStr + "]"); + HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]"); for (int i = 0; i < 10; i++) { - HgOwnerKey key = toOwnerKey(owner, prefixStr + toSuffix(i, 2)); - String value = toStr(session.get(TABLE_NAME, key)); - System.out.println("- get [" + toStr(key.getKey()) + "] = " + value); + HgOwnerKey key = + HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.println("- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); Assert.assertEquals("", value); } @@ -491,52 +537,58 @@ public class HgSessionManagerRaftPDTest { // @Test public void scanIterator() { - println("--- test scanIterator ---"); + HgStoreTestUtil.println("--- test scanIterator ---"); String tableName = "UNIT_SCAN"; String keyName = "SCAN-ITER"; HgStoreSession session = getStoreSession(); - batchPut(session, tableName, keyName, 10000); + HgStoreTestUtil.batchPut(session, tableName, keyName, 10000); int count = 0; int limit = 0; int max = 99999; HgKvIterator iterator = null; - println("-- test 0 element --"); - iterator = session.scanIterator(tableName, toAllPartitionKey("__SCAN-001"), toAllPartitionKey("__SCAN-100"), 0); + HgStoreTestUtil.println("-- test 0 element --"); + iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"), + HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0); Assert.assertFalse(iterator.hasNext()); try { iterator.next(); - Assert.assertTrue(false); + Assert.fail(); } catch (Throwable t) { - println("-- test NoSuchElementException --"); - Assert.assertTrue(NoSuchElementException.class.isInstance(t)); + HgStoreTestUtil.println("-- test NoSuchElementException --"); + Assert.assertTrue(t instanceof NoSuchElementException); } - println("-- test limit 1 to 10 --"); + HgStoreTestUtil.println("-- test limit 1 to 10 --"); for (int i = 1; i <= 10; i++) { - println("- limit " + i + " -"); + HgStoreTestUtil.println("- limit " + i + " -"); limit = i; - iterator = session.scanIterator(tableName, toAllPartitionKey(keyName + "-0"), toAllPartitionKey(keyName + "-1"), limit); + iterator = session.scanIterator(tableName, + HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), + limit); count = 0; while (iterator.hasNext()) { count++; HgKvEntry entry = iterator.next(); - println(entry); + HgStoreTestUtil.println(entry); } Assert.assertEquals(limit, count); } - println("-- test limit 1 to 10 not enough --"); + HgStoreTestUtil.println("-- test limit 1 to 10 not enough --"); for (int i = 1; i <= 10; i++) { - println("- limit " + i + " -"); + HgStoreTestUtil.println("- limit " + i + " -"); limit = i; iterator = session.scanIterator(tableName, - toAllPartitionKey(keyName + "-00001"), toAllPartitionKey(keyName + "-00005"), limit); + HgStoreTestUtil.toAllPartitionKey(keyName + "-00001"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-00005"), + limit); count = 0; while (iterator.hasNext()) { count++; HgKvEntry entry = iterator.next(); - println(entry); + HgStoreTestUtil.println(entry); } if (i <= 5) { Assert.assertEquals(limit, count); @@ -546,42 +598,45 @@ public class HgSessionManagerRaftPDTest { } - println("-- test limit 0 (no limit) --"); + HgStoreTestUtil.println("-- test limit 0 (no limit) --"); limit = 0; - iterator = session.scanIterator(tableName, toAllPartitionKey(keyName + "-0"), toAllPartitionKey(keyName + "-1"), limit); + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), limit); count = 0; while (iterator.hasNext()) { count++; HgKvEntry entry = iterator.next(); if (count % 1000 == 0) { - println(entry); + HgStoreTestUtil.println(entry); } if (count >= max) break; } Assert.assertEquals(10000, count); - println("-- test scan all --"); + HgStoreTestUtil.println("-- test scan all --"); iterator = session.scanIterator(tableName); count = 0; while (iterator.hasNext()) { count++; HgKvEntry entry = iterator.next(); if (count % 1000 == 0) { - println(entry); + HgStoreTestUtil.println(entry); } if (count >= max) break; } Assert.assertEquals(10000, count); - println("-- test scan prefix --"); - iterator = session.scanIterator(tableName, toAllPartitionKey(keyName + "-01")); + HgStoreTestUtil.println("-- test scan prefix --"); + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-01")); count = 0; while (iterator.hasNext()) { count++; HgKvEntry entry = iterator.next(); if (count % 100 == 0) { - println(entry); + HgStoreTestUtil.println(entry); } if (count >= max) break; } @@ -590,13 +645,13 @@ public class HgSessionManagerRaftPDTest { // @Test public void paging() { - println("--- test scanIterator_range ---"); + HgStoreTestUtil.println("--- test scanIterator_range ---"); String graph = "UNIT/paging"; String tableName = "UNIT_SCAN_PAGING"; String keyName = "SCAN-PAGING"; int keyAmt = 100; HgStoreSession session = getStoreSession(graph); - batchPut(session, tableName, keyName, keyAmt); + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); int count = 0; int limit = 0; int max = 99999; @@ -605,7 +660,7 @@ public class HgSessionManagerRaftPDTest { count = 0; iterator = session.scanIterator(tableName - , toAllPartitionKey(keyName + "-000") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-000") , HgStoreClientConst.EMPTY_OWNER_KEY , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES ); @@ -613,10 +668,10 @@ public class HgSessionManagerRaftPDTest { // Assert.assertEquals(100, HgStoreTestUtil.amountOf(iterator)); List positionList = new LinkedList<>(); while (iterator.hasNext()) { - print((count++) + " "); + HgStoreTestUtil.print((count++) + " "); HgKvEntry entry = iterator.next(); - print(entry); - print(" " + Arrays.toString(iterator.position()) + "\n"); + HgStoreTestUtil.print(entry); + HgStoreTestUtil.print(" " + Arrays.toString(iterator.position()) + "\n"); positionList.add(iterator.position()); if (count >= max) break; } @@ -637,7 +692,7 @@ public class HgSessionManagerRaftPDTest { iterator = session.scanIterator(tableName, 100); byte[] position = positionList.get(50); - println("seek: " + Arrays.toString(position)); + HgStoreTestUtil.println("seek: " + Arrays.toString(position)); iterator.seek(position); HgStoreTestUtil.println(iterator); //Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator)); @@ -646,23 +701,23 @@ public class HgSessionManagerRaftPDTest { // @Test public void truncate() { - println("--- test truncate ---"); + HgStoreTestUtil.println("--- test truncate ---"); String graph = "graph_truncate"; String tableName = "UNIT_TRUNCATE_1"; String keyName = "KEY_TRUNCATE"; HgStoreSession session = getStoreSession(graph); - batchPut(session, tableName, keyName, 100); - Assert.assertEquals(100, amountOf(session.scanIterator(tableName))); + HgStoreTestUtil.batchPut(session, tableName, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); String tableName2 = "UNIT_TRUNCATE_2"; - batchPut(session, tableName2, keyName, 100); - Assert.assertEquals(100, amountOf(session.scanIterator(tableName2))); + HgStoreTestUtil.batchPut(session, tableName2, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); session.truncate(); - Assert.assertEquals(0, amountOf(session.scanIterator(tableName))); - Assert.assertEquals(0, amountOf(session.scanIterator(tableName2))); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); } //// @Test @@ -674,8 +729,10 @@ public class HgSessionManagerRaftPDTest { int amount = 10_000_000; HgStoreSession session = getStoreSession(); - if (amountOf(session.scanIterator(tableName, toAllPartitionKey(keyName), 10)) < 10) { - batchPut(session, tableName, keyName, amount); + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) < + 10) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); } int count = 0; @@ -685,7 +742,7 @@ public class HgSessionManagerRaftPDTest { count++; HgKvEntry entry = iterator.next(); if (count % (amount / 10) == 0) { - println(entry); + HgStoreTestUtil.println(entry); } } @@ -704,12 +761,12 @@ public class HgSessionManagerRaftPDTest { // @Test public void testDelGraph() { HgStoreSession session = getStoreSession(); - session.deleteGraph(GRAPH_NAME); + session.deleteGraph(HgStoreTestUtil.GRAPH_NAME); } // @Test public void benchmark_scanBatch() { - println("--- Benchmark scanBatch ---"); + HgStoreTestUtil.println("--- Benchmark scanBatch ---"); String tableName = "Benchmark_SCAN_BATCH"; String keyName = "SCAN-BATCH"; int keyAmt = 30001; @@ -719,7 +776,7 @@ public class HgSessionManagerRaftPDTest { /*if (amountOf(session.scanIterator(tableName, toAllPartitionKey(keyName), 1)) < 1) { batchPut(session, tableName, keyName, keyAmt); }*/ - println("-- Starting scan --"); + HgStoreTestUtil.println("-- Starting scan --"); MetricX metrics = MetricX.ofStart(); //HgStoreTestUtil.println(session.scanIterator(tableName)); List> iterators = session.scanBatch(HgScanQuery.tableOf(tableName)); @@ -741,20 +798,22 @@ public class HgSessionManagerRaftPDTest { @Test public void benchmark_scanBatch2() throws IOException { - println("--- Benchmark scanBatch2 ---"); + HgStoreTestUtil.println("--- Benchmark scanBatch2 ---"); //String tableName = TABLE_NAME; HgStoreSession session = getStoreSession(); String keyName = "SCAN-BATCH"; int keyAmt = 3000; - Map data = batchPut(session, TABLE_NAME, keyName, keyAmt, - //(key) -> toOwnerKey(key.substring(0, keyName.length() + 2), key) - (key) -> toOwnerKey(0, key) - ); + Map data = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, keyName, keyAmt, + //(key) -> toOwnerKey(key.substring(0, keyName.length() + // + 2), key) + (key) -> HgStoreTestUtil.toOwnerKey(0, key) + ); - batchPut(session, TABLE_NAME2, keyName, keyAmt, - //(key) -> toOwnerKey(key.substring(0, keyName.length() + 2), key) - (key) -> toOwnerKey(0, key) + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME2, keyName, keyAmt, + //(key) -> toOwnerKey(key.substring(0, keyName.length() + 2), key) + (key) -> HgStoreTestUtil.toOwnerKey(0, key) ); MetricX metrics = MetricX.ofStart(); @@ -767,35 +826,21 @@ public class HgSessionManagerRaftPDTest { data.forEach((k, v) -> keys.add(k)); List prefixKeys = new ArrayList<>(); - prefixKeys.add(toOwnerKey(0,keyName + "-0")); - prefixKeys.add(toOwnerKey(0,keyName + "-1")); - prefixKeys.add(toOwnerKey(0,keyName + "-2")); - prefixKeys.add(toOwnerKey(0,keyName + "-3")); + prefixKeys.add(HgStoreTestUtil.toOwnerKey(0, keyName + "-0")); + prefixKeys.add(HgStoreTestUtil.toOwnerKey(0, keyName + "-1")); + prefixKeys.add(HgStoreTestUtil.toOwnerKey(0, keyName + "-2")); + prefixKeys.add(HgStoreTestUtil.toOwnerKey(0, keyName + "-3")); - println("-- test every key, one table --"); - count=0; - queryTable = TABLE_NAME; - queryKeys=keys; - iterators = session.scanBatch2(HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(), ScanOrderType.ORDER_NONE) - .builder() - .setScanType(0x40) - .build()); - while (iterators.hasNext()) { - HgKvIterator iterator = iterators.next(); - count += HgStoreTestUtil.amountOf(iterator); - } - iterators.close(); - Assert.assertEquals(keyAmt,count); - log.info(" size is {}", count); - - println("-- test prefix key, one table --"); + HgStoreTestUtil.println("-- test every key, one table --"); count = 0; - queryTable = TABLE_NAME; - queryKeys = prefixKeys; - iterators = session.scanBatch2(HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(), ScanOrderType.ORDER_STRICT) - .builder() - .setScanType(0x40) - .build()); + queryTable = HgStoreTestUtil.TABLE_NAME; + queryKeys = keys; + iterators = session.scanBatch2( + HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(), + ScanOrderType.ORDER_NONE) + .builder() + .setScanType(0x40) + .build()); while (iterators.hasNext()) { HgKvIterator iterator = iterators.next(); count += HgStoreTestUtil.amountOf(iterator); @@ -804,14 +849,34 @@ public class HgSessionManagerRaftPDTest { Assert.assertEquals(keyAmt, count); log.info(" size is {}", count); - println("-- test prefix key, two table --"); - count=0; - queryTable = TABLE_NAME + "," + TABLE_NAME2; + HgStoreTestUtil.println("-- test prefix key, one table --"); + count = 0; + queryTable = HgStoreTestUtil.TABLE_NAME; queryKeys = prefixKeys; - iterators = session.scanBatch2(HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(), ScanOrderType.ORDER_NONE) - .builder() - .setScanType(0x40) - .build()); + iterators = session.scanBatch2( + HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(), + ScanOrderType.ORDER_STRICT) + .builder() + .setScanType(0x40) + .build()); + while (iterators.hasNext()) { + HgKvIterator iterator = iterators.next(); + count += HgStoreTestUtil.amountOf(iterator); + } + iterators.close(); + Assert.assertEquals(keyAmt, count); + log.info(" size is {}", count); + + HgStoreTestUtil.println("-- test prefix key, two table --"); + count = 0; + queryTable = HgStoreTestUtil.TABLE_NAME + "," + HgStoreTestUtil.TABLE_NAME2; + queryKeys = prefixKeys; + iterators = session.scanBatch2( + HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(), + ScanOrderType.ORDER_NONE) + .builder() + .setScanType(0x40) + .build()); while (iterators.hasNext()) { HgKvIterator iterator = iterators.next(); count += HgStoreTestUtil.amountOf(iterator); @@ -820,16 +885,18 @@ public class HgSessionManagerRaftPDTest { Assert.assertEquals(keyAmt * 2, count); log.info(" size is {}", count); - println("-- test prefix key, two table, perKeyMax --"); - count=0; - queryTable = TABLE_NAME + "," + TABLE_NAME2; + HgStoreTestUtil.println("-- test prefix key, two table, perKeyMax --"); + count = 0; + queryTable = HgStoreTestUtil.TABLE_NAME + "," + HgStoreTestUtil.TABLE_NAME2; queryKeys = prefixKeys; - queryKeys.remove(queryKeys.size()-1);//remove the last one. - iterators = session.scanBatch2(HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(), ScanOrderType.ORDER_NONE) - .builder() - .setScanType(0x40) - .setPerKeyMax(10) - .build()); + queryKeys.remove(queryKeys.size() - 1);//remove the last one. + iterators = session.scanBatch2( + HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(), + ScanOrderType.ORDER_NONE) + .builder() + .setScanType(0x40) + .setPerKeyMax(10) + .build()); while (iterators.hasNext()) { HgKvIterator iterator = iterators.next(); count += HgStoreTestUtil.amountOf(iterator); @@ -859,28 +926,32 @@ public class HgSessionManagerRaftPDTest { // @Test public void benchmark_scanBatch_SkipDegree() throws IOException { - println("--- Benchmark scanBatch2 1Owner---"); - String tableName = TABLE_NAME; + HgStoreTestUtil.println("--- Benchmark scanBatch2 1Owner---"); + String tableName = HgStoreTestUtil.TABLE_NAME; HgStoreSession session = getStoreSession(); String keyName = "SCAN-BATCH"; int keyAmt = 300000; byte[] owner = "Owner".getBytes(); - Map data = batchPut(session, tableName, keyName, keyAmt, key -> { - return toOwnerKey(owner, key); - }); - println("-- Starting scan --"); + Map data = + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt, key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + HgStoreTestUtil.println("-- Starting scan --"); MetricX metrics = MetricX.ofStart(); //HgStoreTestUtil.println(session.scanIterator(tableName)); long t = System.currentTimeMillis(); int count = 0; { List keys = new ArrayList<>(); - keys.add(toOwnerKey(owner, keyName)); + keys.add(HgStoreTestUtil.toOwnerKey(owner, keyName)); KvCloseableIterator> iterators = - session.scanBatch2(HgScanQuery.prefixIteratorOf(tableName, keys.iterator(), ScanOrderType.ORDER_NONE) - .builder().setScanType(0x40).setSkipDegree(1).build()); - //KvCloseableIterator> iterators = session.scanBatch2(HgScanQuery.tableOf(tableName)); + session.scanBatch2(HgScanQuery.prefixIteratorOf(tableName, keys.iterator(), + ScanOrderType.ORDER_NONE) + .builder().setScanType(0x40).setSkipDegree(1) + .build()); + //KvCloseableIterator> iterators = session.scanBatch2(HgScanQuery + // .tableOf(tableName)); while (iterators.hasNext()) { HgKvIterator iterator = iterators.next(); // while (iterator.hasNext()){ diff --git a/hg-store-client/src/test/java/com/baidu/hugegraph/store/HgSessionManagerTest.java b/hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerTest.java similarity index 55% rename from hg-store-client/src/test/java/com/baidu/hugegraph/store/HgSessionManagerTest.java rename to hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerTest.java index d2964fdf2..a1ac0e1af 100644 --- a/hg-store-client/src/test/java/com/baidu/hugegraph/store/HgSessionManagerTest.java +++ b/hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerTest.java @@ -1,70 +1,102 @@ -package com.baidu.hugegraph.store; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.pd.common.PartitionUtils; -import com.baidu.hugegraph.store.client.*; -import com.baidu.hugegraph.store.client.util.ExecutorPool; -import com.baidu.hugegraph.store.client.util.HgStoreClientConfig; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import com.baidu.hugegraph.store.client.util.MetricX; -import com.baidu.hugegraph.store.util.HgStoreTestUtil; -import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -// import org.junit.BeforeClass; -// import org.junit.Test; +package org.apache.hugegraph.store; -import java.util.*; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.stream.Collectors; -import static com.baidu.hugegraph.store.client.util.HgAssert.isInvalid; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; -import static com.baidu.hugegraph.store.util.HgStoreTestUtil.*; -import static com.baidu.hugegraph.store.util.HgStoreTestUtil.println; +import org.apache.hugegraph.store.client.HgNodePartitionerBuilder; +import org.apache.hugegraph.store.client.HgStoreNode; +import org.apache.hugegraph.store.client.HgStoreNodeManager; +import org.apache.hugegraph.store.client.HgStoreNodeNotifier; +import org.apache.hugegraph.store.client.HgStoreNodePartitioner; +import org.apache.hugegraph.store.client.HgStoreNodeProvider; +import org.apache.hugegraph.store.client.HgStoreNotice; +import org.apache.hugegraph.store.client.util.ExecutorPool; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.MetricX; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.baidu.hugegraph.pd.common.PartitionUtils; +import com.baidu.hugegraph.store.client.*; + +import lombok.extern.slf4j.Slf4j; @Slf4j public class HgSessionManagerTest { - private static Map leaderMap = new ConcurrentHashMap<>(); - private static Map storeMap = new ConcurrentHashMap<>(); + private static final Map leaderMap = new ConcurrentHashMap<>(); + private static final Map storeMap = new ConcurrentHashMap<>(); - private static ExecutorService pool = Executors.newFixedThreadPool(100, - ExecutorPool.newThreadFactory("unit-test")); + private static final ExecutorService pool = Executors.newFixedThreadPool(100, + ExecutorPool.newThreadFactory( + "unit-test")); - private static int partitionCount = 10; // 需要与store的application.yml的fake-pd.partition-count保持一致 + private static final int partitionCount = 10; // 需要与store的application.yml的fake-pd.partition-count保持一致 //private static String[] storeAddress = {"127.0.0.1:8500"}; - private static String[] storeAddress = {"127.0.0.1:8501", "127.0.0.1:8502", "127.0.0.1:8503"}; + private static final String[] storeAddress = {"127.0.0.1:8501", "127.0.0.1:8502", "127.0.0.1:8503"}; - private static int PARTITION_LENGTH = getPartitionLength(); + private static final int PARTITION_LENGTH = getPartitionLength(); private static int getPartitionLength() { - return PartitionUtils.MAX_VALUE / (partitionCount == 0 ? storeAddress.length : partitionCount) + 1; + return PartitionUtils.MAX_VALUE / + (partitionCount == 0 ? storeAddress.length : partitionCount) + 1; } - @BeforeClass + @BeforeClass public static void init() { for (String address : storeAddress) { storeMap.put((long) address.hashCode(), address); } - for (int i = 0; i < partitionCount; i++) - leaderMap.put(i, (long) storeMap.keySet().iterator().next()); + for (int i = 0; i < partitionCount; i++) { + leaderMap.put(i, storeMap.keySet().iterator().next()); + } HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); nodeManager.setNodePartitioner(new HgStoreNodePartitioner() { @Override - public int partition(HgNodePartitionerBuilder builder, String graphName, byte[] startKey, byte[] endKey) { + public int partition(HgNodePartitionerBuilder builder, String graphName, + byte[] startKey, byte[] endKey) { int startCode = PartitionUtils.calcHashcode(startKey); int endCode = PartitionUtils.calcHashcode(endKey); if (ALL_PARTITION_OWNER == startKey) { storeMap.forEach((k, v) -> { builder.add(k, -1); }); - } else if (endKey == HgStoreClientConst.EMPTY_BYTES || startKey == endKey || Arrays.equals(startKey, endKey)) { + } else if (endKey == HgStoreClientConst.EMPTY_BYTES || startKey == endKey || + Arrays.equals(startKey, endKey)) { //log.info("leader-> {}",leaderMap.get(startCode / PARTITION_LENGTH)); builder.add(leaderMap.get(startCode / PARTITION_LENGTH), startCode); } else { @@ -78,9 +110,10 @@ public class HgSessionManagerTest { nodeManager.setNodeProvider(new HgStoreNodeProvider() { @Override public HgStoreNode apply(String graphName, Long nodeId) { - // System.out.println("HgStoreNodeProvider apply " + graphName + " " + nodeId + " " + storeMap.get(nodeId)); + // System.out.println("HgStoreNodeProvider apply " + graphName + " " + nodeId + + // " " + storeMap.get(nodeId)); return nodeManager.getNodeBuilder().setNodeId(nodeId) - .setAddress(storeMap.get(nodeId)).build(); + .setAddress(storeMap.get(nodeId)).build(); } }); nodeManager.setNodeNotifier(new HgStoreNodeNotifier() { @@ -101,25 +134,26 @@ public class HgSessionManagerTest { } protected static HgStoreSession getStoreSession() { - return HgSessionManager.getInstance().openSession(GRAPH_NAME); + return HgSessionManager.getInstance().openSession(HgStoreTestUtil.GRAPH_NAME); } protected static HgStoreSession getStoreSession(String graph) { return HgSessionManager.getInstance().openSession(graph); } - @Test + @Test public void put_get() { System.out.println("--- test put & get ---"); HgStoreSession session = getStoreSession(); - HgOwnerKey key = toOwnerKey("FOR-PUT-KEY"); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); - byte[] value = toBytes("FOR-PUT-VALUE"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); - Assert.assertTrue(session.put(TABLE_NAME, key, value)); - Assert.assertEquals(toStr(value), toStr(session.get(TABLE_NAME, key))); + Assert.assertTrue(session.put(HgStoreTestUtil.TABLE_NAME, key, value)); + Assert.assertEquals(HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get( + HgStoreTestUtil.TABLE_NAME, key))); } @Test @@ -128,8 +162,9 @@ public class HgSessionManagerTest { HgStoreSession session = getStoreSession(); String keyPrefix = "UNIT-BATCH-GET"; - Map map = batchPut(session, keyPrefix); - List keyList = map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); + Map map = HgStoreTestUtil.batchPut(session, keyPrefix); + List keyList = + map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); Map keyMap = new HashMap<>(); map.entrySet().forEach(e -> { @@ -137,8 +172,8 @@ public class HgSessionManagerTest { }); //printOwner(keyList); - HgKvIterator iterator = session.batchPrefix(TABLE_NAME, keyList); - int amount = println(iterator); + HgKvIterator iterator = session.batchPrefix(HgStoreTestUtil.TABLE_NAME, keyList); + int amount = HgStoreTestUtil.println(iterator); Assert.assertEquals(amount, keyList.size()); /* @@ -159,21 +194,21 @@ public class HgSessionManagerTest { System.out.println("--- test merge (1+1=2)---"); HgStoreSession session = getStoreSession(); String mergeKey = "merge-key"; - HgOwnerKey key = toOwnerKey(mergeKey); - byte[] value = toBytes(1l); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey); + byte[] value = HgStoreTestUtil.toBytes(1L); System.out.println("- put " + mergeKey + ":1 -"); - session.put(TABLE_NAME, key, value); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); System.out.println("- merge " + mergeKey + ":1 -"); - session.merge(TABLE_NAME, key, value); - long res = toLong(session.get(TABLE_NAME, key)); + session.merge(HgStoreTestUtil.TABLE_NAME, key, value); + long res = HgStoreTestUtil.toLong(session.get(HgStoreTestUtil.TABLE_NAME, key)); System.out.printf("after merge " + mergeKey + "=%s%n", res); - Assert.assertEquals(2l, res); + Assert.assertEquals(2L, res); String putStr = "19"; - session.put(TABLE_NAME, key, toBytes(putStr)); - byte[] b1 = session.get(TABLE_NAME, key); - Assert.assertEquals(putStr, toStr(b1)); + session.put(HgStoreTestUtil.TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr)); + byte[] b1 = session.get(HgStoreTestUtil.TABLE_NAME, key); + Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1)); } @Test @@ -183,16 +218,16 @@ public class HgSessionManagerTest { String delKey = "del-key"; String delValue = "del-value"; - HgOwnerKey key = toOwnerKey(delKey); - byte[] value = toBytes(delValue); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); - println("- put " + delKey + ": " + delValue); - session.put(TABLE_NAME, key, value); - println("- delete " + delKey); - session.delete(TABLE_NAME, key); - value = session.get(TABLE_NAME, key); - println("- get " + delKey + ": " + toStr(value)); - Assert.assertTrue(EMPTY_BYTES.equals(value)); + HgStoreTestUtil.println("- put " + delKey + ": " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + HgStoreTestUtil.println("- delete " + delKey); + session.delete(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(EMPTY_BYTES, value); } // @Test @@ -202,48 +237,54 @@ public class HgSessionManagerTest { String delKey = "del-single-key"; String delValue = "del-single-value"; - HgOwnerKey key = toOwnerKey(delKey); - byte[] value = toBytes(delValue); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); - println("- put [" + delKey + "] = " + delValue); - session.put(TABLE_NAME, key, value); + HgStoreTestUtil.println("- put [" + delKey + "] = " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); - value = session.get(TABLE_NAME, key); - println("- before del, get [" + delKey + "] = " + toStr(value)); - Assert.assertEquals(delValue, toStr(value)); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- before del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value)); - println("- delete-single : [" + delKey + "]"); - session.deleteSingle(TABLE_NAME, key); - value = session.get(TABLE_NAME, key); - println("- after del, get [" + delKey + "] = " + toStr(value)); - Assert.assertEquals("", toStr(value)); + HgStoreTestUtil.println("- delete-single : [" + delKey + "]"); + session.deleteSingle(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals("", HgStoreTestUtil.toStr(value)); } // @Test public void deleteRange() { - println("--- test deleteRange ---"); + HgStoreTestUtil.println("--- test deleteRange ---"); HgStoreSession session = getStoreSession(); String rangePrefix = "DEL-RANGE-KEY"; String owner = "batch-delete-owner"; - Map map = batchPut(session, TABLE_NAME, rangePrefix, 10, - key -> toOwnerKey(owner, key) - ); + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, rangePrefix, 10, + key -> HgStoreTestUtil.toOwnerKey(owner, key) + ); - HgOwnerKey startKey = toOwnerKey(owner, rangePrefix + "-00"); - HgOwnerKey endKey = toOwnerKey(owner, rangePrefix + "-05"); + HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00"); + HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05"); - Assert.assertTrue(session.deleteRange(TABLE_NAME, startKey, endKey)); + Assert.assertTrue(session.deleteRange(HgStoreTestUtil.TABLE_NAME, startKey, endKey)); - println("- after delete range from [" - + toStr(startKey.getKey()) + "] to [" - + toStr(endKey.getKey()) + "]"); + HgStoreTestUtil.println("- after delete range from [" + + HgStoreTestUtil.toStr(startKey.getKey()) + "] to [" + + HgStoreTestUtil.toStr(endKey.getKey()) + "]"); for (int i = 0; i < 10; i++) { - HgOwnerKey key = toOwnerKey(owner, rangePrefix + "-" + padLeftZeros(String.valueOf(i), 2)); - String value = toStr(session.get(TABLE_NAME, key)); - println("- get [" + toStr(key.getKey()) + "] = " + value); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" + + HgStoreTestUtil.padLeftZeros( + String.valueOf(i), 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + HgStoreTestUtil.println( + "- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); // TODO: [start,end)? if (i < 5) { @@ -263,24 +304,29 @@ public class HgSessionManagerTest { String prefixStr = "DEL-PREFIX-KEY"; String owner = "batch-delete-owner"; - Map map = batchPut(session, TABLE_NAME, prefixStr, 10, key -> { - return toOwnerKey(owner, key); - }); + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, prefixStr, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); //printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList())); - HgOwnerKey prefix = toOwnerKey(owner, prefixStr); + HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr); - Assert.assertEquals(10, amountOf(session.scanIterator(TABLE_NAME, prefix))); - session.deletePrefix(TABLE_NAME, prefix); - Assert.assertEquals(0, amountOf(session.scanIterator(TABLE_NAME, prefix))); + Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + session.deletePrefix(HgStoreTestUtil.TABLE_NAME, prefix); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); - println("- after delete by prefix:[" + prefixStr + "]"); + HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]"); for (int i = 0; i < 10; i++) { - HgOwnerKey key = toOwnerKey(owner, prefixStr + toSuffix(i, 2)); - String value = toStr(session.get(TABLE_NAME, key)); - System.out.println("- get [" + toStr(key.getKey()) + "] = " + value); + HgOwnerKey key = + HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.println("- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); Assert.assertEquals("", value); } @@ -288,14 +334,16 @@ public class HgSessionManagerTest { // @Test public void scan() { - println("--- test scanIterator ---"); + HgStoreTestUtil.println("--- test scanIterator ---"); String tableName = "UNIT_SCAN_ITERATOR"; String keyName = "SCAN-ITER"; int keyAmt = 100; HgStoreSession session = getStoreSession(); - if (amountOf(session.scanIterator(tableName, toAllPartitionKey(keyName), 10)) < 10) { - batchPut(session, tableName, keyName, keyAmt); + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) < + 10) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); } int count = 0; int limit = 0; @@ -317,14 +365,15 @@ public class HgSessionManagerTest { Assert.assertEquals(keyAmt, count); // println("-- test 0 element --"); - iterator = session.scanIterator(tableName, toAllPartitionKey("__SCAN-001"), toAllPartitionKey("__SCAN-100"), 0); + iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"), + HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0); Assert.assertFalse(iterator.hasNext()); try { iterator.next(); - Assert.assertTrue(false); + Assert.fail(); } catch (Throwable t) { // println("-- test NoSuchElementException --"); - Assert.assertTrue(NoSuchElementException.class.isInstance(t)); + Assert.assertTrue(t instanceof NoSuchElementException); } iterator.close(); @@ -332,7 +381,10 @@ public class HgSessionManagerTest { for (int i = 1; i <= 10; i++) { // println("- limit " + i + " -"); limit = i; - iterator = session.scanIterator(tableName, toAllPartitionKey(keyName + "-0"), toAllPartitionKey(keyName + "-1"), limit); + iterator = session.scanIterator(tableName, + HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), + limit); count = 0; while (iterator.hasNext()) { count++; @@ -348,8 +400,8 @@ public class HgSessionManagerTest { // println("- limit " + i + " -"); limit = i; iterator = session.scanIterator(tableName - , toAllPartitionKey(keyName + "-001") - , toAllPartitionKey(keyName + "-005"), limit); + , HgStoreTestUtil.toAllPartitionKey(keyName + "-001") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-005"), limit); count = 0; while (iterator.hasNext()) { count++; @@ -367,7 +419,9 @@ public class HgSessionManagerTest { // println("-- test limit 0 (no limit) --"); limit = 0; - iterator = session.scanIterator(tableName, toAllPartitionKey(keyName + "-0"), toAllPartitionKey(keyName + "-2"), limit); + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-2"), limit); count = 0; while (iterator.hasNext()) { @@ -382,7 +436,8 @@ public class HgSessionManagerTest { Assert.assertEquals(keyAmt, count); // println("-- test scan prefix --"); - iterator = session.scanIterator(tableName, toAllPartitionKey(keyName + "-")); + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-")); count = 0; while (iterator.hasNext()) { count++; @@ -412,7 +467,7 @@ public class HgSessionManagerTest { // println("-- test range limit scan type --"); iterator = session.scanIterator(tableName - , toAllPartitionKey("WWWWWWW") + , HgStoreTestUtil.toAllPartitionKey("WWWWWWW") , HgStoreClientConst.EMPTY_OWNER_KEY , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES ); @@ -423,7 +478,7 @@ public class HgSessionManagerTest { // println("-- test range limit scan type -100"); iterator = session.scanIterator(tableName - , toAllPartitionKey(keyName + "-100") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-100") , HgStoreClientConst.EMPTY_OWNER_KEY , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES ); @@ -434,7 +489,7 @@ public class HgSessionManagerTest { // println("-- test range limit scan type -51 "); iterator = session.scanIterator(tableName - , toAllPartitionKey(keyName + "-051") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-051") , HgStoreClientConst.EMPTY_OWNER_KEY ); @@ -443,20 +498,22 @@ public class HgSessionManagerTest { Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator)); // println("-- test range limit scan type -51 assertEquals"); //TODO: add more... - println("-- test scanIterator end"); + HgStoreTestUtil.println("-- test scanIterator end"); } // @Test public void scan_close() { /*************** test scan close **************/ - println("--- test scan close ---"); + HgStoreTestUtil.println("--- test scan close ---"); String tableName = "UNIT_ITER_CLOSE"; String keyName = "UNIT_ITER_CLOSE"; int amount = 1_000_000; HgStoreSession session = getStoreSession(); - if (amountOf(session.scanIterator(tableName, toAllPartitionKey(keyName), 1)) < 1) { - batchPut(session, tableName, keyName, amount); + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); } HgKvIterator iterator = null; @@ -501,12 +558,12 @@ public class HgSessionManagerTest { /* server timeout, enough -> close */ iterator = session.scanIterator(tableName); iterator.next(); - // HgStoreTestUtil.sleeping(11000); + // HgStoreTestUtil.sleeping(11000); try { HgStoreTestUtil.amountOf(iterator); } catch (Throwable t) { - println("-- passed server waiting timeout --"); + HgStoreTestUtil.println("-- passed server waiting timeout --"); } iterator.close(); @@ -515,15 +572,17 @@ public class HgSessionManagerTest { // @Test public void paging() { - println("--- test scanIterator_range ---"); + HgStoreTestUtil.println("--- test scanIterator_range ---"); String graph = "UNIT/paging"; String tableName = "UNIT_SCAN_PAGING"; String keyName = "SCAN-PAGING"; int keyAmt = 100; HgStoreSession session = getStoreSession(graph); - if (amountOf(session.scanIterator(tableName, toAllPartitionKey(keyName), 1)) < 1) { - batchPut(session, tableName, keyName, keyAmt); + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); } int count = 0; @@ -534,7 +593,7 @@ public class HgSessionManagerTest { count = 0; iterator = session.scanIterator(tableName - , toAllPartitionKey(keyName + "-000") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-000") , HgStoreClientConst.EMPTY_OWNER_KEY , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES ); @@ -542,40 +601,42 @@ public class HgSessionManagerTest { // Assert.assertEquals(100, HgStoreTestUtil.amountOf(iterator)); List positionList = new LinkedList<>(); while (iterator.hasNext()) { - print((count++) + " "); + HgStoreTestUtil.print((count++) + " "); HgKvEntry entry = iterator.next(); - print(entry); - print(" " + Arrays.toString(iterator.position()) + "\n"); + HgStoreTestUtil.print(entry); + HgStoreTestUtil.print(" " + Arrays.toString(iterator.position()) + "\n"); positionList.add(iterator.position()); if (count >= max) break; } iterator = session.scanIterator(tableName - , toAllPartitionKey(keyName + "-000") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-000") , HgStoreClientConst.EMPTY_OWNER_KEY , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES ); byte[] position = positionList.get(50); - println("seek: " + Arrays.toString(position)); + HgStoreTestUtil.println("seek: " + Arrays.toString(position)); iterator.seek(position); //println("amt after seek: "+HgStoreTestUtil.println(iterator)); Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator)); } - @Test + @Test public void scanBatch() { - println("--- test scanBatch ---"); + HgStoreTestUtil.println("--- test scanBatch ---"); String tableName = "UNIT_SCAN_BATCH_1"; String keyName = "SCAN-BATCH"; int keyAmt = 10_000; HgStoreSession session = getStoreSession(); - if (amountOf(session.scanIterator(tableName, toAllPartitionKey(keyName), keyAmt)) < keyAmt) { - batchPut(session, tableName, keyName, keyAmt); + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), + keyAmt)) < keyAmt) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); } Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); @@ -584,34 +645,34 @@ public class HgSessionManagerTest { List prefixes = null; List startList = Arrays.asList( - toAllPartitionKey(keyName + "-001") - , toAllPartitionKey(keyName + "-003") - , toAllPartitionKey(keyName + "-005") - , toAllPartitionKey(keyName + "-007") - , toAllPartitionKey(keyName + "-009") + HgStoreTestUtil.toAllPartitionKey(keyName + "-001") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-003") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-005") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-007") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-009") ); List endList = Arrays.asList( - toAllPartitionKey(keyName + "-002") - , toAllPartitionKey(keyName + "-004") - , toAllPartitionKey(keyName + "-006") - , toAllPartitionKey(keyName + "-008") - , toAllPartitionKey(keyName + "-010") + HgStoreTestUtil.toAllPartitionKey(keyName + "-002") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-004") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-006") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-008") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-010") ); List prefixList = Arrays.asList( - toAllPartitionKey(keyName + "-001") - , toAllPartitionKey(keyName + "-002") - , toAllPartitionKey(keyName + "-003") - , toAllPartitionKey(keyName + "-004") - , toAllPartitionKey(keyName + "-005") - , toAllPartitionKey(keyName + "-006") - , toAllPartitionKey(keyName + "-007") - , toAllPartitionKey(keyName + "-008") - , toAllPartitionKey(keyName + "-009") + HgStoreTestUtil.toAllPartitionKey(keyName + "-001") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-002") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-003") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-004") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-005") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-006") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-007") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-008") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-009") ); - println("-- test scan-batch all --"); + HgStoreTestUtil.println("-- test scan-batch all --"); HgScanQuery scanQuery = HgScanQuery.tableOf(tableName); iterators = session.scanBatch(scanQuery); @@ -619,7 +680,7 @@ public class HgSessionManagerTest { Assert.assertEquals(keyAmt, HgStoreTestUtil.println(iterators)); - println("-- test scan-batch prefix --"); + HgStoreTestUtil.println("-- test scan-batch prefix --"); iterators = session.scanBatch( @@ -627,46 +688,50 @@ public class HgSessionManagerTest { ); Assert.assertEquals(3, iterators.size()); Assert.assertEquals(900, - iterators.parallelStream().mapToInt(e -> HgStoreTestUtil.amountOf(e)).sum() + iterators.parallelStream().mapToInt(e -> HgStoreTestUtil.amountOf(e)) + .sum() ); - println("-- test scan-batch range --"); + HgStoreTestUtil.println("-- test scan-batch range --"); iterators = session.scanBatch(HgScanQuery.rangeOf(tableName, startList, endList)); Assert.assertEquals(3, iterators.size()); Assert.assertEquals( HgStoreTestUtil.amountOf( session.scanIterator(tableName - , toAllPartitionKey(keyName + "-001") - , toAllPartitionKey(keyName + "-010") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-001") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-010") ) ) , iterators.parallelStream().mapToInt(e -> HgStoreTestUtil.amountOf(e)).sum() ); - println("-- test scan-batch limit --"); + HgStoreTestUtil.println("-- test scan-batch limit --"); int limit = 1; iterators = session.scanBatch( HgScanQuery.rangeOf(tableName, startList, endList) - .builder() - .setLimit(limit) - .build() + .builder() + .setLimit(limit) + .build() ); //HgStoreTestUtil.println(iterators); Assert.assertEquals(iterators.size() * limit, - iterators.parallelStream().mapToInt(e -> HgStoreTestUtil.amountOf(e)).sum() + iterators.parallelStream().mapToInt(e -> HgStoreTestUtil.amountOf(e)) + .sum() ); - println("-- test scan-batch multi-table --"); - if (amountOf(session.scanIterator("g+oe", toAllPartitionKey(keyName), keyAmt)) < keyAmt) { - batchPut(session, "g+oe", keyName, keyAmt); - batchPut(session, "g+ie", keyName, keyAmt); + HgStoreTestUtil.println("-- test scan-batch multi-table --"); + if (HgStoreTestUtil.amountOf( + session.scanIterator("g+oe", HgStoreTestUtil.toAllPartitionKey(keyName), keyAmt)) < + keyAmt) { + HgStoreTestUtil.batchPut(session, "g+oe", keyName, keyAmt); + HgStoreTestUtil.batchPut(session, "g+ie", keyName, keyAmt); } - prefixes = Collections.singletonList(toAllPartitionKey(keyName)); + prefixes = Collections.singletonList(HgStoreTestUtil.toAllPartitionKey(keyName)); tableName = "g+oe,g+ie"; iterators = session.scanBatch( @@ -680,19 +745,21 @@ public class HgSessionManagerTest { ); Assert.assertEquals(900 * 2, HgStoreTestUtil.amountIn(iterators)); - println("-- test scan-batch per-key-limit --"); + HgStoreTestUtil.println("-- test scan-batch per-key-limit --"); tableName = "PER_KEY_LIMIT_TABLE"; keyName = "PER_KEY_LIMIT"; - if (amountOf(session.scanIterator(tableName, toAllPartitionKey(keyName), keyAmt)) < keyAmt) { - batchPut(session, tableName, keyName, keyAmt); + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), + keyAmt)) < keyAmt) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); } prefixes = Arrays.asList( - toAllPartitionKey(keyName + "-01") - , toAllPartitionKey(keyName + "-02") - , toAllPartitionKey(keyName + "-03") - , toAllPartitionKey(keyName + "-04") - , toAllPartitionKey(keyName + "-05") + HgStoreTestUtil.toAllPartitionKey(keyName + "-01") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-02") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-03") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-04") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-05") ); iterators = session.scanBatch( @@ -705,96 +772,98 @@ public class HgSessionManagerTest { // @Test public void truncate() { - println("--- test truncate ---"); + HgStoreTestUtil.println("--- test truncate ---"); String graph = "graph_truncate"; String tableName = "UNIT_TRUNCATE_1"; String keyName = "KEY_TRUNCATE"; HgStoreSession session = getStoreSession(graph); - batchPut(session, tableName, keyName, 100); - Assert.assertEquals(100, amountOf(session.scanIterator(tableName))); + HgStoreTestUtil.batchPut(session, tableName, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); String tableName2 = "UNIT_TRUNCATE_2"; - batchPut(session, tableName2, keyName, 100); - Assert.assertEquals(100, amountOf(session.scanIterator(tableName2))); + HgStoreTestUtil.batchPut(session, tableName2, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); session.truncate(); - Assert.assertEquals(0, amountOf(session.scanIterator(tableName))); - Assert.assertEquals(0, amountOf(session.scanIterator(tableName2))); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); } // @Test public void handle_table() { - println("--- test table ---"); + HgStoreTestUtil.println("--- test table ---"); String tableName = "UNIT_TABLE_" + System.currentTimeMillis(); - HgOwnerKey key = toOwnerKey("FOR-PUT-KEY"); - byte[] value = toBytes("FOR-PUT-VALUE"); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); HgStoreSession session = getStoreSession(); - println("-- test createTable --"); + HgStoreTestUtil.println("-- test createTable --"); session.createTable(tableName); - Assert.assertEquals(true, session.existsTable(tableName)); + Assert.assertTrue(session.existsTable(tableName)); - println("-- test deleteTable --"); + HgStoreTestUtil.println("-- test deleteTable --"); session.put(tableName, key, value); - Assert.assertEquals(toStr(value), toStr(session.get(tableName, key))); + Assert.assertEquals(HgStoreTestUtil.toStr(value), + HgStoreTestUtil.toStr(session.get(tableName, key))); session.deleteTable(tableName); - Assert.assertNotEquals(toStr(value), toStr(session.get(tableName, key))); - Assert.assertEquals(true, session.existsTable(tableName)); + Assert.assertNotEquals(HgStoreTestUtil.toStr(value), + HgStoreTestUtil.toStr(session.get(tableName, key))); + Assert.assertTrue(session.existsTable(tableName)); - println("-- test dropTable --"); - Assert.assertEquals(true, session.dropTable(tableName)); - Assert.assertEquals(false, session.existsTable(tableName)); + HgStoreTestUtil.println("-- test dropTable --"); + Assert.assertTrue(session.dropTable(tableName)); + Assert.assertFalse(session.existsTable(tableName)); - println("-- test existsTable --"); - Assert.assertEquals(false, session.existsTable(tableName)); + HgStoreTestUtil.println("-- test existsTable --"); + Assert.assertFalse(session.existsTable(tableName)); } // @Test public void tx() { - println("--- test tx ---"); + HgStoreTestUtil.println("--- test tx ---"); HgStoreSession session = getStoreSession(); String tableName = "UNIT_TABLE_TX"; String keyPrefix = "TX"; int keyAmt = 100; - println("-- unsupported tx operation --"); - println("- tx deleteTable -"); - batchPut(session, tableName, keyPrefix, keyAmt); - Assert.assertEquals(keyAmt, amountOf(session.scanIterator(tableName))); + HgStoreTestUtil.println("-- unsupported tx operation --"); + HgStoreTestUtil.println("- tx deleteTable -"); + HgStoreTestUtil.batchPut(session, tableName, keyPrefix, keyAmt); + Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); session.beginTx(); session.deleteTable(tableName); session.commit(); - Assert.assertEquals(0, amountOf(session.scanIterator(tableName))); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); - println("- tx dropTable -"); - batchPut(session, tableName, keyPrefix, keyAmt); - Assert.assertEquals(keyAmt, amountOf(session.scanIterator(tableName))); + HgStoreTestUtil.println("- tx dropTable -"); + HgStoreTestUtil.batchPut(session, tableName, keyPrefix, keyAmt); + Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); session.beginTx(); session.dropTable(tableName); session.commit(); - Assert.assertEquals(false, session.existsTable(tableName)); - Assert.assertEquals(0, amountOf(session.scanIterator(tableName))); + Assert.assertFalse(session.existsTable(tableName)); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); - println("- tx truncate -"); - batchPut(session, tableName, keyPrefix, keyAmt); - Assert.assertEquals(keyAmt, amountOf(session.scanIterator(tableName))); + HgStoreTestUtil.println("- tx truncate -"); + HgStoreTestUtil.batchPut(session, tableName, keyPrefix, keyAmt); + Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); session.beginTx(); session.truncate(); session.commit(); - Assert.assertEquals(false, session.existsTable(tableName)); - Assert.assertEquals(0, amountOf(session.scanIterator(tableName))); + Assert.assertFalse(session.existsTable(tableName)); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); //TODO : add others } //// @Test public void scanIterator_WithNonePartition() { - println("--- test scanIterator with none partition ---"); + HgStoreTestUtil.println("--- test scanIterator with none partition ---"); int count = 0; HgStoreSession session = getStoreSession(); @@ -803,7 +872,7 @@ public class HgSessionManagerTest { while (iterator.hasNext()) { count++; HgKvEntry entry = iterator.next(); - println(entry); + HgStoreTestUtil.println(entry); } } @@ -814,7 +883,7 @@ public class HgSessionManagerTest { //// @Test public void repeatedly_parallel_scan() { MetricX metrics = MetricX.ofStart(); - repeatedly_test(100, () -> parallel_scan()); + HgStoreTestUtil.repeatedly_test(100, () -> parallel_scan()); metrics.end(); log.info("*************************************************"); @@ -828,13 +897,15 @@ public class HgSessionManagerTest { log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize()); log.info("*************************************************"); - // runWaiting(); + // runWaiting(); } // @Test public void parallel_scan() { - println("--- test scanIterator in parallel ---"); - parallel_test(100, () -> {this.scan();}, t -> t.printStackTrace()); + HgStoreTestUtil.println("--- test scanIterator in parallel ---"); + HgStoreTestUtil.parallel_test(100, () -> { + this.scan(); + }, t -> t.printStackTrace()); } //// @Test @@ -851,16 +922,19 @@ public class HgSessionManagerTest { long start = System.currentTimeMillis(); for (int i = 0; i < amount; i++) { - HgOwnerKey key = toOwnerKey(keyPrefix + "-" + padLeftZeros(String.valueOf(i), length)); - byte[] value = toBytes(keyPrefix + "-V-" + i); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey( + keyPrefix + "-" + HgStoreTestUtil.padLeftZeros(String.valueOf(i), length)); + byte[] value = HgStoreTestUtil.toBytes(keyPrefix + "-V-" + i); session.put(tableName, key, value); if ((i + 1) % 100_000 == 0) { - println("---------- " + (i + 1) + " --------"); - println("Preparing took: " + (System.currentTimeMillis() - start) + " ms."); + HgStoreTestUtil.println("---------- " + (i + 1) + " --------"); + HgStoreTestUtil.println( + "Preparing took: " + (System.currentTimeMillis() - start) + " ms."); session.commit(); - println("Committing took: " + (System.currentTimeMillis() - start) + " ms."); + HgStoreTestUtil.println( + "Committing took: " + (System.currentTimeMillis() - start) + " ms."); start = System.currentTimeMillis(); session.beginTx(); } @@ -870,7 +944,7 @@ public class HgSessionManagerTest { session.commit(); } - Assert.assertEquals(amount, amountOf(session.scanIterator(tableName))); + Assert.assertEquals(amount, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); } @@ -925,17 +999,19 @@ public class HgSessionManagerTest { //// @Test public void benchmark_scanBatch() { - println("--- Benchmark scanBatch ---"); + HgStoreTestUtil.println("--- Benchmark scanBatch ---"); String tableName = "Benchmark_SCAN_BATCH"; String keyName = "SCAN-BATCH"; int keyAmt = 10_000_000; HgStoreSession session = getStoreSession(); - if (amountOf(session.scanIterator(tableName, toAllPartitionKey(keyName), 1)) < 1) { - batchPut(session, tableName, keyName, keyAmt); + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); } - println("-- Starting scan --"); + HgStoreTestUtil.println("-- Starting scan --"); // while (true) { MetricX metrics = MetricX.ofStart(); @@ -971,8 +1047,10 @@ public class HgSessionManagerTest { HgStoreSession session = getStoreSession(); /*Initialization*/ - if (amountOf(session.scanIterator(tableName, toAllPartitionKey(keyName), 1)) < 1) { - batchPut(session, tableName, keyName, amount); + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); } MetricX metricX = MetricX.ofStart(); @@ -984,7 +1062,7 @@ public class HgSessionManagerTest { count++; HgKvEntry entry = iterator.next(); if (count % (amount / 10) == 0) { - println(entry); + HgStoreTestUtil.println(entry); } if (count >= max) break; } @@ -1012,8 +1090,10 @@ public class HgSessionManagerTest { int amount = 1_000_000; HgStoreSession session = getStoreSession(); - if (amountOf(session.scanIterator(tableName, toAllPartitionKey(keyName), 1)) < 1) { - batchPut(session, tableName, keyName, amount); + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); } HgKvIterator iterator = null; /* server not over, enough, extreme -> close */ @@ -1021,20 +1101,20 @@ public class HgSessionManagerTest { iterator = session.scanIterator(tableName); iterator.next(); iterator.close(); - println("extreme loop: " + i); + HgStoreTestUtil.println("extreme loop: " + i); } - runWaiting(); + HgStoreTestUtil.runWaiting(); } //// @Test public void parallel_scan_close() { - parallel_test(10, () -> this.scan_close(), t -> t.printStackTrace()); + HgStoreTestUtil.parallel_test(10, () -> this.scan_close(), t -> t.printStackTrace()); } //// @Test public void repeat_parallel_scan_close() { - repeatedly_test(1000, () -> this.parallel_scan_close()); - runWaiting(); + HgStoreTestUtil.repeatedly_test(1000, () -> this.parallel_scan_close()); + HgStoreTestUtil.runWaiting(); } //// @Test diff --git a/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgKvStoreTest.java b/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgKvStoreTest.java similarity index 76% rename from hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgKvStoreTest.java rename to hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgKvStoreTest.java index 1b38cfc17..b32ddc116 100644 --- a/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgKvStoreTest.java +++ b/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgKvStoreTest.java @@ -1,21 +1,40 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.pd.client.PDClient; -import com.baidu.hugegraph.pd.client.PDConfig; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgStoreClient; -import com.baidu.hugegraph.store.HgStoreSession; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.client; + +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toBytes; +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toOwnerKey; +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toStr; +import static org.apache.hugegraph.store.util.HgStoreTestUtil.TABLE_NAME; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreClient; +import org.apache.hugegraph.store.HgStoreSession; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toBytes; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toOwnerKey; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toStr; -import static com.baidu.hugegraph.store.util.HgStoreTestUtil.TABLE_NAME; +import com.baidu.hugegraph.pd.client.PDClient; +import com.baidu.hugegraph.pd.client.PDConfig; + +import lombok.extern.slf4j.Slf4j; @Slf4j @@ -25,12 +44,12 @@ public class HgKvStoreTest { private HgStoreClient storeClient; private PDClient pdClient; - private String tableName0 = "cli-table0"; + private final String tableName0 = "cli-table0"; @Before - public void init(){ + public void init() { storeClient = HgStoreClient.create(PDConfig.of("127.0.0.1:8686") - .setEnableCache(true)); + .setEnableCache(true)); pdClient = storeClient.getPdClient(); } @@ -43,7 +62,7 @@ public class HgKvStoreTest { graph0.truncate(); graph1.truncate(); - for (int i=0; i < 100; i++) { + for (int i = 0; i < 100; i++) { HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i); byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i); graph0.put(TABLE_NAME, key, value0); @@ -54,14 +73,14 @@ public class HgKvStoreTest { HgKvIterator iterator = graph0.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); // log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); Assert.assertEquals(0, toStr(entry.value()).indexOf("g0")); } iterator = graph1.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); // log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); @@ -73,7 +92,7 @@ public class HgKvStoreTest { iterator = graph1.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); @@ -88,7 +107,7 @@ public class HgKvStoreTest { graph0.truncate(); graph1.truncate(); - for (int i=0; i < 10; i++) { + for (int i = 0; i < 10; i++) { HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i); byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i); graph0.put(TABLE_NAME, key, value0); @@ -99,14 +118,14 @@ public class HgKvStoreTest { HgKvIterator iterator = graph0.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); // log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); Assert.assertEquals(0, toStr(entry.value()).indexOf("g0")); } iterator = graph1.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); // log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); @@ -116,7 +135,7 @@ public class HgKvStoreTest { Assert.assertTrue(graph0.existsTable(TABLE_NAME)); iterator = graph0.scanIterator(TABLE_NAME); Assert.assertFalse(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); // Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); @@ -124,7 +143,7 @@ public class HgKvStoreTest { iterator = graph1.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); // log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); @@ -139,7 +158,7 @@ public class HgKvStoreTest { graph0.truncate(); graph1.truncate(); - for (int i=0; i < 10; i++) { + for (int i = 0; i < 10; i++) { HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i); byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i); graph0.put(TABLE_NAME, key, value0); @@ -150,14 +169,14 @@ public class HgKvStoreTest { HgKvIterator iterator = graph0.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); // log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); Assert.assertEquals(0, toStr(entry.value()).indexOf("g0")); } iterator = graph1.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); // log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); @@ -167,7 +186,7 @@ public class HgKvStoreTest { Assert.assertTrue(graph0.existsTable(TABLE_NAME)); iterator = graph0.scanIterator(TABLE_NAME); Assert.assertFalse(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); // Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); @@ -175,7 +194,7 @@ public class HgKvStoreTest { iterator = graph1.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); // log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); @@ -189,7 +208,7 @@ public class HgKvStoreTest { HgStoreSession graph0 = storeClient.openSession("hugegraph0"); graph0.truncate(); - for (int i=0; i < 10; i++) { + for (int i = 0; i < 10; i++) { HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i); byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i); graph0.put(TABLE_NAME, key, value0); @@ -197,7 +216,7 @@ public class HgKvStoreTest { HgKvIterator iterator = graph0.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); Assert.assertEquals(0, toStr(entry.value()).indexOf("g0")); diff --git a/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreClientBase.java b/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreClientBase.java new file mode 100644 index 000000000..9af6d5b2a --- /dev/null +++ b/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreClientBase.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +import org.apache.hugegraph.store.HgStoreClient; +import org.apache.hugegraph.store.HgStoreSession; +import org.junit.Before; + +import com.baidu.hugegraph.pd.client.PDClient; +import com.baidu.hugegraph.pd.client.PDConfig; + +public class HgStoreClientBase { + + protected HgStoreClient storeClient; + protected PDClient pdClient; + + protected static String Graph_Name = "testGraphName"; + protected static String Table_Name = "testTableName"; + + @Before + public void init() { + storeClient = HgStoreClient.create(PDConfig.of("127.0.0.1:8686") + .setEnableCache(true)); + pdClient = storeClient.getPdClient(); + + HgStoreSession session = storeClient.openSession(Graph_Name); + session.dropTable(Table_Name); + } +} diff --git a/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreClientTest.java b/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreClientTest.java similarity index 68% rename from hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreClientTest.java rename to hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreClientTest.java index 95d87b8d1..ada03b831 100644 --- a/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreClientTest.java +++ b/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreClientTest.java @@ -1,38 +1,54 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; + +import java.util.List; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvStore; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreClient; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; import com.baidu.hugegraph.pd.client.PDClient; import com.baidu.hugegraph.pd.client.PDConfig; import com.baidu.hugegraph.pd.common.PDException; import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgKvStore; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgStoreClient; -import com.baidu.hugegraph.store.HgStoreSession; -import com.baidu.hugegraph.store.util.HgStoreTestUtil; -import org.apache.commons.codec.binary.Hex; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -// import org.junit.Test; - -import java.util.List; - -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; public class HgStoreClientTest { private HgStoreClient storeClient; private PDClient pdClient; public final static byte[] EMPTY_BYTES = new byte[0]; - private static String Graph_Name = "testGraphName"; + private static final String Graph_Name = "testGraphName"; private static String Table_Name = "testTableName"; @Before public void init() { storeClient = HgStoreClient.create(PDConfig.of("127.0.0.1:8686") - .setEnableCache(true)); + .setEnableCache(true)); pdClient = storeClient.getPdClient(); } @@ -84,8 +100,12 @@ public class HgStoreClientTest { byte[] start = new byte[]{0x0}; byte[] end = new byte[]{-1}; try (HgKvIterator iterator = session.scanIterator(Table_Name, - HgOwnerKey.of(ALL_PARTITION_OWNER, start), - HgOwnerKey.of(ALL_PARTITION_OWNER, end))) { + HgOwnerKey.of( + ALL_PARTITION_OWNER, + start), + HgOwnerKey.of( + ALL_PARTITION_OWNER, + end))) { iterator.seek(position); while (iterator.hasNext()) { iterator.next(); @@ -112,8 +132,8 @@ public class HgStoreClientTest { HgStoreSession session = storeClient.openSession(Graph_Name); int ownerCode = 1; HgStoreTestUtil.batchPut(session, Table_Name, "T", 10, (key) -> { - return HgStoreTestUtil.toOwnerKey(ownerCode, key); - } + return HgStoreTestUtil.toOwnerKey(ownerCode, key); + } ); try (HgKvIterator iterators = session.scanIterator(Table_Name)) { // while (iterators.hasNext()){ @@ -176,10 +196,13 @@ public class HgStoreClientTest { HgStoreSession session = storeClient.openSession("DEFAULT/hugegraph/g"); for (Metapb.Partition partition : partitions) { try (HgKvIterator iterators = session.scanIterator("g+v", - (int) (partition.getStartKey()), (int) (partition.getEndKey()), - HgKvStore.SCAN_HASHCODE, EMPTY_BYTES)) { + (int) (partition.getStartKey()), + (int) (partition.getEndKey()), + HgKvStore.SCAN_HASHCODE, + EMPTY_BYTES)) { - System.out.println(" " + partition.getId() + " " + HgStoreTestUtil.amountOf(iterators)); + System.out.println( + " " + partition.getId() + " " + HgStoreTestUtil.amountOf(iterators)); } } } diff --git a/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreNodeSessionTest.java b/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreNodeSessionTest.java new file mode 100644 index 000000000..eb4d633ab --- /dev/null +++ b/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreNodeSessionTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +import static org.apache.hugegraph.store.util.HgStoreTestUtil.GRAPH_NAME; +import static org.apache.hugegraph.store.util.HgStoreTestUtil.amountOf; +import static org.apache.hugegraph.store.util.HgStoreTestUtil.batchPut; +import static org.apache.hugegraph.store.util.HgStoreTestUtil.println; + +import org.apache.hugegraph.store.HgStoreSession; +import org.junit.Assert; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/12 + */ +public class HgStoreNodeSessionTest { + private static final HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); + private static HgStoreNode node; + + // @BeforeClass + public static void init() { + node = nodeManager.addNode(GRAPH_NAME, + nodeManager.getNodeBuilder().setAddress("localhost:9080") + .build()); + } + + private HgStoreNode getOneNode() { + return node; + } + + private static HgStoreSession getStoreSession() { + return node.openSession(GRAPH_NAME); + } + + // @Test + public void truncate() { + println("--- test truncate ---"); + String tableName = "UNIT_TRUNCATE_1"; + String keyName = "KEY_TRUNCATE"; + + HgStoreSession session = getStoreSession(); + batchPut(session, tableName, keyName, 100); + Assert.assertEquals(100, amountOf(session.scanIterator(tableName))); + + String tableName2 = "UNIT_TRUNCATE_2"; + batchPut(session, tableName2, keyName, 100); + Assert.assertEquals(100, amountOf(session.scanIterator(tableName2))); + + + session.truncate(); + Assert.assertEquals(0, amountOf(session.scanIterator(tableName))); + Assert.assertEquals(0, amountOf(session.scanIterator(tableName2))); + } + +} \ No newline at end of file diff --git a/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreNodeStateTest.java b/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreNodeStateTest.java new file mode 100644 index 000000000..62dd972dd --- /dev/null +++ b/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreNodeStateTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +// import org.junit.Test; + +import static org.apache.hugegraph.store.util.HgStoreTestUtil.GRAPH_NAME; +import static org.apache.hugegraph.store.util.HgStoreTestUtil.println; + +import java.util.concurrent.atomic.AtomicInteger; + + +public class HgStoreNodeStateTest { + private final static HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); + static int nodeNumber = 0; + + static { + registerNode(GRAPH_NAME, Long.valueOf(nodeNumber++), "localhost:9180"); + registerNode(GRAPH_NAME, Long.valueOf(nodeNumber++), "localhost:9280"); + registerNode(GRAPH_NAME, Long.valueOf(nodeNumber++), "localhost:9380"); + } + + private static void registerNode(String graphName, Long nodeId, String address) { + nodeManager.addNode(graphName, + nodeManager.getNodeBuilder().setNodeId(nodeId).setAddress(address) + .build()); + } + + + // @Test + public void isNodeHealthy() { + AtomicInteger count = new AtomicInteger(0); + + for (int i = 0; i < Integer.MAX_VALUE; i++) { + nodeManager.getStoreNodes(GRAPH_NAME) + .stream().map( + node -> { + println(node.getNodeId() + " " + node.getAddress() + "is " + + "healthy: " + + node.isHealthy()); + return node.isHealthy(); + } + ).count(); + + Thread.yield(); + + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + +} \ No newline at end of file diff --git a/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreNodeStreamTest.java b/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreNodeStreamTest.java similarity index 64% rename from hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreNodeStreamTest.java rename to hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreNodeStreamTest.java index 1716d304a..675034dfa 100644 --- a/hg-store-client/src/test/java/com/baidu/hugegraph/store/client/HgStoreNodeStreamTest.java +++ b/hg-store-client/src/test/java/org/apache/hugegraph/store/client/HgStoreNodeStreamTest.java @@ -1,24 +1,46 @@ -package com.baidu.hugegraph.store.client; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.*; -import org.junit.Assert; -// import org.junit.BeforeClass; -// import org.junit.Test; +package org.apache.hugegraph.store.client; + +import static org.apache.hugegraph.store.util.HgStoreTestUtil.GRAPH_NAME; +import static org.apache.hugegraph.store.util.HgStoreTestUtil.batchPut; +import static org.apache.hugegraph.store.util.HgStoreTestUtil.println; +import static org.apache.hugegraph.store.util.HgStoreTestUtil.toOwnerKey; import java.util.NoSuchElementException; -import static com.baidu.hugegraph.store.util.HgStoreTestUtil.*; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgStoreSession; +import org.junit.Assert; /** * @author lynn.bond@hotmail.com created on 2021/10/12 */ public class HgStoreNodeStreamTest { - private static HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); + private static final HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); private static HgStoreNode node; // @BeforeClass public static void init() { - node = nodeManager.addNode(GRAPH_NAME, nodeManager.getNodeBuilder().setAddress("localhost:9080").build()); + node = nodeManager.addNode(GRAPH_NAME, + nodeManager.getNodeBuilder().setAddress("localhost:9080") + .build()); } private HgStoreNode getOneNode() { @@ -37,28 +59,31 @@ public class HgStoreNodeStreamTest { String tableName = "UNIT_SCAN"; String keyName = "SCAN-ITER"; HgStoreSession session = getStoreSession(); - batchPut(session,tableName, keyName, 10000); + batchPut(session, tableName, keyName, 10000); int count = 0; int limit = 0; int max = 99999; HgKvIterator iterator = null; println("-- test 0 element --"); - iterator = session.scanIterator(tableName, toOwnerKey("__SCAN-001"), toOwnerKey("__SCAN-100"), 0); + iterator = + session.scanIterator(tableName, toOwnerKey("__SCAN-001"), toOwnerKey("__SCAN-100"), + 0); Assert.assertFalse(iterator.hasNext()); try { iterator.next(); - Assert.assertTrue(false); + Assert.fail(); } catch (Throwable t) { println("-- test NoSuchElementException --"); - Assert.assertTrue(NoSuchElementException.class.isInstance(t)); + Assert.assertTrue(t instanceof NoSuchElementException); } println("-- test limit 1 to 10 --"); for (int i = 1; i <= 10; i++) { println("- limit " + i + " -"); limit = i; - iterator = session.scanIterator(tableName, toOwnerKey(keyName + "-0"), toOwnerKey(keyName + "-1"), limit); + iterator = session.scanIterator(tableName, toOwnerKey(keyName + "-0"), + toOwnerKey(keyName + "-1"), limit); count = 0; while (iterator.hasNext()) { count++; @@ -73,7 +98,8 @@ public class HgStoreNodeStreamTest { println("- limit " + i + " -"); limit = i; iterator = session.scanIterator(tableName, - toOwnerKey(keyName + "-00001"), toOwnerKey(keyName + "-00005"), limit); + toOwnerKey(keyName + "-00001"), + toOwnerKey(keyName + "-00005"), limit); count = 0; while (iterator.hasNext()) { count++; @@ -90,7 +116,8 @@ public class HgStoreNodeStreamTest { println("-- test limit 0 (no limit) --"); limit = 0; - iterator = session.scanIterator(tableName, toOwnerKey(keyName + "-0"), toOwnerKey(keyName + "-1"), limit); + iterator = session.scanIterator(tableName, toOwnerKey(keyName + "-0"), + toOwnerKey(keyName + "-1"), limit); count = 0; while (iterator.hasNext()) { diff --git a/hg-store-client/src/test/java/org/apache/hugegraph/store/common/BitsTest.java b/hg-store-client/src/test/java/org/apache/hugegraph/store/common/BitsTest.java new file mode 100644 index 000000000..ff15db659 --- /dev/null +++ b/hg-store-client/src/test/java/org/apache/hugegraph/store/common/BitsTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.common; + +import org.apache.hugegraph.store.term.Bits; +import org.junit.Assert; +// import org.junit.Test; + +public class BitsTest { + // @Test + public void test() { + for (int i = 0; i < Integer.MAX_VALUE; i = i + 10) { + byte[] val = new byte[4]; + Bits.putInt(val, 0, i); + int n = Bits.getInt(val, 0); + Assert.assertEquals(i, n); + } + } +} diff --git a/hg-store-client/src/test/java/com/baidu/hugegraph/store/test/ChangeShardNumTest.java b/hg-store-client/src/test/java/org/apache/hugegraph/store/test/ChangeShardNumTest.java similarity index 50% rename from hg-store-client/src/test/java/com/baidu/hugegraph/store/test/ChangeShardNumTest.java rename to hg-store-client/src/test/java/org/apache/hugegraph/store/test/ChangeShardNumTest.java index b7f8a0354..55adc6efb 100644 --- a/hg-store-client/src/test/java/com/baidu/hugegraph/store/test/ChangeShardNumTest.java +++ b/hg-store-client/src/test/java/org/apache/hugegraph/store/test/ChangeShardNumTest.java @@ -1,13 +1,31 @@ -package com.baidu.hugegraph.store.test; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.test; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.client.HgStoreClientBase; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; import com.baidu.hugegraph.pd.common.PDException; import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgStoreSession; -import com.baidu.hugegraph.store.client.HgStoreClientBase; -import com.baidu.hugegraph.store.util.HgStoreTestUtil; -import org.junit.Assert; // import org.junit.Test; /** @@ -21,7 +39,7 @@ public class ChangeShardNumTest extends HgStoreClientBase { HgStoreSession session = storeClient.openSession(Graph_Name); HgStoreTestUtil.batchPut(session, Table_Name, "testKey", number); - try(HgKvIterator iterators = session.scanIterator(Table_Name)){ + try (HgKvIterator iterators = session.scanIterator(Table_Name)) { Assert.assertEquals(number, HgStoreTestUtil.amountOf(iterators)); } @@ -38,7 +56,7 @@ public class ChangeShardNumTest extends HgStoreClientBase { HgStoreSession session = storeClient.openSession(Graph_Name); HgStoreTestUtil.batchPut(session, Table_Name, "testKey", number); - try(HgKvIterator iterators = session.scanIterator(Table_Name)){ + try (HgKvIterator iterators = session.scanIterator(Table_Name)) { Assert.assertEquals(number, HgStoreTestUtil.amountOf(iterators)); } diff --git a/hg-store-client/src/test/java/com/baidu/hugegraph/store/util/HgStoreTestUtil.java b/hg-store-client/src/test/java/org/apache/hugegraph/store/util/HgStoreTestUtil.java similarity index 87% rename from hg-store-client/src/test/java/com/baidu/hugegraph/store/util/HgStoreTestUtil.java rename to hg-store-client/src/test/java/org/apache/hugegraph/store/util/HgStoreTestUtil.java index 393e41afe..b18f98f55 100644 --- a/hg-store-client/src/test/java/com/baidu/hugegraph/store/util/HgStoreTestUtil.java +++ b/hg-store-client/src/test/java/org/apache/hugegraph/store/util/HgStoreTestUtil.java @@ -1,14 +1,30 @@ -package com.baidu.hugegraph.store.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.store.*; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import com.baidu.hugegraph.store.client.util.MetricX; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.util; import java.io.Closeable; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -17,6 +33,17 @@ import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.MetricX; + +import com.baidu.hugegraph.store.*; + +import lombok.extern.slf4j.Slf4j; + /** * @author lynn.bond@hotmail.com created on 2021/10/12 */ @@ -33,7 +60,8 @@ public class HgStoreTestUtil { return batchPut(session, keyPrefix, 100); } - public static Map batchPut(HgStoreSession session, String keyPrefix, int loop) { + public static Map batchPut(HgStoreSession session, String keyPrefix, + int loop) { return batchPut(session, TABLE_NAME, keyPrefix, loop); } @@ -44,7 +72,8 @@ public class HgStoreTestUtil { public static Map batchPut(HgStoreSession session, String tableName , byte[] keyPrefix, int loop) { - return batchPut(session, tableName, keyPrefix, loop, 1, (prefix, key) -> toOwnerKey(prefix, key)); + return batchPut(session, tableName, keyPrefix, loop, 1, + (prefix, key) -> toOwnerKey(prefix, key)); } public static Map batchPut(HgStoreSession session, String tableName @@ -349,10 +378,10 @@ public class HgStoreTestUtil { break; } } - if (Closeable.class.isInstance(iterator)) { + if (iterator instanceof Closeable) { try { ((Closeable) iterator).close(); - }catch (Exception e){ + } catch (Exception e) { } } @@ -361,7 +390,8 @@ public class HgStoreTestUtil { } public static int amountIn(List iterators) { - return iterators.parallelStream().map(e -> HgStoreTestUtil.amountOf(e)).reduce(0, Integer::sum); + return iterators.parallelStream().map(e -> HgStoreTestUtil.amountOf(e)) + .reduce(0, Integer::sum); } @@ -379,7 +409,8 @@ public class HgStoreTestUtil { } } - public static void parallel_test(int threads, Runnable runner, Consumer throwableConsumer) { + public static void parallel_test(int threads, Runnable runner, + Consumer throwableConsumer) { int threadsAmount = threads; CountDownLatch countDownLatch = new CountDownLatch(threadsAmount); ExecutorService pool = Executors.newFixedThreadPool(threadsAmount); diff --git a/hg-store-client/src/test/resources/log4j2.xml b/hg-store-client/src/test/resources/log4j2.xml index 9f805af75..bd88bec75 100644 --- a/hg-store-client/src/test/resources/log4j2.xml +++ b/hg-store-client/src/test/resources/log4j2.xml @@ -1,5 +1,22 @@ + + @@ -28,7 +45,7 @@ - + @@ -36,8 +53,8 @@ - - + + @@ -56,7 +73,7 @@ - + @@ -64,8 +81,8 @@ - - + + diff --git a/hg-store-common/.mvn/wrapper/maven-wrapper.properties b/hg-store-common/.mvn/wrapper/maven-wrapper.properties index 642d572ce..a0c0a3761 100644 --- a/hg-store-common/.mvn/wrapper/maven-wrapper.properties +++ b/hg-store-common/.mvn/wrapper/maven-wrapper.properties @@ -1,2 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar diff --git a/hg-store-common/pom.xml b/hg-store-common/pom.xml index b05b91161..e0d43c6ec 100644 --- a/hg-store-common/pom.xml +++ b/hg-store-common/pom.xml @@ -1,15 +1,31 @@ + + 4.0.0 - - com.baidu.hugegraph + org.apache.hugegraph hugegraph-store-root - 3.6.3-SNAPSHOT + ${revision} hg-store-common diff --git a/hg-store-common/src/main/java/com/baidu/hugegraph/store/buffer/KVByteBuffer.java b/hg-store-common/src/main/java/com/baidu/hugegraph/store/buffer/KVByteBuffer.java deleted file mode 100644 index f41b15559..000000000 --- a/hg-store-common/src/main/java/com/baidu/hugegraph/store/buffer/KVByteBuffer.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.baidu.hugegraph.store.buffer; - -import java.nio.ByteBuffer; - -public class KVByteBuffer { - ByteBuffer buffer; - public KVByteBuffer(int capacity){ - buffer = ByteBuffer.allocate(capacity); - } - - public KVByteBuffer(byte[] buffer){ - this.buffer = ByteBuffer.wrap(buffer); - } - - public KVByteBuffer(ByteBuffer buffer){ - this.buffer = buffer; - } - public void clear(){ - this.buffer.clear(); - } - - public KVByteBuffer flip(){ - buffer.flip(); - return this; - } - public ByteBuffer getBuffer(){ return buffer;} - - public ByteBuffer copyBuffer(){ - byte[] buf = new byte[buffer.position()]; - System.arraycopy(buffer.array(), 0, buf, 0, buffer.position()); - return ByteBuffer.wrap(buf); - } - public void put(byte data){ - buffer.put(data); - } - - public void put(byte[] data){ - if ( data != null ) { - buffer.putInt(data.length); - buffer.put(data); - } - } - - public byte[] getBytes(){ - int len = buffer.getInt(); - byte[] data = new byte[len]; - buffer.get(data); - return data; - } - - public byte get(){ - return buffer.get(); - } - public void putInt(int data){ - buffer.putInt(data); - } - - public int getInt(){ - return buffer.getInt(); - } - public byte[] array(){ - return this.buffer.array(); - } - - public int position(){ return this.buffer.position();} - - public final boolean hasRemaining(){ return this.buffer.hasRemaining();} -} diff --git a/hg-store-common/src/main/java/com/baidu/hugegraph/store/term/HgTriple.java b/hg-store-common/src/main/java/com/baidu/hugegraph/store/term/HgTriple.java deleted file mode 100644 index ea8e1ecfc..000000000 --- a/hg-store-common/src/main/java/com/baidu/hugegraph/store/term/HgTriple.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.baidu.hugegraph.store.term; - -import java.util.Objects; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/19 - */ -public class HgTriple { - private int hash=-1; - private X x; - private Y y; - private Z z; - - public HgTriple(X x, Y y, Z z) { - this.x = x; - this.y = y; - this.z = z; - } - - public X getX() { - return x; - } - - public Y getY() { - return y; - } - - public Z getZ() { - return z; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - HgTriple hgTriple = (HgTriple) o; - return Objects.equals(x, hgTriple.x) && Objects.equals(y, hgTriple.y) && Objects.equals(z, hgTriple.z); - } - - @Override - public int hashCode() { - if(hash==-1){ - hash= Objects.hash(x, y, z); - } - return this.hash; - } - - @Override - public String toString() { - return "HgTriple{" + - "x=" + x + - ", y=" + y + - ", z=" + z + - '}'; - } -} diff --git a/hg-store-common/src/main/java/com/baidu/hugegraph/store/buffer/ByteBufferAllocator.java b/hg-store-common/src/main/java/org/apache/hugegraph/store/buffer/ByteBufferAllocator.java similarity index 52% rename from hg-store-common/src/main/java/com/baidu/hugegraph/store/buffer/ByteBufferAllocator.java rename to hg-store-common/src/main/java/org/apache/hugegraph/store/buffer/ByteBufferAllocator.java index 33285a079..29d5bacfd 100644 --- a/hg-store-common/src/main/java/com/baidu/hugegraph/store/buffer/ByteBufferAllocator.java +++ b/hg-store-common/src/main/java/org/apache/hugegraph/store/buffer/ByteBufferAllocator.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.buffer; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.buffer; import java.nio.ByteBuffer; @@ -18,7 +35,7 @@ public class ByteBufferAllocator { // 空闲队列 final BlockingQueue freeQueue = new LinkedBlockingQueue<>(); - public ByteBufferAllocator(int cap, int count){ + public ByteBufferAllocator(int cap, int count) { this.capacity = cap; this.maxCount = count; this.totalCount = new AtomicInteger(0); @@ -27,9 +44,9 @@ public class ByteBufferAllocator { public ByteBuffer get() throws InterruptedException { ByteBuffer buffer = null; while (buffer == null) { - if (freeQueue.size() > 0) + if (freeQueue.size() > 0) { buffer = freeQueue.poll(); - else if (totalCount.get() < maxCount) { + } else if (totalCount.get() < maxCount) { buffer = ByteBuffer.allocate(capacity); totalCount.incrementAndGet(); } else { @@ -40,7 +57,7 @@ public class ByteBufferAllocator { } public void release(ByteBuffer buffer) { - if ( freeQueue.size() < maxCount) { + if (freeQueue.size() < maxCount) { buffer.clear(); freeQueue.add(buffer); } diff --git a/hg-store-common/src/main/java/org/apache/hugegraph/store/buffer/KVByteBuffer.java b/hg-store-common/src/main/java/org/apache/hugegraph/store/buffer/KVByteBuffer.java new file mode 100644 index 000000000..da9025e3c --- /dev/null +++ b/hg-store-common/src/main/java/org/apache/hugegraph/store/buffer/KVByteBuffer.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.buffer; + +import java.nio.ByteBuffer; + +public class KVByteBuffer { + ByteBuffer buffer; + + public KVByteBuffer(int capacity) { + buffer = ByteBuffer.allocate(capacity); + } + + public KVByteBuffer(byte[] buffer) { + this.buffer = ByteBuffer.wrap(buffer); + } + + public KVByteBuffer(ByteBuffer buffer) { + this.buffer = buffer; + } + + public void clear() { + this.buffer.clear(); + } + + public KVByteBuffer flip() { + buffer.flip(); + return this; + } + + public ByteBuffer getBuffer() { + return buffer; + } + + public ByteBuffer copyBuffer() { + byte[] buf = new byte[buffer.position()]; + System.arraycopy(buffer.array(), 0, buf, 0, buffer.position()); + return ByteBuffer.wrap(buf); + } + + public void put(byte data) { + buffer.put(data); + } + + public void put(byte[] data) { + if (data != null) { + buffer.putInt(data.length); + buffer.put(data); + } + } + + public byte[] getBytes() { + int len = buffer.getInt(); + byte[] data = new byte[len]; + buffer.get(data); + return data; + } + + public byte get() { + return buffer.get(); + } + + public void putInt(int data) { + buffer.putInt(data); + } + + public int getInt() { + return buffer.getInt(); + } + + public byte[] array() { + return this.buffer.array(); + } + + public int position() { + return this.buffer.position(); + } + + public final boolean hasRemaining() { + return this.buffer.hasRemaining(); + } +} diff --git a/hg-store-common/src/main/java/com/baidu/hugegraph/store/term/Bits.java b/hg-store-common/src/main/java/org/apache/hugegraph/store/term/Bits.java similarity index 58% rename from hg-store-common/src/main/java/com/baidu/hugegraph/store/term/Bits.java rename to hg-store-common/src/main/java/org/apache/hugegraph/store/term/Bits.java index b286abea0..c38d4b679 100644 --- a/hg-store-common/src/main/java/com/baidu/hugegraph/store/term/Bits.java +++ b/hg-store-common/src/main/java/org/apache/hugegraph/store/term/Bits.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.term; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.term; import java.nio.ByteBuffer; @@ -17,12 +34,13 @@ public class Bits { buf[offSet + 2] = (byte) (x >> 8); buf[offSet + 3] = (byte) (x); } + /** * 大头字节序读取short */ public static int getShort(byte[] buf, int offSet) { int x = buf[offSet] & 0xff; - x = (x << 8) + (buf[offSet + 1] & 0xff); + x = (x << 8) + (buf[offSet + 1] & 0xff); return x; } @@ -33,6 +51,7 @@ public class Bits { + (buf[offSet + 3] & 0xff); return x; } + public static void put(byte[] buf, int offSet, byte[] srcBuf) { System.arraycopy(srcBuf, 0, buf, offSet, srcBuf.length); } diff --git a/hg-store-common/src/main/java/com/baidu/hugegraph/store/term/HgPair.java b/hg-store-common/src/main/java/org/apache/hugegraph/store/term/HgPair.java similarity index 62% rename from hg-store-common/src/main/java/com/baidu/hugegraph/store/term/HgPair.java rename to hg-store-common/src/main/java/org/apache/hugegraph/store/term/HgPair.java index a5e8da238..9c893ae39 100644 --- a/hg-store-common/src/main/java/com/baidu/hugegraph/store/term/HgPair.java +++ b/hg-store-common/src/main/java/org/apache/hugegraph/store/term/HgPair.java @@ -1,13 +1,31 @@ -package com.baidu.hugegraph.store.term; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.term; import java.io.Serializable; import java.util.Objects; /** * Copy from javafx.util:Pair + * * @author lynn.bond@hotmail.com created on 2021/10/15 */ -public class HgPair implements Serializable { +public class HgPair implements Serializable { /** * Key of this Pair. @@ -16,10 +34,16 @@ public class HgPair implements Serializable { /** * Gets the key for this pair. + * * @return key for this pair */ - public K getKey() { return key; } - public void setKey(K key){ this.key = key; } + public K getKey() { + return key; + } + + public void setKey(K key) { + this.key = key; + } /** * Value of this this Pair. @@ -28,20 +52,28 @@ public class HgPair implements Serializable { /** * Gets the value for this pair. + * * @return value for this pair */ - public V getValue() { return value; } - public void setValue(V value){ this.value = value;} + public V getValue() { + return value; + } - public HgPair(){ + public void setValue(V value) { + this.value = value; + } + + public HgPair() { } + /** * Creates a new pair - * @param key The key for this pair + * + * @param key The key for this pair * @param value The value to use for this pair */ - public HgPair(K key,V value) { + public HgPair(K key, V value) { this.key = key; this.value = value; } @@ -52,7 +84,7 @@ public class HgPair implements Serializable { * *

    The default name/value delimiter '=' is always used.

    * - * @return String representation of this Pair + * @return String representation of this Pair */ @Override public String toString() { @@ -89,7 +121,7 @@ public class HgPair implements Serializable { * both the names and values are equal.

    * * @param o the Object to test for - * equality with this Pair + * equality with this Pair * @return true if the given Object is * equal to this Pair else false */ @@ -99,8 +131,8 @@ public class HgPair implements Serializable { if (o instanceof HgPair) { HgPair pair = (HgPair) o; if (!Objects.equals(key, pair.key)) return false; - if (!Objects.equals(value, pair.value)) return false; - return true; + return Objects.equals(value, pair.value); } return false; - }} \ No newline at end of file + } +} \ No newline at end of file diff --git a/hg-store-common/src/main/java/org/apache/hugegraph/store/term/HgTriple.java b/hg-store-common/src/main/java/org/apache/hugegraph/store/term/HgTriple.java new file mode 100644 index 000000000..9a817f894 --- /dev/null +++ b/hg-store-common/src/main/java/org/apache/hugegraph/store/term/HgTriple.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.term; + +import java.util.Objects; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/19 + */ +public class HgTriple { + private int hash = -1; + private final X x; + private final Y y; + private final Z z; + + public HgTriple(X x, Y y, Z z) { + this.x = x; + this.y = y; + this.z = z; + } + + public X getX() { + return x; + } + + public Y getY() { + return y; + } + + public Z getZ() { + return z; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + HgTriple hgTriple = (HgTriple) o; + return Objects.equals(x, hgTriple.x) && Objects.equals(y, hgTriple.y) && + Objects.equals(z, hgTriple.z); + } + + @Override + public int hashCode() { + if (hash == -1) { + hash = Objects.hash(x, y, z); + } + return this.hash; + } + + @Override + public String toString() { + return "HgTriple{" + + "x=" + x + + ", y=" + y + + ", z=" + z + + '}'; + } +} diff --git a/hg-store-core/build.sh b/hg-store-core/build.sh index 8a50e35a7..4265ff844 100755 --- a/hg-store-core/build.sh +++ b/hg-store-core/build.sh @@ -1,5 +1,22 @@ #!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + set -e diff --git a/hg-store-core/pom.xml b/hg-store-core/pom.xml index c3844beb0..7e701d824 100644 --- a/hg-store-core/pom.xml +++ b/hg-store-core/pom.xml @@ -1,12 +1,29 @@ + + 4.0.0 - com.baidu.hugegraph + org.apache.hugegraph hugegraph-store-root - 3.6.3-SNAPSHOT + ${revision} hg-store-core @@ -69,8 +86,9 @@ - com.baidu.hugegraph + org.apache.hugegraph hg-store-rocksdb + ${revision} org.apache.logging.log4j @@ -79,18 +97,19 @@ - com.baidu.hugegraph + org.apache.hugegraph hg-pd-client - ${project.version} + ${revision} compile - com.baidu.hugegraph + org.apache.hugegraph hg-store-common + ${revision} - com.baidu.hugegraph + org.apache.hugegraph hugegraph-core ${hugegraph.server.version} @@ -100,11 +119,11 @@ 2.8.9     - com.baidu.hugegraph + org.apache.hugegraph hg-store-grpc - com.baidu.hugegraph + org.apache.hugegraph hg-store-client test diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/HgStoreStateListener.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/HgStoreStateListener.java deleted file mode 100644 index 43c9c045a..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/HgStoreStateListener.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.baidu.hugegraph.store; - -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.meta.Store; - -public interface HgStoreStateListener { - void stateChanged(Store store, Metapb.StoreState oldState, Metapb.StoreState newState); -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/PartitionInstructionProcessor.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/PartitionInstructionProcessor.java deleted file mode 100644 index 232aca64f..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/PartitionInstructionProcessor.java +++ /dev/null @@ -1,305 +0,0 @@ -package com.baidu.hugegraph.store; - -import com.alipay.sofa.jraft.Status; -import com.alipay.sofa.jraft.util.Utils; -import com.baidu.hugegraph.pd.common.PDException; -import com.baidu.hugegraph.pd.grpc.MetaTask; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.pd.grpc.pulse.ChangeShard; -import com.baidu.hugegraph.pd.grpc.pulse.CleanPartition; -import com.baidu.hugegraph.pd.grpc.pulse.DbCompaction; -import com.baidu.hugegraph.pd.grpc.pulse.MovePartition; -import com.baidu.hugegraph.pd.grpc.pulse.PartitionKeyRange; -import com.baidu.hugegraph.pd.grpc.pulse.SplitPartition; -import com.baidu.hugegraph.pd.grpc.pulse.TransferLeader; -import com.baidu.hugegraph.store.cmd.CleanDataRequest; -import com.baidu.hugegraph.store.cmd.DbCompactionRequest; -import com.baidu.hugegraph.store.meta.MetadataKeyHelper; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.pd.PartitionInstructionListener; -import com.baidu.hugegraph.store.raft.RaftClosure; -import com.baidu.hugegraph.store.raft.RaftOperation; -import com.baidu.hugegraph.util.Log; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.slf4j.Logger; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -/** - * PD发给Store的分区指令处理器 - */ -public class PartitionInstructionProcessor implements PartitionInstructionListener { - private static final Logger LOG = Log.logger(PartitionInstructionProcessor.class); - private HgStoreEngine storeEngine; - private final ExecutorService threadPool; - - public PartitionInstructionProcessor(HgStoreEngine storeEngine) { - this.storeEngine = storeEngine; - ThreadFactory namedThreadFactory = new ThreadFactoryBuilder().setNameFormat("instruct-process-pool-%d").build(); - threadPool = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors(), - 1000000, - 180L, - TimeUnit.SECONDS, - new LinkedBlockingQueue<>(1000000), - namedThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); - } - - @Override - public void onChangeShard(long taskId, Partition partition, ChangeShard changeShard, Consumer consumer) { - PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId()); - - if (engine != null) { - // 清理所有的任务，有失败的情况 - engine.getTaskManager().deleteTask(partition.getId(), MetaTask.TaskType.Change_Shard.name()); - } - - if (engine != null && engine.isLeader()) { - LOG.info("Partition {}-{} Receive change shard message, {}", partition.getGraphName(), - partition.getId(), changeShard); - String graphName = partition.getGraphName(); - int partitionId = partition.getId(); - MetaTask.Task task = MetaTask.Task.newBuilder() - .setId(taskId) - .setPartition(partition.getProtoObj()) - .setType(MetaTask.TaskType.Change_Shard) - .setState(MetaTask.TaskState.Task_Ready) - .setChangeShard(changeShard) - .build(); - try { - storeEngine.addRaftTask(graphName, partitionId, - RaftOperation.create(RaftOperation.SYNC_PARTITION_TASK, task), - new RaftClosure() { - @Override - public void run(Status status) { - LOG.info("Partition {}-{} onChangeShard complete, status is {}", - graphName, partitionId, status); - consumer.accept(0); - } - }); - } catch (Exception e) { - LOG.error("Partition {}-{} onSplitPartition exception {}", - graphName, partitionId, e); - } - } - } - - @Override - public void onTransferLeader(long taskId, Partition partition, TransferLeader transferLeader, Consumer consumer) { - PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId()); - if (engine != null && engine.isLeader()) { - consumer.accept(0); - Utils.runInThread(() -> { - LOG.info("Partition {}-{} receive TransferLeader instruction, new leader is {}" - , partition.getGraphName(), partition.getId(), transferLeader.getShard()); - engine.transferLeader(partition.getGraphName(), transferLeader.getShard()); - }); - } - } - - /** - * Leader接收到PD发送的分区分裂任务 - * 添加到raft任务队列，由raft进行任务分发。 - */ - @Override - public void onSplitPartition(long taskId, Partition partition, SplitPartition splitPartition, - Consumer consumer) { - PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId()); - - if (preCheckTaskId(taskId, partition.getId())) { - return; - } - - if (engine != null && engine.isLeader()) { - // 先应答，避免超时造成pd重复发送 - consumer.accept(0); - - String graphName = partition.getGraphName(); - int partitionId = partition.getId(); - MetaTask.Task task = MetaTask.Task.newBuilder() - .setId(taskId) - .setPartition(partition.getProtoObj()) - .setType(MetaTask.TaskType.Split_Partition) - .setState(MetaTask.TaskState.Task_Ready) - .setSplitPartition(splitPartition) - .build(); - try { - threadPool.submit(()->{ - engine.moveData(task); - }); - } catch (Exception e) { - LOG.error("Partition {}-{} onSplitPartition exception {}", - graphName, partitionId, e); - } - } - } - - /** - * Leader接收到PD发送的rocksdb compaction任务 - * 添加到raft任务队列，由raft进行任务分发。 - */ - @Override - public void onDbCompaction(long taskId, Partition partition, DbCompaction dbCompaction, - Consumer consumer) { - PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId()); - if (engine != null && engine.isLeader()) { - try { - DbCompactionRequest dbCompactionRequest = new DbCompactionRequest(); - dbCompactionRequest.setPartitionId(partition.getId()); - dbCompactionRequest.setTableName(dbCompaction.getTableName()); - dbCompactionRequest.setGraphName(partition.getGraphName()); - engine.addRaftTask(RaftOperation.create(RaftOperation.DB_COMPACTION, - dbCompactionRequest), - new RaftClosure() { - @Override - public void run(Status status) { - LOG.info("onRocksdbCompaction {}-{} sync partition status is {}", - partition.getGraphName(), partition.getId(), status); - } - } - ); - } finally { - consumer.accept(0); - } - } - } - - @Override - public void onMovePartition(long taskId, Partition partition, MovePartition movePartition, - Consumer consumer) { - PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId()); - - if (preCheckTaskId(taskId, partition.getId())) { - return; - } - - if (engine != null && engine.isLeader()){ - // 先应答，避免超时造成pd重复发送 - consumer.accept(0); - - String graphName = partition.getGraphName(); - int partitionId = partition.getId(); - MetaTask.Task task = MetaTask.Task.newBuilder() - .setId(taskId) - .setPartition(partition.getProtoObj()) - .setType(MetaTask.TaskType.Move_Partition) - .setState(MetaTask.TaskState.Task_Ready) - .setMovePartition(movePartition) - .build(); - try { - threadPool.submit(()->{ - engine.moveData(task); - }); - } catch (Exception e) { - LOG.error("Partition {}-{} onMovePartition exception {}", - graphName, partitionId, e); - } - } - } - - @Override - public void onCleanPartition(long taskId, Partition partition, CleanPartition cleanPartition, - Consumer consumer) { - - if (preCheckTaskId(taskId, partition.getId())) { - return; - } - - PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId()); - if (engine != null && engine.isLeader()){ - consumer.accept(0); - - CleanDataRequest request = CleanDataRequest.fromCleanPartitionTask(cleanPartition, partition, taskId); - - storeEngine.addRaftTask(partition.getGraphName(), partition.getId(), - RaftOperation.create(RaftOperation.IN_CLEAN_OP, request), - status -> { - LOG.info("onCleanPartition {}-{}, cleanType: {}, range:{}-{}, status:{}", - partition.getGraphName(), - partition.getId(), - cleanPartition.getCleanType(), - cleanPartition.getKeyStart(), - cleanPartition.getKeyEnd(), - status); - }); - } - - } - - @Override - public void onPartitionKeyRangeChanged(long taskId, Partition partition, PartitionKeyRange partitionKeyRange, - Consumer consumer) { - PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId()); - if (engine != null && engine.isLeader()){ - consumer.accept(0); - var partitionManager = storeEngine.getPartitionManager(); - var localPartition = partitionManager.getPartition(partition.getGraphName(), partition.getId()); - - if (localPartition == null) { - // 如果分区数据为空，本地不会存储 - localPartition = partitionManager.getPartitionFromPD(partition.getGraphName(), partition.getId()); - LOG.info("onPartitionKeyRangeChanged, get from pd:{}-{} -> {}", - partition.getGraphName(), partition.getId(), localPartition); - if (localPartition == null){ - return; - } - } - - var newPartition = localPartition.getProtoObj().toBuilder() - .setStartKey(partitionKeyRange.getKeyStart()) - .setEndKey(partitionKeyRange.getKeyEnd()) - .setState(Metapb.PartitionState.PState_Normal) - .build(); - partitionManager.updatePartition(newPartition, true); - - try { - engine.addRaftTask(RaftOperation.create(RaftOperation.SYNC_PARTITION, newPartition), status -> { - LOG.info("onPartitionKeyRangeChanged, {}-{},key range: {}-{} status{}", - newPartition.getGraphName(), - newPartition.getId(), - partitionKeyRange.getKeyStart(), - partitionKeyRange.getKeyEnd(), - status); - }); - LOG.info("onPartitionKeyRangeChanged: {}, update to pd", newPartition); - partitionManager.updatePartitionToPD(List.of(newPartition)); - } catch (IOException e) { - LOG.error("Partition {}-{} onPartitionKeyRangeChanged exception {}", - newPartition.getGraphName(), newPartition.getId(), e); - } catch (PDException e) { - throw new RuntimeException(e); - } - } - } - - /** - * is the task exists - * @param taskId task id - * @param partId partition id - * @return true if exists, false otherwise - */ - private boolean preCheckTaskId(long taskId, int partId){ - - if (storeEngine.getPartitionEngine(partId) == null ){ - return false; - } - - byte[] key = MetadataKeyHelper.getInstructionIdKey(taskId); - var wrapper = storeEngine.getPartitionManager().getWrapper(); - byte[] value = wrapper.get(partId, key); - - if (value != null) { - return true; - } - - wrapper.put(partId, key, new byte[0]); - return false; - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/PartitionStateListener.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/PartitionStateListener.java deleted file mode 100644 index 7971a3133..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/PartitionStateListener.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.baidu.hugegraph.store; - -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.meta.PartitionRole; - -import java.util.List; - -public interface PartitionStateListener { - // 分区角色发生改变 - void partitionRoleChanged(Partition partition, PartitionRole newRole); - // 分区发生改变 - void partitionShardChanged(Partition partition, List oldShards, List newShards); -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/BatchPutRequest.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/BatchPutRequest.java deleted file mode 100644 index 6e7053056..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/BatchPutRequest.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -import lombok.Data; - -import java.io.Serializable; -import java.util.List; - -@Data -public class BatchPutRequest extends HgCmdBase.BaseRequest{ - private List entries; - @Override - public byte magic() { - return HgCmdBase.BATCH_PUT; - } - - @Data - public static class KV implements Serializable { - private String table; - private int code; - private byte[] key; - private byte[] value; - - public static KV of(String table, int code, byte[] key, byte[] value){ - KV kv = new KV(); - kv.table = table; - kv.code = code; - kv.key = key; - kv.value = value; - return kv; - } - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/BatchPutResponse.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/BatchPutResponse.java deleted file mode 100644 index 762a84e09..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/BatchPutResponse.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -public class BatchPutResponse extends HgCmdBase.BaseResponse { -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/CleanDataRequest.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/CleanDataRequest.java deleted file mode 100644 index befa83472..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/CleanDataRequest.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -import com.baidu.hugegraph.pd.grpc.pulse.CleanPartition; -import com.baidu.hugegraph.pd.grpc.pulse.CleanType; -import com.baidu.hugegraph.store.meta.Partition; -import lombok.Data; - - -@Data -public class CleanDataRequest extends HgCmdBase.BaseRequest{ - - private long keyStart; - private long keyEnd; - - private CleanType cleanType; - - private boolean deletePartition; - - private long taskId; - - @Override - public byte magic() { - return HgCmdBase.CLEAN_DATA; - } - - public static CleanDataRequest fromCleanPartitionTask(CleanPartition task, Partition partition, long taskId) { - return fromCleanPartitionTask(partition.getGraphName(), partition.getId(), taskId, task); - } - - public static CleanDataRequest fromCleanPartitionTask(String graphName, int partitionId, long taskId, - CleanPartition task){ - CleanDataRequest request = new CleanDataRequest(); - request.setGraphName(graphName); - request.setPartitionId(partitionId); - request.setCleanType(task.getCleanType()); - request.setKeyStart(task.getKeyStart()); - request.setKeyEnd(task.getKeyEnd()); - request.setDeletePartition(task.getDeletePartition()); - request.setTaskId(taskId); - return request; - } - - public static CleanPartition toCleanPartitionTask(CleanDataRequest request){ - return CleanPartition.newBuilder() - .setKeyStart(request.keyStart) - .setKeyEnd(request.keyEnd) - .setDeletePartition(request.deletePartition) - .setCleanType(request.cleanType) - .build(); - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/CleanDataResponse.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/CleanDataResponse.java deleted file mode 100644 index 5b4a756be..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/CleanDataResponse.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -public class CleanDataResponse extends HgCmdBase.BaseResponse { -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/CreateRaftResponse.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/CreateRaftResponse.java deleted file mode 100644 index 2614d00fa..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/CreateRaftResponse.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -public class CreateRaftResponse extends HgCmdBase.BaseResponse{ -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/DbCompactionRequest.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/DbCompactionRequest.java deleted file mode 100644 index a089df981..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/DbCompactionRequest.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -import lombok.Data; - -@Data -public class DbCompactionRequest extends HgCmdBase.BaseRequest{ - private String tableName; - - @Override - public byte magic() { - return HgCmdBase.ROCKSDB_COMPACTION; - } -} - diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/DbCompactionResponse.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/DbCompactionResponse.java deleted file mode 100644 index 35817967d..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/DbCompactionResponse.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -public class DbCompactionResponse extends HgCmdBase.BaseResponse { -} \ No newline at end of file diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/DestroyRaftRequest.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/DestroyRaftRequest.java deleted file mode 100644 index 79f791a7d..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/DestroyRaftRequest.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -import lombok.Data; - -import java.util.ArrayList; -import java.util.List; - -@Data -public class DestroyRaftRequest extends HgCmdBase.BaseRequest{ - private List graphNames = new ArrayList<>(); - - public void addGraphName(String graphName) { - graphNames.add(graphName); - } - @Override - public byte magic() { - return HgCmdBase.DESTROY_RAFT; - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/DestroyRaftResponse.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/DestroyRaftResponse.java deleted file mode 100644 index f98efa749..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/DestroyRaftResponse.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -public class DestroyRaftResponse extends HgCmdBase.BaseResponse { - -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/FutureClosureAdapter.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/FutureClosureAdapter.java deleted file mode 100644 index dfd3b3cb5..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/FutureClosureAdapter.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -import com.alipay.sofa.jraft.Closure; -import com.alipay.sofa.jraft.Status; - -import java.util.concurrent.CompletableFuture; - -public class FutureClosureAdapter implements Closure { - public final CompletableFuture future = new CompletableFuture<>(); - private T resp; - - public T getResponse() { return this.resp; } - - public void setResponse(T resp) { - this.resp = resp; - future.complete(resp); - run(Status.OK()); - } - - public void failure(Throwable t){ - future.completeExceptionally(t); - run(new Status(-1, t.getMessage())); - } - - @Override - public void run(Status status) { - - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/GetStoreInfoRequest.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/GetStoreInfoRequest.java deleted file mode 100644 index 1bf41355c..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/GetStoreInfoRequest.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -public class GetStoreInfoRequest extends HgCmdBase.BaseRequest{ - @Override - public byte magic() { - return HgCmdBase.GET_STORE_INFO; - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/GetStoreInfoResponse.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/GetStoreInfoResponse.java deleted file mode 100644 index d217aa676..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/GetStoreInfoResponse.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.meta.Store; -import com.google.protobuf.InvalidProtocolBufferException; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class GetStoreInfoResponse extends HgCmdBase.BaseResponse{ - private byte[] store; - - public void setStore(Store store) { - this.store = store.getProtoObj().toByteArray(); - } - - public Store getStore(){ - try { - return new Store(Metapb.Store.parseFrom(this.store)); - } catch (InvalidProtocolBufferException e) { - log.error("GetStoreResponse parse exception {}", e); - } - return null; - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/HgCmdBase.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/HgCmdBase.java deleted file mode 100644 index c67d71338..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/HgCmdBase.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -import lombok.Data; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -public class HgCmdBase { - - public static final byte GET_STORE_INFO = 0x01; - public static final byte BATCH_PUT = 0x02; - public static final byte CLEAN_DATA = 0x03; - public static final byte RAFT_UPDATE_PARTITION = 0x04; - public static final byte ROCKSDB_COMPACTION = 0x05; - public static final byte CREATE_RAFT = 0x06; - public static final byte DESTROY_RAFT = 0x07; - @Data - public abstract static class BaseRequest implements Serializable { - private String graphName; - private int partitionId; - - public abstract byte magic(); - } - - @Data - public abstract static class BaseResponse implements Serializable { - private HgCmdProcessor.Status status; - List partitionLeaders; - - public static class PartitionLeader implements Serializable { - private Integer partId; - private Long storeId; - - public PartitionLeader(Integer partId, Long storeId) { - this.partId = partId; - this.storeId = storeId; - } - - public Long getStoreId() { - return storeId; - } - - public Integer getPartId() { - return partId; - } - } - - public synchronized BaseResponse addPartitionLeader(PartitionLeader ptLeader) { - if (partitionLeaders == null) { - partitionLeaders = new ArrayList<>(); - } - partitionLeaders.add(ptLeader); - return this; - } - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/UpdatePartitionRequest.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/UpdatePartitionRequest.java deleted file mode 100644 index baadd0f6c..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/UpdatePartitionRequest.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -import com.baidu.hugegraph.pd.grpc.Metapb; -import lombok.Data; - - -@Data -public class UpdatePartitionRequest extends HgCmdBase.BaseRequest{ - private int startKey; - private int endKey; - - private Metapb.PartitionState workState; - @Override - public byte magic() { - return HgCmdBase.RAFT_UPDATE_PARTITION; - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/UpdatePartitionResponse.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/UpdatePartitionResponse.java deleted file mode 100644 index 4fe4d972c..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/UpdatePartitionResponse.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.baidu.hugegraph.store.cmd; - -public class UpdatePartitionResponse extends HgCmdBase.BaseResponse { -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/MetadataKeyHelper.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/MetadataKeyHelper.java deleted file mode 100644 index 577dd259c..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/MetadataKeyHelper.java +++ /dev/null @@ -1,297 +0,0 @@ -package com.baidu.hugegraph.store.meta; - -import java.nio.charset.StandardCharsets; - -public class MetadataKeyHelper { - private static final char DELIMITER = '/'; - private static final String HUGEGRAPH = "HUGEGRAPH"; - private static final String STORE = "STORE"; - private static final String PARTITION = "PARTITION"; - private static final String TASK = "TASK"; - private static final String ASYNC_TASK = "A_TASK"; - private static final String INSTRUCTION_TASK = "INS_TASK"; - private static final String TASK_DONE = "TASK_DONE"; - private static final String GRAPH = "GRAPH"; - private static final String PARTITION_STORE = "PARTITION_STORE"; - private static final String PARTITION_RAFT = "PARTITION_Raft"; - private static final String DELETED_FILE = "DELETED_FILE"; - private static final String SHARD_GROUP = "SHARDGROUP"; - - private static final String CID_PREFIX = "CID"; - private static final String CID_SLOT_PREFIX = "CID_SLOT"; - private static final String GRAPH_ID_PREFIX = "GRAPH_ID"; - - public static byte[] getPartitionKey(String graph, Integer partId) { - // HUGEGRAPH/Partition/{graph}/partId - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(PARTITION).append(DELIMITER) - .append(graph).append(DELIMITER) - .append(partId) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getPartitionPrefixKey(String graph) { - // HUGEGRAPH/Partition/{graph}/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(PARTITION).append(DELIMITER) - .append(graph).append(DELIMITER) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - /** - * 查询分区内的所有partition prefix， 不包含 graph name - * @return - */ - public static byte[] getPartitionPrefixKey() { - // HUGEGRAPH/Partition/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(PARTITION).append(DELIMITER) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getShardGroupKey(int partitionId) { - // HUGEGRAPH/SHARDGROUP/{partition_id} - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(SHARD_GROUP).append(DELIMITER) - .append(partitionId) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getGraphKey(String graph) { - // HUGEGRAPH/Graph/{graph} - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(GRAPH).append(DELIMITER) - .append(graph) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getGraphKeyPrefix() { - // HUGEGRAPH/Graph/{graph} - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(GRAPH).append(DELIMITER) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getStoreKey() { - // HUGEGRAPH/STORE/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(STORE) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getTaskKey(int partId, String type, long taskId) { - // HUGEGRAPH/TASK/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(TASK).append(DELIMITER) - .append(partId).append(DELIMITER) - .append(type).append(DELIMITER) - .append(String.format("%016x", taskId)) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getInstructionIdKey(long taskId){ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(INSTRUCTION_TASK).append(DELIMITER) - .append(taskId).append(DELIMITER) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getAsyncTaskKey(int partId, String graphName, String taskId) { - // HUGEGRAPH/A_TASK/ part id / graphName / task id - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(ASYNC_TASK).append(DELIMITER) - .append(partId).append(DELIMITER) - .append(graphName).append(DELIMITER) - .append(taskId) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getAsyncTaskPrefix(int partId, String graphName) { - // HUGEGRAPH/A_TASK/ part id / graphName / task id - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(ASYNC_TASK).append(DELIMITER) - .append(partId).append(DELIMITER) - .append(graphName).append(DELIMITER) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getTaskPrefix(int partId, String type) { - // HUGEGRAPH/TASK/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(TASK).append(DELIMITER) - .append(partId).append(DELIMITER) - .append(type).append(DELIMITER) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getTaskPrefix(int partId) { - // HUGEGRAPH/TASK/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(TASK).append(DELIMITER) - .append(partId).append(DELIMITER) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getTaskPrefix() { - // HUGEGRAPH/TASK/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(TASK).append(DELIMITER) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getDoneTaskKey(long taskId) { - // HUGEGRAPH/TASK/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(TASK_DONE).append(DELIMITER) - .append(String.format("%016x", taskId)) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getPartitionStoreKey(int partId) { - // HUGEGRAPH/TASK/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(PARTITION_STORE).append(DELIMITER) - .append(partId) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getPartitionStorePrefix() { - // HUGEGRAPH/TASK/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(PARTITION_STORE).append(DELIMITER) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getPartitionRaftKey(int partId) { - // HUGEGRAPH/TASK/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(PARTITION_RAFT).append(DELIMITER) - .append(partId) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getPartitionRaftPrefix() { - // HUGEGRAPH/TASK/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(PARTITION_RAFT).append(DELIMITER) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getDeletedFileKey(String filePath) { - // HUGEGRAPH/TASK/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(DELETED_FILE).append(DELIMITER) - .append(filePath) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getDeletedFilePrefix() { - // HUGEGRAPH/TASK/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(DELETED_FILE).append(DELIMITER) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getCidKey(String name) { - // HUGEGRAPH/CID/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(CID_PREFIX).append(DELIMITER) - .append(name) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getCidSlotKeyPrefix(String name) { - // HUGEGRAPH/CID_SLOT/ - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(CID_SLOT_PREFIX).append(DELIMITER) - .append(name).append(DELIMITER) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] getGraphIDKey(String graph) { - // HUGEGRAPH/Graph/{graph} - String key = StringBuilderHelper.get() - .append(HUGEGRAPH).append(DELIMITER) - .append(GRAPH_ID_PREFIX).append(DELIMITER) - .append(graph) - .toString(); - return key.getBytes(StandardCharsets.UTF_8); - } - - static class StringBuilderHelper { - private static final int DISCARD_LIMIT = 1024 << 3; // 8k - - private static final ThreadLocal holderThreadLocal = ThreadLocal - .withInitial(StringBuilderHolder::new); - - public static StringBuilder get() { - final StringBuilderHolder holder = holderThreadLocal.get(); - return holder.getStringBuilder(); - } - - public static void truncate() { - final StringBuilderHolder holder = holderThreadLocal.get(); - holder.truncate(); - } - - private static class StringBuilderHolder { - - private final StringBuilder buf = new StringBuilder(); - - private StringBuilder getStringBuilder() { - truncate(); - return buf; - } - - private void truncate() { - buf.setLength(0); - } - } - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/Partition.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/Partition.java deleted file mode 100644 index 5eb003782..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/Partition.java +++ /dev/null @@ -1,86 +0,0 @@ -package com.baidu.hugegraph.store.meta; - -import com.baidu.hugegraph.pd.grpc.Metapb; - -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.PartitionEngine; -import lombok.Data; - -/** - * @projectName: hugegraph-store - * @package: com.baidu.hugegraph.store.partitions - * @className: Partition - * @author: tyzer - * @description: TODO - * @date: 2021/10/19 18:16 - * @version: 1.0 - */ -@Data -public class Partition implements Cloneable{ - private int id; // region id - private String graphName; - // Region key range [startKey, endKey) - private long startKey; - private long endKey; - private long version; - // shardlist版本，shardlist每次改变加1 - // private long confVer; - - private Metapb.PartitionState workState; - // private PartitionRole role; - // private List shardsList; - // exclusive - - public - Partition(){ - workState = Metapb.PartitionState.PState_Normal; - } - - public Partition(Metapb.Partition protoObj){ - id = protoObj.getId(); - graphName = protoObj.getGraphName(); - startKey = protoObj.getStartKey(); - endKey = protoObj.getEndKey(); - // shardsList = protoObj.getShardsList(); - workState = protoObj.getState(); - version = protoObj.getVersion(); - // confVer = protoObj.getConfVer(); - if (workState == Metapb.PartitionState.UNRECOGNIZED || workState == Metapb.PartitionState.PState_None) { - workState = Metapb.PartitionState.PState_Normal; - } - } - - public boolean isLeader() { - PartitionEngine engine = HgStoreEngine.getInstance().getPartitionEngine(id); - return engine == null ? false : engine.isLeader(); - } - - public Metapb.Partition getProtoObj(){ - return Metapb.Partition.newBuilder() - .setId(id) - .setVersion(version) - // .setConfVer(confVer) - .setGraphName(graphName) - .setStartKey(startKey) - .setEndKey(endKey) - .setState(workState) - // .addAllShards(shardsList) - .build(); - } - - @Override - public Partition clone() { - Partition obj = null; - try { - obj = (Partition) super.clone(); - } catch (CloneNotSupportedException e) { - e.printStackTrace(); - } - return obj; - } - - @Override - public String toString() { - return getProtoObj().toString(); - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/PartitionStats.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/PartitionStats.java deleted file mode 100644 index bfba3a1b6..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/PartitionStats.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.baidu.hugegraph.store.meta; - -import com.baidu.hugegraph.pd.grpc.Metapb; -import lombok.Data; - -import java.util.ArrayList; -import java.util.List; - -@Data -public class PartitionStats { - // 分区ID - private int id; - private String namespace; - private String graphName; - // 分区leader所在shard - Metapb.Shard leader; - // 分区离线的shard - List offlineShards = new ArrayList<>(); - long committedIndex; - long leaderTerm; - long approximateSize; - long approximateKeys; - - public PartitionStats addOfflineShard(Metapb.Shard shard){ - offlineShards.add(shard); - return this; - } - - public Metapb.PartitionStats getProtoObj(){ - return Metapb.PartitionStats.newBuilder() - .setId(id) - .addGraphName(graphName) - .setLeader(leader) - .addAllShard(offlineShards) - .setApproximateKeys(approximateKeys) - .setApproximateSize(approximateSize) - .setLeaderTerm(leaderTerm) - .build(); - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/Shard.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/Shard.java deleted file mode 100644 index 6f3fa600d..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/Shard.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.baidu.hugegraph.store.meta; - -import com.baidu.hugegraph.pd.grpc.Metapb; -import lombok.Data; - -/** - * 一个分片 - */ -@Data -public class Shard { - private long storeId; - private Metapb.ShardRole role; - - public Metapb.Shard toMetaPbShard() { - return Metapb.Shard.newBuilder().setStoreId(storeId).setRole(role).build(); - } - - public static Shard fromMetaPbShard(Metapb.Shard shard){ - Shard s = new Shard(); - s.setRole(shard.getRole()); - s.setStoreId(shard.getStoreId()); - return s; - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/Store.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/Store.java deleted file mode 100644 index e69f01ae6..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/Store.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.baidu.hugegraph.store.meta; - -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.util.Asserts; -import com.baidu.hugegraph.store.util.Version; -import lombok.Data; - -import java.util.Map; - -/** - * @projectName: hugegraph-store - * @package: com.baidu.hugegraph.store.core - * @className: HgStoreNode - * @author: tyzer - * @description: TODO - * @date: 2021/10/19 19:44 - * @version: 1.0 - */ -@Data -public class Store { - private long id = 0; - private String storeAddress; - private String pdAddress; - private String raftAddress; - private String version; - private String deployPath; - private String dataPath; // 数据存储路径 - private int dataVersion; - private int partitionCount; - private int startTime; - private int usedSize; //rocksdb存储大小 - private int pdHeartbeatInterval; - private Metapb.StoreState state; - private Map labels; - private int cores; - - public Store(){ - this.id = 0; - this.version = Version.getVersion(); - } - public Store(int dataVersion){ - this.id = 0; - this.dataVersion = dataVersion; - this.version = Version.getVersion(); - } - public Store(Metapb.Store protoObj){ - if ( protoObj != null) { - this.id = protoObj.getId(); - this.raftAddress = protoObj.getRaftAddress(); - this.storeAddress = protoObj.getAddress(); - this.dataVersion = protoObj.getDataVersion(); - } - else - this.id = 0; - this.version = Version.getVersion(); - } - - - public Metapb.Store getProtoObj(){ - Asserts.isNonNull(storeAddress); - Asserts.isNonNull(raftAddress); - Metapb.Store.Builder builder = Metapb.Store.newBuilder() - .setId(id).setVersion(version) - .setDataVersion(dataVersion) - .setAddress(storeAddress) - .setRaftAddress(raftAddress) - .setState(Metapb.StoreState.Up) - .setCores(cores) - .setDeployPath(deployPath) - .setDataPath(dataPath); - - if (labels != null) - labels.forEach((k,v)->{ - builder.addLabels(Metapb.StoreLabel.newBuilder().setKey(k).setValue(v).build()); - }); - - return builder.build(); - - } - public boolean checkState(Metapb.StoreState state) { - return this.state == state; - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/asynctask/AsyncTask.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/asynctask/AsyncTask.java deleted file mode 100644 index 13d75169e..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/asynctask/AsyncTask.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.baidu.hugegraph.store.meta.asynctask; - -public interface AsyncTask { - - /** - * 需要检查异步任务时候，检查当前的状态，根据状态去做对应的处理 - */ - void handleTask(); - - /** - * 任务ID - */ - String getId(); - - /** - * 针对哪个图的 - */ - String getGraphName(); - - /** - * 针对哪个分区的 - */ - int getPartitionId(); - - /** - * 用来进行序列化 - * @return - */ - byte[] toBytes(); - - /** - * 设置执行状态 - * @param newState - */ - void setState(AsyncTaskState newState); -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/asynctask/AsyncTaskState.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/asynctask/AsyncTaskState.java deleted file mode 100644 index 944c3e662..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/asynctask/AsyncTaskState.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.baidu.hugegraph.store.meta.asynctask; - -public enum AsyncTaskState { - START, SUCCESS, FAILED -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/asynctask/CleanTask.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/asynctask/CleanTask.java deleted file mode 100644 index c22bb3c58..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/asynctask/CleanTask.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.baidu.hugegraph.store.meta.asynctask; - -import com.baidu.hugegraph.pd.grpc.pulse.CleanType; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.cmd.CleanDataRequest; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CleanTask extends AbstractAsyncTask{ - - public CleanTask(int partitionId, String graphName, AsyncTaskState state, Object attach) { - super(partitionId, graphName, state, attach); - } - - @Override - public String getType() { - return "CLEAN_TYPE"; - } - - @Override - protected void onError() { - cleanTask(); - } - - @Override - protected void onNotFinished() { - cleanTask(); - } - - private void cleanTask(){ - log.info("CleanTask begin to run:{}", this); - var storeEngine = HgStoreEngine.getInstance(); - if (storeEngine != null) { - if (getExtra() != null) { - CleanDataRequest request = (CleanDataRequest) getExtra(); - var partition = storeEngine.getPartitionManager().getPartition(getGraphName(), getPartitionId()); - // 只允许清理本分区之外的数据。 缩容等任务会造成干扰, 而且不能删除分区 - if (request.getKeyEnd() == partition.getStartKey() && request.getKeyEnd() == partition.getEndKey() && - request.getCleanType() == CleanType.CLEAN_TYPE_EXCLUDE_RANGE && ! request.isDeletePartition()) { - storeEngine.getBusinessHandler().cleanPartition(getGraphName(), getPartitionId(), - request.getKeyStart(), request.getKeyEnd(), request.getCleanType()); - } - }else { - storeEngine.getBusinessHandler().cleanPartition(getGraphName(), getPartitionId()); - } - - storeEngine.getPartitionEngine(getPartitionId()).getTaskManager() - .updateAsyncTaskState(getPartitionId(), getGraphName(), getId(), AsyncTaskState.SUCCESS); - } - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/DBSessionBuilder.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/DBSessionBuilder.java deleted file mode 100644 index 8d9edd452..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/DBSessionBuilder.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.baidu.hugegraph.store.meta.base; - -import org.apache.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.store.util.HgStoreException; - -public interface DBSessionBuilder { - RocksDBSession getSession(int partId) throws HgStoreException; -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/metric/HgStoreMetric.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/metric/HgStoreMetric.java deleted file mode 100644 index dece29123..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/metric/HgStoreMetric.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.baidu.hugegraph.store.metric; - -import lombok.Data; - -import java.util.List; - -public class HgStoreMetric { - - @Data - public static class Table { - private String tableName; - private long keyCount; - private String dataSize; - } - - @Data - public static class Partition { - private int partitionId; - private List tables; - } - - @Data - public static class Graph { - private long approxDataSize; - private long approxKeyCount; - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/HgStoreEngineOptions.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/HgStoreEngineOptions.java deleted file mode 100644 index 363c6726a..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/HgStoreEngineOptions.java +++ /dev/null @@ -1,122 +0,0 @@ -package com.baidu.hugegraph.store.options; - -import com.alipay.sofa.jraft.util.Utils; -import com.baidu.hugegraph.store.business.DataMover; -import com.baidu.hugegraph.store.pd.PdProvider; -import com.baidu.hugegraph.store.raft.RaftTaskHandler; -import lombok.Data; - -import java.util.Map; - -/** - * Storage engine configuration - */ -@Data -public class HgStoreEngineOptions { - public static String Raft_Path_Prefix = "raft"; - public static String DB_Path_Prefix = "db"; - public static String Snapshot_Path_Prefix = "snapshot"; - // pd 服务器地址 - private String pdAddress; - // 对外服务地址 - private String grpcAddress; - // Raft 对外服务地址 - private String raftAddress; - // 存储路径，支持多个位置，逗号分割 - private String dataPath; - - private String raftPath; - - private Map rocksdbConfig; - // store心跳间隔，单位秒 - private int storeHBInterval = 30; - // 分区心跳间隔，单位秒 - private int partitionHBInterval = 5; - // 等待leader超时时间，单位秒 - private int waitLeaderTimeout = 30; - // 没有PD模式，用于开发调试使用 - private boolean fakePD = false; - - private int raftRpcThreadPoolSize = Utils.cpus() * 6; - // 自定义的标签，传给pd - private Map labels; - // fakePd配置项 - private FakePdOptions fakePdOptions = new FakePdOptions(); - private RaftOptions raftOptions = new RaftOptions(); - - // Raft任务处理器 - private RaftTaskHandler taskHandler; - - private PdProvider pdProvider; - - // 数据迁移服务 - private DataMover dataTransfer; - - @Data - public static class FakePdOptions{ - private String storeList; - private String peersList; - private int partitionCount = 0; - private int shardCount = 0; - } - - @Data - public static class RaftOptions{ - - /* - * Rpc connect timeout in milliseconds - * The time should be less than electionTimeoutMs, otherwise the election will timeout - */ - private int rpcConnectTimeoutMs = 1000; - - /** - * RPC request default timeout in milliseconds - */ - private int rpcDefaultTimeout = 5000; - - // A follower would become a candidate if it doesn't receive any message - // from the leader in |election_timeout_ms| milliseconds - - private int electionTimeoutMs = 3000; - /** - * Install snapshot RPC request default timeout in milliseconds - */ - private int rpcInstallSnapshotTimeout = 60 * 60 * 1000; - // A snapshot saving would be triggered every |snapshot_interval_s| seconds - // if this was reset as a positive number - // If |snapshot_interval_s| <= 0, the time based snapshot would be disabled. - // - // Default: 3600 (1 hour) - private int snapshotIntervalSecs = 3600; - // A snapshot saving would be triggered every |snapshot_interval_s| seconds, - // and at this moment when state machine's lastAppliedIndex value - // minus lastSnapshotId value is greater than snapshotLogIndexMargin value, - // the snapshot action will be done really. - // If |snapshotLogIndexMargin| <= 0, the distance based snapshot would be disable. - // - // Default: 0 - private int snapshotLogIndexMargin = 1024; - private boolean metrics = true; - // 等待leader超时时间，单位秒 - private int waitLeaderTimeout = 30; - /** Internal disruptor buffers size for Node/FSMCaller/LogManager etc. */ - private int disruptorBufferSize = 4096; - /** The maximum number of entries in AppendEntriesRequest */ - private int maxEntriesSize = 256; - /** The maximum replicator pipeline in-flight requests/responses, only valid when enable replicator pipeline. */ - private int maxReplicatorInflightMsgs = 256; - /** Raft集群发生数据积压后，限速等待时间 单位毫秒**/ - private int overloadRateLimit = 100; - - /** The maximum byte size of log allowed by user. */ - private long maxLogSize = 100 * 1024 * 1024; - /** The ratio of exponential approximation for average size of log entry. */ - private double aveLogEntrySizeRatio = 0.95; - - private boolean useRocksDBSegmentLogStorage = true; - private int maxSegmentFileSize = 64 * 1024 * 1024; - private int keepInMemorySegmentCount = 2; - private int preAllocateSegmentCount = 1; - private int splitPartitionLogIndexMargin = 10; - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/MetadataOptions.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/MetadataOptions.java deleted file mode 100644 index 761bb0a5a..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/MetadataOptions.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.baidu.hugegraph.store.options; - -import lombok.Data; - -@Data -public class MetadataOptions { - private String dataPath; - private String raftPath; -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/PartitionEngineOptions.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/PartitionEngineOptions.java deleted file mode 100644 index e59d865dd..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/PartitionEngineOptions.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.baidu.hugegraph.store.options; - -import com.alipay.sofa.jraft.conf.Configuration; -import com.baidu.hugegraph.store.raft.RaftTaskHandler; -import lombok.Data; - -import java.util.List; -import java.util.concurrent.Executor; - -/** - * Partition engine configuration - * - */ -@Data -public class PartitionEngineOptions { - // raft存储路径 - private String raftDataPath; - private String raftSnapShotPath; - private Integer groupId; - private String raftAddress; - private List peerList; - private Configuration conf; - // 异步任务执行时间间隔, 单位秒 - private int taskScheduleTime = 60; - // 分裂过程，等待数据对齐超时时间 - private long splitPartitionTimeout = 30 * 60 * 1000; - // raft 任务处理器 - private RaftTaskHandler taskHandler; - - HgStoreEngineOptions.RaftOptions raftOptions; -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/RaftRocksdbOptions.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/RaftRocksdbOptions.java deleted file mode 100644 index 09fee92af..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/options/RaftRocksdbOptions.java +++ /dev/null @@ -1,135 +0,0 @@ -package com.baidu.hugegraph.store.options; - -import com.alipay.sofa.jraft.storage.impl.RocksDBLogStorage; -import com.alipay.sofa.jraft.util.StorageOptionsFactory; -import com.baidu.hugegraph.config.HugeConfig; -import org.apache.hugegraph.rocksdb.access.RocksDBOptions; -import com.baidu.hugegraph.store.business.BusinessHandlerImpl; - -import lombok.extern.slf4j.Slf4j; -import org.rocksdb.*; -import org.rocksdb.util.SizeUnit; - -import java.util.Map; - -@Slf4j -public class RaftRocksdbOptions { - static class RocksdbConfig{ - private Env env; - private LRUCache blockCache; - private LRUCache writeCache; - private WriteBufferManager bufferManager; - private BlockBasedTableConfig tableConfig; - private long blockCacheCapacity; - private long writeCacheCapacity; - - public Env getEnv(){ return env; } - public LRUCache getBlockCache() { return blockCache; } - public LRUCache getWriteCache() { return writeCache; } - public WriteBufferManager getBufferManager() { return bufferManager;} - public BlockBasedTableConfig getTableConfig(){ return tableConfig; } - public long getBlockCacheCapacity(){ return blockCacheCapacity; } - public long getWriteCacheCapacity(){ return writeCacheCapacity; } - - public RocksdbConfig(HugeConfig options) { - RocksDB.loadLibrary(); - this.env = Env.getDefault(); - double writeBufferRatio = options.get(RocksDBOptions.WRITE_BUFFER_RATIO); - this.writeCacheCapacity = (long) (options.get(RocksDBOptions.TOTAL_MEMORY_SIZE) * writeBufferRatio); - this.blockCacheCapacity = options.get(RocksDBOptions.TOTAL_MEMORY_SIZE) - writeCacheCapacity; - this.writeCache = new LRUCache(writeCacheCapacity); - this.blockCache = new LRUCache(blockCacheCapacity); - this.bufferManager = new WriteBufferManager(writeCacheCapacity, writeCache, - options.get(RocksDBOptions.WRITE_BUFFER_ALLOW_STALL)); - this.tableConfig = new BlockBasedTableConfig() // - .setIndexType(IndexType.kTwoLevelIndexSearch) // - .setPartitionFilters(true) // - .setMetadataBlockSize(8 * SizeUnit.KB) // - .setCacheIndexAndFilterBlocks(options.get(RocksDBOptions.PUT_FILTER_AND_INDEX_IN_CACHE)) // - .setCacheIndexAndFilterBlocksWithHighPriority(true) // - .setPinL0FilterAndIndexBlocksInCache(options.get(RocksDBOptions.PIN_L0_FILTER_AND_INDEX_IN_CACHE)) // - .setBlockSize(4 * SizeUnit.KB)// - .setBlockCache(blockCache); - - int bitsPerKey = options.get(RocksDBOptions.BLOOM_FILTER_BITS_PER_KEY); - if (bitsPerKey >= 0) { - tableConfig.setFilterPolicy(new BloomFilter(bitsPerKey, - options.get(RocksDBOptions.BLOOM_FILTER_MODE))); - } - tableConfig.setWholeKeyFiltering( - options.get(RocksDBOptions.BLOOM_FILTER_WHOLE_KEY)); - log.info("RocksdbConfig {}", options.get(RocksDBOptions.BLOOM_FILTER_BITS_PER_KEY)); - } - } - - private static RocksdbConfig rocksdbConfig = null; - private static RocksdbConfig getRocksdbConfig(HugeConfig options){ - if ( rocksdbConfig == null){ - synchronized (RocksdbConfig.class){ - rocksdbConfig = new RocksdbConfig(options); - } - } - return rocksdbConfig; - } - - private static void registerRaftRocksdbConfig(HugeConfig options) { - Cache blockCache = new LRUCache(1 * SizeUnit.GB); - BlockBasedTableConfig tableConfig = new BlockBasedTableConfig() // - .setIndexType(IndexType.kTwoLevelIndexSearch) // - .setPartitionFilters(true) // - .setMetadataBlockSize(8 * SizeUnit.KB) // - .setCacheIndexAndFilterBlocks(options.get(RocksDBOptions.PUT_FILTER_AND_INDEX_IN_CACHE)) // - .setCacheIndexAndFilterBlocksWithHighPriority(true) // - .setPinL0FilterAndIndexBlocksInCache(options.get(RocksDBOptions.PIN_L0_FILTER_AND_INDEX_IN_CACHE)) // - .setBlockSize(4 * SizeUnit.KB)// - .setBlockCache(blockCache); - - StorageOptionsFactory.registerRocksDBTableFormatConfig(RocksDBLogStorage.class, - tableConfig); - - DBOptions dbOptions = StorageOptionsFactory.getDefaultRocksDBOptions(); - dbOptions.setEnv(rocksdbConfig.getEnv()); - - // raft rocksdb数量固定，通过max_write_buffer_number可以控制 - //dbOptions.setWriteBufferManager(rocksdbConfig.getBufferManager()); - dbOptions.setUnorderedWrite(true); - StorageOptionsFactory.registerRocksDBOptions(RocksDBLogStorage.class, - dbOptions); - - ColumnFamilyOptions cfOptions = StorageOptionsFactory.getDefaultRocksDBColumnFamilyOptions(); - cfOptions.setTargetFileSizeBase(256 * SizeUnit.MB); - cfOptions.setWriteBufferSize(8 * SizeUnit.MB); - cfOptions.setNumLevels(3); - cfOptions.setMaxWriteBufferNumber(3); - cfOptions.setCompressionType(CompressionType.NO_COMPRESSION); - cfOptions.setMaxBytesForLevelBase(2048 * SizeUnit.GB); - - StorageOptionsFactory.registerRocksDBColumnFamilyOptions(RocksDBLogStorage.class, - cfOptions); - } - - - public static void initRocksdbGlobalConfig(Map config){ - HugeConfig hugeConfig = BusinessHandlerImpl.initRocksdb(config, null); - RocksdbConfig rocksdbConfig = getRocksdbConfig(hugeConfig); - registerRaftRocksdbConfig(hugeConfig); - config.put(RocksDBOptions.ENV, rocksdbConfig.getEnv()); - config.put(RocksDBOptions.WRITE_BUFFER_MANAGER, rocksdbConfig.getBufferManager()); - config.put(RocksDBOptions.BLOCK_TABLE_CONFIG, rocksdbConfig.getTableConfig()); - config.put(RocksDBOptions.BLOCK_CACHE, rocksdbConfig.getBlockCache()); - config.put(RocksDBOptions.WRITE_CACHE, rocksdbConfig.getWriteCache()); - } - - public static WriteBufferManager getWriteBufferManager(){ - return rocksdbConfig.getBufferManager(); - } - - public static Env getEnv(){ - return rocksdbConfig.getEnv(); - } - - public static Cache getWriteCache() { return rocksdbConfig.getWriteCache();} - public static Cache getBlockCache() { return rocksdbConfig.getBlockCache();} - public static long getWriteCacheCapacity() { return rocksdbConfig.getWriteCacheCapacity();} - public static long getBlockCacheCapacity() { return rocksdbConfig.getBlockCacheCapacity();} -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/pd/PartitionInstructionListener.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/pd/PartitionInstructionListener.java deleted file mode 100644 index 8de938e7e..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/pd/PartitionInstructionListener.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.baidu.hugegraph.store.pd; - - -import com.baidu.hugegraph.pd.grpc.pulse.ChangeShard; -import com.baidu.hugegraph.pd.grpc.pulse.CleanPartition; -import com.baidu.hugegraph.pd.grpc.pulse.DbCompaction; -import com.baidu.hugegraph.pd.grpc.pulse.MovePartition; -import com.baidu.hugegraph.pd.grpc.pulse.PartitionKeyRange; -import com.baidu.hugegraph.pd.grpc.pulse.SplitPartition; -import com.baidu.hugegraph.pd.grpc.pulse.TransferLeader; -import com.baidu.hugegraph.store.meta.Partition; - -import java.util.function.Consumer; - -public interface PartitionInstructionListener { - void onChangeShard(long taskId, Partition partition, ChangeShard changeShard, Consumer consumer); - - void onTransferLeader(long taskId, Partition partition, TransferLeader transferLeader, Consumer consumer); - - void onSplitPartition(long taskId, Partition partition, SplitPartition splitPartition, Consumer consumer); - - void onDbCompaction(long taskId, Partition partition, DbCompaction rocksdbCompaction, - Consumer consumer); - - void onMovePartition(long taskId, Partition partition, MovePartition movePartition, Consumer consumer); - - void onCleanPartition(long taskId, Partition partition, CleanPartition cleanPartition, Consumer consumer); - - void onPartitionKeyRangeChanged(long taskId, Partition partition, PartitionKeyRange partitionKeyRange, - Consumer consumer); -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/RaftClosure.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/RaftClosure.java deleted file mode 100644 index 305cde6c0..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/RaftClosure.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.baidu.hugegraph.store.raft; - -import com.alipay.sofa.jraft.Closure; - -public interface RaftClosure extends Closure { - default void onLeaderChanged(Integer partId, Long storeId){} - -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/RaftTaskHandler.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/RaftTaskHandler.java deleted file mode 100644 index 6a7c8cbf6..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/RaftTaskHandler.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.baidu.hugegraph.store.raft; - -import com.baidu.hugegraph.store.util.HgStoreException; - -/** - * 接收raft发送的数据 - */ -public interface RaftTaskHandler { - boolean invoke(final int groupId, final byte[] request, RaftClosure response) throws HgStoreException; - boolean invoke(final int groupId, final byte methodId, final Object req, RaftClosure response) throws HgStoreException; -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/HgStoreConst.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/HgStoreConst.java deleted file mode 100644 index b9d3a78f1..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/HgStoreConst.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.baidu.hugegraph.store.util; - -import java.util.Collections; -import java.util.List; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/22 - */ -public final class HgStoreConst { - public final static byte[] EMPTY_BYTES=new byte[0]; - - public static final List EMPTY_LIST = Collections.EMPTY_LIST; - - public final static int SCAN_ALL_PARTITIONS_ID=-1; // means scan all partitions. - - private HgStoreConst(){} - -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/Lifecycle.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/Lifecycle.java deleted file mode 100644 index 0d0aec3cb..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/Lifecycle.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.baidu.hugegraph.store.util; -/** - * @author YanJinbing - */ -public interface Lifecycle { - - /** - * Initialize the service. - * - * @return true when successes. - */ - boolean init(final T opts); - - /** - * Dispose the resources for service. - */ - void shutdown(); -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/PartitionMetaStoreWrapper.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/PartitionMetaStoreWrapper.java deleted file mode 100644 index 29f21b6af..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/PartitionMetaStoreWrapper.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.baidu.hugegraph.store.util; - -import org.apache.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.meta.base.MetaStoreBase; - -import java.util.List; - -public class PartitionMetaStoreWrapper{ - - private static InnerMetaStore store = new InnerMetaStore(); - - - public void put(int partitionId, byte[] key, byte[] value){ - store.setPartitionId(partitionId); - store.put(key, value); - } - - public T get(int partitionId, byte[] key, com.google.protobuf.Parser parser){ - store.setPartitionId(partitionId); - return store.get(parser, key); - } - - public byte[] get(int partitionId, byte[] key) { - store.setPartitionId(partitionId); - return store.get(key); - } - - public void delete(int partitionId, byte[] key){ - store.setPartitionId(partitionId); - store.delete(key); - } - - public List scan(int partitionId, com.google.protobuf.Parser parser, byte[] prefix) { - store.setPartitionId(partitionId); - return store.scan(parser, prefix); - } - - private static class InnerMetaStore extends MetaStoreBase{ - - private int partitionId ; - - private void setPartitionId(int partitionId){ - this.partitionId = partitionId; - } - - @Override - protected RocksDBSession getRocksDBSession() { - return HgStoreEngine.getInstance().getBusinessHandler().getSession(this.partitionId); - } - - @Override - protected String getCFName() { - return "default"; - } - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/Version.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/Version.java deleted file mode 100644 index f83c4971e..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/Version.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.baidu.hugegraph.store.util; - -import lombok.extern.slf4j.Slf4j; - -import java.io.InputStream; -import java.io.Reader; - -@Slf4j -public class Version { - - private static String version = ""; - /** - * 软件版本号 - * @return - */ - public static String getVersion() { - if (version.isEmpty()) { - try (InputStream is = Version.class.getResourceAsStream("/version.txt")) { - byte[] buf = new byte[64]; - int len = is.read(buf); - version = new String(buf, 0, len); - } catch (Exception e) { - log.error("Version.getVersion exception {}", e); - } - } - return version; - } - - /** - * 存储格式版本号 - * @return - */ - public static int getDataFmtVersion(){ - return 1; - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/ZipUtils.java b/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/ZipUtils.java deleted file mode 100644 index 956e6c5ec..000000000 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/ZipUtils.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.baidu.hugegraph.store.util; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.output.NullOutputStream; - -import java.io.*; -import java.nio.file.Paths; -import java.util.zip.*; - -@Slf4j -public final class ZipUtils { - - public static void compress(final String rootDir, final String sourceDir, - final String outputFile, final Checksum checksum) throws IOException { - try (final FileOutputStream fos = new FileOutputStream(outputFile); - final CheckedOutputStream cos = new CheckedOutputStream(fos, checksum); - final ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(cos))) { - ZipUtils.compressDirectoryToZipFile(rootDir, sourceDir, zos); - zos.flush(); - fos.getFD().sync(); - } - } - - private static void compressDirectoryToZipFile(final String rootDir, final String sourceDir, - final ZipOutputStream zos) throws IOException { - final String dir = Paths.get(rootDir, sourceDir).toString(); - final File[] files = new File(dir).listFiles(); - for (final File file : files) { - final String child = Paths.get(sourceDir, file.getName()).toString(); - if (file.isDirectory()) { - compressDirectoryToZipFile(rootDir, child, zos); - } else { - zos.putNextEntry(new ZipEntry(child)); - try (final FileInputStream fis = new FileInputStream(file); - final BufferedInputStream bis = new BufferedInputStream(fis)) { - IOUtils.copy(bis, zos); - } - } - } - } - - public static void decompress(final String sourceFile, final String outputDir, - final Checksum checksum) throws IOException { - try (final FileInputStream fis = new FileInputStream(sourceFile); - final CheckedInputStream cis = new CheckedInputStream(fis, checksum); - final ZipInputStream zis = new ZipInputStream(new BufferedInputStream(cis))) { - ZipEntry entry; - while ((entry = zis.getNextEntry()) != null) { - final String fileName = entry.getName(); - final File entryFile = new File(Paths.get(outputDir, fileName).toString()); - FileUtils.forceMkdir(entryFile.getParentFile()); - try (final FileOutputStream fos = new FileOutputStream(entryFile); - final BufferedOutputStream bos = new BufferedOutputStream(fos)) { - IOUtils.copy(zis, bos); - bos.flush(); - fos.getFD().sync(); - } - } - IOUtils.copy(cis, NullOutputStream.NULL_OUTPUT_STREAM); - } - } -} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/HeartbeatService.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/HeartbeatService.java similarity index 71% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/HeartbeatService.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/HeartbeatService.java index 9c84453dd..095b17857 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/HeartbeatService.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/HeartbeatService.java @@ -1,4 +1,37 @@ -package com.baidu.hugegraph.store; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.meta.PartitionRole; +import org.apache.hugegraph.store.meta.Store; +import org.apache.hugegraph.store.meta.StoreMetadata; +import org.apache.hugegraph.store.options.HgStoreEngineOptions; +import org.apache.hugegraph.store.options.RaftRocksdbOptions; +import org.apache.hugegraph.store.pd.PdProvider; +import org.apache.hugegraph.store.util.IpUtil; +import org.apache.hugegraph.store.util.Lifecycle; +import org.rocksdb.MemoryUsageType; import com.alipay.sofa.jraft.core.Replicator; import com.alipay.sofa.jraft.entity.PeerId; @@ -6,25 +39,15 @@ import com.alipay.sofa.jraft.util.Utils; import com.baidu.hugegraph.pd.common.PDException; import com.baidu.hugegraph.pd.grpc.Metapb; import com.baidu.hugegraph.pd.grpc.Pdpb; -import com.baidu.hugegraph.store.options.HgStoreEngineOptions; -import com.baidu.hugegraph.store.options.RaftRocksdbOptions; import com.baidu.hugegraph.store.meta.*; -import com.baidu.hugegraph.store.pd.PdProvider; -import com.baidu.hugegraph.store.util.IpUtil; -import com.baidu.hugegraph.store.util.Lifecycle; -import lombok.extern.slf4j.Slf4j; -import org.rocksdb.MemoryUsageType; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import lombok.extern.slf4j.Slf4j; /** * Register and heartbeat, Keep the system online */ @Slf4j -public class HeartbeatService implements Lifecycle , PartitionStateListener{ +public class HeartbeatService implements Lifecycle, PartitionStateListener { private static final int MAX_HEARTBEAT_RETRY_COUNT = 5; // 心跳重试次数 private static final int REGISTER_RETRY_INTERVAL = 1; //注册重试时间间隔，单位秒 @@ -36,9 +59,9 @@ public class HeartbeatService implements Lifecycle , Parti private StoreMetadata storeMetadata; private final HgStoreEngine storeEngine; - private List stateListeners; - private Object partitionThreadLock = new Object(); - private Object storeThreadLock = new Object(); + private final List stateListeners; + private final Object partitionThreadLock = new Object(); + private final Object storeThreadLock = new Object(); // 心跳失败次数 private int heartbeatFailCount = 0; private int reportErrCount = 0; @@ -55,8 +78,9 @@ public class HeartbeatService implements Lifecycle , Parti public boolean init(HgStoreEngineOptions opts) { this.options = opts; storeInfo = storeMetadata.getStore(); - if (storeInfo == null) + if (storeInfo == null) { storeInfo = new Store(); + } storeInfo.setStoreAddress(options.getGrpcAddress()); storeInfo.setPdAddress(options.getPdAddress()); storeInfo.setRaftAddress(options.getRaftAddress()); @@ -92,12 +116,16 @@ public class HeartbeatService implements Lifecycle , Parti public Store getStoreInfo() { return storeInfo; } - public void setStoreMetadata(StoreMetadata storeMetadata){ + + public void setStoreMetadata(StoreMetadata storeMetadata) { this.storeMetadata = storeMetadata; } // 集群是否准备就绪 - public boolean isClusterReady() { return clusterStats.getState() == Metapb.ClusterState.Cluster_OK;} + public boolean isClusterReady() { + return clusterStats.getState() == Metapb.ClusterState.Cluster_OK; + } + /** * 服务状态有四种 * 就绪，在线、离线、死亡（从集群排除） @@ -118,7 +146,7 @@ public class HeartbeatService implements Lifecycle , Parti break; } - synchronized (storeThreadLock){ + synchronized (storeThreadLock) { storeThreadLock.wait(timerNextDelay); } } catch (Throwable e) { @@ -134,8 +162,6 @@ public class HeartbeatService implements Lifecycle , Parti } catch (Exception e) { log.error("doPartitionHeartbeat error: ", e); - } finally { - } try { synchronized (partitionThreadLock) { @@ -160,18 +186,18 @@ public class HeartbeatService implements Lifecycle , Parti this.clusterStats = pdProvider.getClusterStats(); if (clusterStats.getState() == Metapb.ClusterState.Cluster_OK) { timerNextDelay = options.getStoreHBInterval() * 1000; - } - else { + } else { timerNextDelay = REGISTER_RETRY_INTERVAL * 1000; } log.info("Register Store id= {} successfully. store = {}, clusterStats {}", - storeInfo.getId(), storeInfo, this.clusterStats); + storeInfo.getId(), storeInfo, this.clusterStats); // 监听partition消息 - pdProvider.startHeartbeatStream(error->{ + pdProvider.startHeartbeatStream(error -> { onStateChanged(Metapb.StoreState.Offline); timerNextDelay = REGISTER_RETRY_INTERVAL * 1000; wakeupHeartbeatThread(); - log.error("Connection closed. The store state changes to {}", Metapb.StoreState.Offline); + log.error("Connection closed. The store state changes to {}", + Metapb.StoreState.Offline); }); onStateChanged(Metapb.StoreState.Up); } else { @@ -180,16 +206,20 @@ public class HeartbeatService implements Lifecycle , Parti } catch (PDException e) { int exceptCode = e.getErrorCode(); if (exceptCode == Pdpb.ErrorType.STORE_ID_NOT_EXIST_VALUE) { - log.error("The store ID {} does not match the PD. Check that the correct PD is connected, " + - "and then delete the store ID!!!", + log.error( + "The store ID {} does not match the PD. Check that the correct PD is " + + "connected, " + + "and then delete the store ID!!!", storeInfo.getId()); System.exit(-1); } else if (exceptCode == Pdpb.ErrorType.STORE_HAS_BEEN_REMOVED_VALUE) { log.error("The store ID {} has been removed, please delete all data and restart!", - storeInfo.getId()); + storeInfo.getId()); System.exit(-1); } else if (exceptCode == Pdpb.ErrorType.STORE_PROHIBIT_DUPLICATE_VALUE) { - log.error("The store ID {} maybe duplicated, please check out store raft address and restart later!", + log.error( + "The store ID {} maybe duplicated, please check out store raft address " + + "and restart later!", storeInfo.getId()); System.exit(-1); } @@ -207,24 +237,26 @@ public class HeartbeatService implements Lifecycle , Parti int exceptCode = e.getErrorCode(); if (exceptCode == Pdpb.ErrorType.STORE_ID_NOT_EXIST_VALUE) { log.error("The store ID {} does not match the PD. Check that the correct PD is " + - "connected, and then delete the store ID!!!", storeInfo.getId()); + "connected, and then delete the store ID!!!", storeInfo.getId()); System.exit(-1); } else if (exceptCode == Pdpb.ErrorType.STORE_HAS_BEEN_REMOVED_VALUE) { log.error("The store ID {} has been removed, please delete all data and restart!", - storeInfo.getId()); + storeInfo.getId()); System.exit(-1); } } if (clusterStats.getState().getNumber() >= Metapb.ClusterState.Cluster_Fault.getNumber()) { - if (reportErrCount == 0) + if (reportErrCount == 0) { log.info("The cluster is abnormal, {}", clusterStats); + } reportErrCount = (++reportErrCount) % 30; } - if ( clusterStats.getState() == Metapb.ClusterState.Cluster_OK) + if (clusterStats.getState() == Metapb.ClusterState.Cluster_OK) { timerNextDelay = options.getStoreHBInterval() * 1000; - else + } else { timerNextDelay = REGISTER_RETRY_INTERVAL * 1000; + } if (clusterStats.getState() == Metapb.ClusterState.Cluster_Fault) { heartbeatFailCount++; @@ -236,7 +268,8 @@ public class HeartbeatService implements Lifecycle , Parti onStateChanged(Metapb.StoreState.Offline); timerNextDelay = REGISTER_RETRY_INTERVAL * 1000; this.clusterStats = clusterStats; - log.error("Store heart beat failure. The store state changes to {}", Metapb.StoreState.Offline); + log.error("Store heart beat failure. The store state changes to {}", + Metapb.StoreState.Offline); } } @@ -245,23 +278,25 @@ public class HeartbeatService implements Lifecycle , Parti Metapb.StoreState oldState = this.storeInfo.getState(); this.storeInfo.setState(newState); stateListeners.forEach((e) -> - e.stateChanged(this.storeInfo, oldState, newState)); + e.stateChanged(this.storeInfo, oldState, newState)); }); } protected void partitionHeartbeat() { - if ( storeEngine == null) return; + if (storeEngine == null) return; List partitions = storeEngine.getLeaderPartition(); final List statsList = new ArrayList<>(partitions.size()); Metapb.Shard localLeader = Metapb.Shard.newBuilder() - .setStoreId(storeEngine.getPartitionManager().getStore().getId()) - .setRole(Metapb.ShardRole.Leader) - .build(); + .setStoreId( + storeEngine.getPartitionManager().getStore() + .getId()) + .setRole(Metapb.ShardRole.Leader) + .build(); // 获取各个shard信息. - for(PartitionEngine partition : partitions) { + for (PartitionEngine partition : partitions) { Metapb.PartitionStats.Builder stats = Metapb.PartitionStats.newBuilder(); stats.setId(partition.getGroupId()); stats.addAllGraphName(partition.getPartitions().keySet()); @@ -278,16 +313,19 @@ public class HeartbeatService implements Lifecycle , Parti partition.getShardGroup().getShards().forEach(shard -> { Metapb.ShardState state = Metapb.ShardState.SState_Normal; if (aliveShards.containsKey(shard.getStoreId())) { - Replicator.State s = partition.getRaftNode().getReplicatorState(aliveShards.get(shard.getStoreId())); - if (s == Replicator.State.Snapshot) + Replicator.State s = partition.getRaftNode().getReplicatorState( + aliveShards.get(shard.getStoreId())); + if (s == Replicator.State.Snapshot) { state = Metapb.ShardState.SState_Snapshot; - } else + } + } else { state = Metapb.ShardState.SState_Offline; + } shardStats.add(Metapb.ShardStats.newBuilder() - .setStoreId(shard.getStoreId()) - .setRole(shard.getRole()) - .setState(state).build()); + .setStoreId(shard.getStoreId()) + .setRole(shard.getRole()) + .setState(state).build()); }); stats.addAllShardStats(shardStats); stats.setTimestamp(System.currentTimeMillis()); @@ -295,7 +333,7 @@ public class HeartbeatService implements Lifecycle , Parti statsList.add(stats.build()); } // 发送心跳 - if ( statsList.size() > 0 ) { + if (statsList.size() > 0) { pdProvider.partitionHeartbeat(statsList); } @@ -307,9 +345,11 @@ public class HeartbeatService implements Lifecycle , Parti Map mems = storeEngine.getBusinessHandler().getApproximateMemoryUsageByType(null); - if (mems.get(MemoryUsageType.kCacheTotal) > RaftRocksdbOptions.getWriteCacheCapacity() * 0.9 && - mems.get(MemoryUsageType.kMemTableUnFlushed) > RaftRocksdbOptions.getWriteCacheCapacity() * 0.1) { - // storeEngine.getBusinessHandler().flushAll(); + if (mems.get(MemoryUsageType.kCacheTotal) > + RaftRocksdbOptions.getWriteCacheCapacity() * 0.9 && + mems.get(MemoryUsageType.kMemTableUnFlushed) > + RaftRocksdbOptions.getWriteCacheCapacity() * 0.1) { + // storeEngine.getBusinessHandler().flushAll(); log.warn("Less memory, start flush dbs, {}", mems); } } catch (Exception e) { @@ -328,7 +368,7 @@ public class HeartbeatService implements Lifecycle , Parti @Override public void partitionRoleChanged(Partition partition, PartitionRole newRole) { - if ( newRole == PartitionRole.LEADER ){ + if (newRole == PartitionRole.LEADER) { // leader发生改变，激活心跳 synchronized (partitionThreadLock) { partitionThreadLock.notifyAll(); @@ -337,15 +377,16 @@ public class HeartbeatService implements Lifecycle , Parti } @Override - public void partitionShardChanged(Partition partition, List oldShards, List newShards) { - if ( partition.isLeader()){ + public void partitionShardChanged(Partition partition, List oldShards, + List newShards) { + if (partition.isLeader()) { synchronized (partitionThreadLock) { partitionThreadLock.notifyAll(); } } } - private void wakeupHeartbeatThread(){ + private void wakeupHeartbeatThread() { synchronized (storeThreadLock) { storeThreadLock.notifyAll(); } diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/HgStoreEngine.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/HgStoreEngine.java similarity index 68% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/HgStoreEngine.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/HgStoreEngine.java index 88df8a622..29c52c138 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/HgStoreEngine.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/HgStoreEngine.java @@ -1,4 +1,52 @@ -package com.baidu.hugegraph.store; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.store.business.BusinessHandler; +import org.apache.hugegraph.store.business.BusinessHandlerImpl; +import org.apache.hugegraph.store.business.DataMover; +import org.apache.hugegraph.store.cmd.HgCmdClient; +import org.apache.hugegraph.store.cmd.HgCmdProcessor; +import org.apache.hugegraph.store.cmd.UpdatePartitionRequest; +import org.apache.hugegraph.store.cmd.UpdatePartitionResponse; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.meta.PartitionManager; +import org.apache.hugegraph.store.meta.ShardGroup; +import org.apache.hugegraph.store.meta.Store; +import org.apache.hugegraph.store.metric.HgMetricService; +import org.apache.hugegraph.store.options.HgStoreEngineOptions; +import org.apache.hugegraph.store.options.PartitionEngineOptions; +import org.apache.hugegraph.store.pd.DefaultPdProvider; +import org.apache.hugegraph.store.pd.FakePdServiceProvider; +import org.apache.hugegraph.store.pd.PdProvider; +import org.apache.hugegraph.store.raft.RaftClosure; +import org.apache.hugegraph.store.raft.RaftOperation; +import org.apache.hugegraph.store.util.HgRaftError; +import org.apache.hugegraph.store.util.Lifecycle; import com.alipay.sofa.jraft.JRaftUtils; import com.alipay.sofa.jraft.Status; @@ -11,31 +59,9 @@ import com.alipay.sofa.jraft.rpc.RpcServer; import com.alipay.sofa.jraft.util.Endpoint; import com.baidu.hugegraph.pd.common.PDException; import com.baidu.hugegraph.pd.grpc.Metapb; -import org.apache.hugegraph.rocksdb.access.RocksDBFactory; -import com.baidu.hugegraph.store.business.DataMover; -import com.baidu.hugegraph.store.business.BusinessHandler; -import com.baidu.hugegraph.store.business.BusinessHandlerImpl; -import com.baidu.hugegraph.store.cmd.HgCmdClient; -import com.baidu.hugegraph.store.cmd.HgCmdProcessor; - -import com.baidu.hugegraph.store.cmd.UpdatePartitionRequest; -import com.baidu.hugegraph.store.cmd.UpdatePartitionResponse; -import com.baidu.hugegraph.store.metric.HgMetricService; -import com.baidu.hugegraph.store.options.HgStoreEngineOptions; -import com.baidu.hugegraph.store.options.PartitionEngineOptions; import com.baidu.hugegraph.store.meta.*; -import com.baidu.hugegraph.store.pd.DefaultPdProvider; -import com.baidu.hugegraph.store.pd.FakePdServiceProvider; -import com.baidu.hugegraph.store.pd.PdProvider; -import com.baidu.hugegraph.store.raft.RaftClosure; -import com.baidu.hugegraph.store.raft.RaftOperation; -import com.baidu.hugegraph.store.util.HgRaftError; -import com.baidu.hugegraph.store.util.Lifecycle; -import lombok.extern.slf4j.Slf4j; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; +import lombok.extern.slf4j.Slf4j; /** * The core class of the storage engine, initializing PD client and raft client @@ -45,7 +71,10 @@ import java.util.concurrent.ConcurrentHashMap; @Slf4j public class HgStoreEngine implements Lifecycle, HgStoreStateListener { private final static HgStoreEngine instance = new HgStoreEngine(); - public static HgStoreEngine getInstance() { return instance; } + + public static HgStoreEngine getInstance() { + return instance; + } private RpcServer rpcServer; private HgStoreEngineOptions options; @@ -62,8 +91,7 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt private DataMover dataMover; - - private HgStoreEngine(){ + private HgStoreEngine() { } @@ -71,6 +99,7 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt * 1、读取StoreId，向pd注册，初次注册由PD生成StoreId，存储到本地 * 2、注册成功，启动raft服务 * 3、定时发送Store心跳和Partition心跳，与PD保持联系 + * * @param opts * @return */ @@ -85,9 +114,9 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt BusinessHandlerImpl.initRocksdb(opts.getRocksdbConfig(), getRocksdbListener()); - if (opts.isFakePD()) + if (opts.isFakePD()) { pdProvider = new FakePdServiceProvider(opts.getFakePdOptions()); - else { + } else { pdProvider = new DefaultPdProvider(opts.getPdAddress()); pdProvider.addPartitionInstructionListener(new PartitionInstructionProcessor(this)); } @@ -107,11 +136,13 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt hgCmdClient.init(new RpcOptions(), (graphName, ptId) -> { // 分裂的时候，还未及时的上报pd if (getPartitionEngine(ptId) != null) { - return getPartitionEngine(ptId).waitForLeader(options.getWaitLeaderTimeout() * 1000); + return getPartitionEngine(ptId).waitForLeader( + options.getWaitLeaderTimeout() * 1000); } else { // 可能出现跨分区的迁移 Metapb.Shard shard = pdProvider.getPartitionLeader(graphName, ptId); - return JRaftUtils.getEndPoint(pdProvider.getStoreByID(shard.getStoreId()).getRaftAddress()); + return JRaftUtils.getEndPoint( + pdProvider.getStoreByID(shard.getStoreId()).getRaftAddress()); } }); @@ -123,7 +154,7 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt metricService.setHgStoreEngine(this).init(null); dataMover = opts.getDataTransfer(); - if ( dataMover != null){ + if (dataMover != null) { this.dataMover.setBusinessHandler(this.businessHandler); this.dataMover.setCmdClient(hgCmdClient); } @@ -136,8 +167,10 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt private RpcServer createRaftRpcServer(String raftAddr) { Endpoint endpoint = JRaftUtils.getEndPoint(raftAddr); RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(endpoint, - JRaftUtils.createExecutor("RAFT-RPC-", options.getRaftRpcThreadPoolSize()), - null); + JRaftUtils.createExecutor( + "RAFT-RPC-", + options.getRaftRpcThreadPoolSize()), + null); HgCmdProcessor.registerProcessor(rpcServer, this); rpcServer.init(null); return rpcServer; @@ -153,7 +186,8 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt }); partitionEngines.clear(); rpcServer.shutdown(); - // HgStoreEngine.init function check rpcServer whether is null, skipped if the instance exists even shut down. + // HgStoreEngine.init function check rpcServer whether is null, skipped if the instance + // exists even shut down. rpcServer = null; heartbeatService.shutdown(); metricService.shutdown(); @@ -167,6 +201,7 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt v.snapshot(); }); } + /** * Store注册状态发生改变 */ @@ -190,7 +225,8 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt try { if (!options.isFakePD()) { // FakePD模式不需要同步 partitionManager.syncPartitionsFromPD(partition -> { - log.warn("The local partition information is inconsistent with the PD server. " + + log.warn( + "The local partition information is inconsistent with the PD server. " + "Please delete the redundant data manually, {}", partition); }); } @@ -198,8 +234,9 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt g.forEach((id, p) -> { try { createPartitionEngine(p); - }catch (Exception e){ - log.error("Partition {}-{} restore exception {}", p.getGraphName(), p.getId(), e); + } catch (Exception e) { + log.error("Partition {}-{} restore exception {}", p.getGraphName(), + p.getId(), e); } }); }); @@ -210,13 +247,14 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt /** * 收到store raft addr 变更，需要重新创建raft group + * * @param storeId 变更的store id */ - public void rebuildRaftGroup(long storeId){ + public void rebuildRaftGroup(long storeId) { partitionEngines.forEach((partId, engine) -> { try { var partitions = pdProvider.getPartitionsByStore(storeId); - if (partitions.size() > 0 ) { + if (partitions.size() > 0) { var shards = pdProvider.getShardGroup(partId).getShardsList(); if (shards.stream().anyMatch(s -> s.getStoreId() == storeId)) { var peers = partitionManager.shards2Peers(shards); @@ -224,7 +262,8 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt if (initConf == null) { engine.getOptions().setPeerList(peers); } else { - peers.stream().forEach(peer -> initConf.addPeer(JRaftUtils.getPeerId(peer))); + peers.stream() + .forEach(peer -> initConf.addPeer(JRaftUtils.getPeerId(peer))); } // engine.getOptions().getConf().setPeers(); @@ -243,35 +282,39 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt * @param partition * @return */ - public PartitionEngine createPartitionEngine(Partition partition){ + public PartitionEngine createPartitionEngine(Partition partition) { return createPartitionEngine(partition, null); } - public PartitionEngine createPartitionEngine(Partition partition, Configuration conf){ + public PartitionEngine createPartitionEngine(Partition partition, Configuration conf) { partitionManager.updatePartition(partition, false); var shardGroup = partitionManager.getShardGroup(partition.getId()); return createPartitionEngine(partition.getId(), shardGroup, conf); } - private PartitionEngine createPartitionEngine(int groupId, ShardGroup shardGroup, Configuration conf){ + private PartitionEngine createPartitionEngine(int groupId, ShardGroup shardGroup, + Configuration conf) { PartitionEngine engine; if ((engine = partitionEngines.get(groupId)) == null) { synchronized (this) { // 分区分裂时特殊情况(集群中图分区数量不一样)，会导致分裂的分区，可能不在本机器上. - if (conf != null){ + if (conf != null) { var list = conf.listPeers(); list.addAll(conf.listLearners()); - if (! list.stream().anyMatch(p -> p.getEndpoint().toString().equals(options.getRaftAddress()))) { - log.info("raft {}, conf {} does not contains raft address:{}, skipped create partition engine", + if (!list.stream().anyMatch( + p -> p.getEndpoint().toString().equals(options.getRaftAddress()))) { + log.info( + "raft {}, conf {} does not contains raft address:{}, skipped " + + "create partition engine", groupId, conf, options.getRaftAddress()); return null; } - }else { + } else { var storeId = partitionManager.getStore().getId(); - if (!shardGroup.getShards().stream().anyMatch(s -> s.getStoreId() == storeId)){ + if (!shardGroup.getShards().stream().anyMatch(s -> s.getStoreId() == storeId)) { log.info("raft {}, shard group {} does not contains current storeId {}, " + - "skipped create partition engine", groupId, shardGroup, storeId); + "skipped create partition engine", groupId, shardGroup, storeId); return null; } } @@ -281,10 +324,9 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt engine = new PartitionEngine(this, shardGroup); PartitionEngineOptions ptOpts = new PartitionEngineOptions(); - if ( conf != null) { + if (conf != null) { ptOpts.setConf(conf); - } - else { + } else { ptOpts.setPeerList(partitionManager.getPartitionPeers(shardGroup)); } ptOpts.setGroupId(groupId); @@ -313,6 +355,7 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt * 1、遍历partition.shards * 2、根据storeId获取Store信息 * 3、建立向其他store的raft rpc，发送StartRaft消息 + * * @param partition * @return */ @@ -333,11 +376,14 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt return; } // 向其他peer发消息，创建raft 分组。此处是异步发送 - hgCmdClient.createRaftNode(store.getRaftAddress(), Arrays.asList(partition), - status -> { - log.info("send to {} createRaftNode rpc call result {} partitionId {}", - store.getRaftAddress(), status, partition.getId()); - }); + hgCmdClient.createRaftNode(store.getRaftAddress(), List.of(partition), + status -> { + log.info( + "send to {} createRaftNode rpc call " + + "result {} partitionId {}", + store.getRaftAddress(), status, + partition.getId()); + }); }); } } else { @@ -356,21 +402,27 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt return; } // 向其他peer发消息，创建raft 分组。此处是异步发送 - hgCmdClient.destroyRaftNode(store.getRaftAddress(), Arrays.asList(new Partition[]{partition}), - status -> { - log.info("send to {} - {} DestroyRaftNode rpc call result {}", - store.getRaftAddress(), partition.getId(), status); - }); + hgCmdClient.destroyRaftNode(store.getRaftAddress(), + Arrays.asList(new Partition[]{partition}), + status -> { + log.info( + "send to {} - {} DestroyRaftNode rpc call" + + " result {}", + store.getRaftAddress(), partition.getId(), + status); + }); }); } } + /** * 停止分区，并销毁数据 */ public synchronized void destroyPartitionEngine(Integer groupId, List graphNames) { log.info("Partition {} start to be destroyed", groupId); - if (!partitionEngines.containsKey(groupId)) + if (!partitionEngines.containsKey(groupId)) { return; + } PartitionEngine ptEngine = partitionEngines.get(groupId); graphNames.forEach(graphName -> { ptEngine.removePartition(graphName); @@ -383,7 +435,7 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt partitionEngines.remove(groupId); // 删除对应的db文件夹 businessHandler.destroyGraphDB(graphNames.get(0), groupId); - }else { + } else { graphNames.forEach(graphName -> { businessHandler.dbCompaction(graphName, groupId); }); @@ -396,8 +448,9 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt */ public void deletePartition(Integer groupId, String graphName) { log.info("Partition {}-{} deletePartition", graphName, groupId); - if (!partitionEngines.containsKey(groupId)) + if (!partitionEngines.containsKey(groupId)) { return; + } PartitionEngine ptEngine = partitionEngines.get(groupId); ptEngine.removePartition(graphName); // 删除数据 @@ -409,29 +462,34 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt } } } + /** * 获取所有的leader分区 + * * @return */ public List getLeaderPartition() { List partitions = new ArrayList<>(); this.partitionEngines.forEach((k, v) -> { - if (v.isLeader()) + if (v.isLeader()) { partitions.add(v); + } }); return partitions; } /** * 获取分区所有活跃的peer + * * @return */ - public Map getAlivePeers(int groupId){ + public Map getAlivePeers(int groupId) { PartitionEngine engine = this.partitionEngines.get(groupId); try { - if (engine != null) + if (engine != null) { return engine.getAlivePeers(); - }catch (Exception e){ + } + } catch (Exception e) { log.error("getAlivePeers {}", e); } return new HashMap<>(); @@ -439,17 +497,18 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt /** * 获取分区的最后提交的日志id + * * @param groupId * @return */ - public long getLeaderTerm(int groupId){ + public long getLeaderTerm(int groupId) { PartitionEngine engine = this.partitionEngines.get(groupId); return engine.getLeaderTerm(); } - public long getCommittedIndex(int groupId){ + public long getCommittedIndex(int groupId) { PartitionEngine engine = this.partitionEngines.get(groupId); - if (engine != null ) return engine.getCommittedIndex(); + if (engine != null) return engine.getCommittedIndex(); return 0; } @@ -462,9 +521,11 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt return partitionManager; } - public DataMover getDataMover(){ return dataMover; } + public DataMover getDataMover() { + return dataMover; + } - public PdProvider getPdProvider(){ + public PdProvider getPdProvider() { return pdProvider; } @@ -472,29 +533,38 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt return businessHandler; } - public HgCmdClient getHgCmdClient(){ return hgCmdClient;} + public HgCmdClient getHgCmdClient() { + return hgCmdClient; + } - public HeartbeatService getHeartbeatService() { return heartbeatService; } + public HeartbeatService getHeartbeatService() { + return heartbeatService; + } - public boolean isClusterReady() { return heartbeatService.isClusterReady();} + public boolean isClusterReady() { + return heartbeatService.isClusterReady(); + } public List getDataLocations() { return partitionManager.getStoreMetadata().getDataLocations(); } + /** * 添加raft任务 * 1、检查partition是否存在 - * 1.1、如果不存在，则向PD查询分区是否属于本地 - * 1.1.1 如果分区属于本地，则创建raft分组,并通知其他Store - * 1.1.2 如果分区不属于本地，则抛出异常 - * 1.2 检查Partition是否是leader - * 1.2.1 如果是leader，则提交任务 - * 1.2.2 否则，返回错误 + * 1.1、如果不存在，则向PD查询分区是否属于本地 + * 1.1.1 如果分区属于本地，则创建raft分组,并通知其他Store + * 1.1.2 如果分区不属于本地，则抛出异常 + * 1.2 检查Partition是否是leader + * 1.2.1 如果是leader，则提交任务 + * 1.2.2 否则，返回错误 + * * @param partId * @param operation */ - public void addRaftTask(String graphName, Integer partId, RaftOperation operation, RaftClosure closure) { + public void addRaftTask(String graphName, Integer partId, RaftOperation operation, + RaftClosure closure) { PartitionEngine engine = getPartitionEngine(graphName, partId); if (engine == null) { synchronized (this) { @@ -519,8 +589,9 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt engine.addRaftTask(operation, closure); } else if (leader != null) { // 当前不是leader，返回leader所在的storeId - Store store = partitionManager.getStoreByRaftEndpoint(engine.getShardGroup(), leader.toString()); - if (store.getId() == 0){ + Store store = partitionManager.getStoreByRaftEndpoint(engine.getShardGroup(), + leader.toString()); + if (store.getId() == 0) { // 本地未找到Leader的Store信息，可能Partition还未同步过来，重新向Leader获取。 Store leaderStore = hgCmdClient.getStoreInfo(leader.toString()); store = leaderStore != null ? leaderStore : store; @@ -530,15 +601,18 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt // Leader 不是本机，通知客户端 closure.onLeaderChanged(partId, store.getId()); closure.run(new Status(HgRaftError.NOT_LEADER.getNumber(), - String.format("Partition %s-%d leader changed to %x", graphName, partId, store.getId()))); - log.error("Raft Partition {}-{} not leader, redirectTo leader {}.", graphName, partId, leader); + String.format("Partition %s-%d leader changed to %x", + graphName, partId, store.getId()))); + log.error("Raft Partition {}-{} not leader, redirectTo leader {}.", graphName, + partId, leader); } else { closure.run(new Status(HgRaftError.WAIT_LEADER_TIMEOUT.getNumber(), - HgRaftError.WAIT_LEADER_TIMEOUT.getMsg())); + HgRaftError.WAIT_LEADER_TIMEOUT.getMsg())); log.error("Partition {}-{} waiting for leader timeout.", graphName, partId); } } else { - closure.run(new Status(HgRaftError.NOT_LOCAL.getNumber(), HgRaftError.NOT_LOCAL.getMsg())); + closure.run( + new Status(HgRaftError.NOT_LOCAL.getNumber(), HgRaftError.NOT_LOCAL.getMsg())); log.error("Partition {}-{} does not belong to local store.", graphName, partId); } } @@ -548,20 +622,22 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt PartitionEngine engine = partitionEngines.get(partitionId); return engine; } - public PartitionEngine getPartitionEngine( String graphName, Integer partitionId) { + + public PartitionEngine getPartitionEngine(String graphName, Integer partitionId) { PartitionEngine engine = partitionEngines.get(partitionId); - if ( engine != null && engine.hasPartition(graphName)) + if (engine != null && engine.hasPartition(graphName)) { return engine; + } return null; } - public Map getPartitionEngines(){ + public Map getPartitionEngines() { return partitionEngines; } - public Map getNodeMetrics(){ + public Map getNodeMetrics() { Map metrics = new HashMap(); - partitionEngines.forEach((k,v)->{ + partitionEngines.forEach((k, v) -> { metrics.put(Integer.toString(k), v.getNodeMetrics()); }); return metrics; @@ -569,17 +645,19 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt /** * Number of raft-group. + * * @return */ - public int getRaftGroupCount(){ + public int getRaftGroupCount() { return partitionEngines.size(); } /** * 监听rocksdb事件 + * * @return */ - private RocksDBFactory.RocksdbChangedListener getRocksdbListener(){ + private RocksDBFactory.RocksdbChangedListener getRocksdbListener() { return new RocksDBFactory.RocksdbChangedListener() { @Override public void onCompacted(String dbName) { @@ -587,9 +665,10 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt try { Integer groupId = Integer.parseInt(sid); PartitionEngine engine = getPartitionEngine(groupId); - if (engine != null) + if (engine != null) { engine.addBlankRaftTask(); - }catch (Exception e){ + } + } catch (Exception e) { } } @@ -597,7 +676,7 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt } // For test - public void setPartitionManager(PartitionManager ptm){ + public void setPartitionManager(PartitionManager ptm) { this.partitionManager = ptm; } @@ -612,18 +691,22 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt if (engine != null && engine.isLeader()) { try { - engine.addRaftTask(RaftOperation.create(RaftOperation.SYNC_PARTITION, partition.getProtoObj()), - new RaftClosure() { - @Override - public void run(Status status) { - log.info("Partition {}-{}-{} sync partition status is {}", - partition.getGraphName(), partition.getId(), partition.getWorkState(), - status); - } - }); + engine.addRaftTask(RaftOperation.create(RaftOperation.SYNC_PARTITION, + partition.getProtoObj()), + new RaftClosure() { + @Override + public void run(Status status) { + log.info( + "Partition {}-{}-{} sync partition status " + + "is {}", + partition.getGraphName(), partition.getId(), + partition.getWorkState(), + status); + } + }); } catch (IOException e) { log.error("Partition {}-{} sync partition exception {}", - partition.getGraphName(), partition.getId(), e); + partition.getGraphName(), partition.getId(), e); } } } @@ -638,8 +721,9 @@ public class HgStoreEngine implements Lifecycle, HgStoreSt response = hgCmdClient.raftUpdatePartition(request); log.info("not leader request threadId:{} pId:{} range:{}-{} state:{} response:{}", - Thread.currentThread().getId(), request.getPartitionId(), request.getStartKey(), - request.getEndKey(), request.getWorkState(), response.getStatus()); + Thread.currentThread().getId(), request.getPartitionId(), + request.getStartKey(), + request.getEndKey(), request.getWorkState(), response.getStatus()); } catch (Exception e) { e.printStackTrace(); diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/HgStoreStateListener.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/HgStoreStateListener.java new file mode 100644 index 000000000..2ffd5a345 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/HgStoreStateListener.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import org.apache.hugegraph.store.meta.Store; + +import com.baidu.hugegraph.pd.grpc.Metapb; + +public interface HgStoreStateListener { + void stateChanged(Store store, Metapb.StoreState oldState, Metapb.StoreState newState); +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/PartitionEngine.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/PartitionEngine.java similarity index 69% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/PartitionEngine.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/PartitionEngine.java index 5699d2c5f..2c6222d43 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/PartitionEngine.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/PartitionEngine.java @@ -1,4 +1,63 @@ -package com.baidu.hugegraph.store; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import static com.baidu.hugegraph.pd.grpc.MetaTask.TaskType.Clean_Partition; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.collections.ListUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hugegraph.store.cmd.BatchPutRequest; +import org.apache.hugegraph.store.cmd.CleanDataRequest; +import org.apache.hugegraph.store.cmd.DbCompactionRequest; +import org.apache.hugegraph.store.cmd.HgCmdClient; +import org.apache.hugegraph.store.cmd.UpdatePartitionRequest; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.meta.PartitionManager; +import org.apache.hugegraph.store.meta.Shard; +import org.apache.hugegraph.store.meta.ShardGroup; +import org.apache.hugegraph.store.meta.Store; +import org.apache.hugegraph.store.meta.TaskManager; +import org.apache.hugegraph.store.options.PartitionEngineOptions; +import org.apache.hugegraph.store.raft.HgStoreStateMachine; +import org.apache.hugegraph.store.raft.RaftClosure; +import org.apache.hugegraph.store.raft.RaftOperation; +import org.apache.hugegraph.store.raft.RaftStateListener; +import org.apache.hugegraph.store.raft.RaftTaskHandler; +import org.apache.hugegraph.store.raft.util.RaftUtils; +import org.apache.hugegraph.store.snapshot.HgSnapshotHandler; +import org.apache.hugegraph.store.util.FutureClosure; +import org.apache.hugegraph.store.util.HgRaftError; +import org.apache.hugegraph.store.util.HgStoreException; +import org.apache.hugegraph.store.util.Lifecycle; import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.JRaftUtils; @@ -20,53 +79,12 @@ import com.alipay.sofa.jraft.storage.log.RocksDBSegmentLogStorage; import com.alipay.sofa.jraft.util.Endpoint; import com.alipay.sofa.jraft.util.Utils; import com.alipay.sofa.jraft.util.internal.ThrowUtil; - import com.baidu.hugegraph.pd.common.PDException; import com.baidu.hugegraph.pd.grpc.MetaTask; import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.cmd.BatchPutRequest; -import com.baidu.hugegraph.store.cmd.CleanDataRequest; -import com.baidu.hugegraph.store.cmd.DbCompactionRequest; -import com.baidu.hugegraph.store.cmd.HgCmdClient; -import com.baidu.hugegraph.store.cmd.UpdatePartitionRequest; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.meta.PartitionManager; -import com.baidu.hugegraph.store.meta.Shard; -import com.baidu.hugegraph.store.meta.ShardGroup; -import com.baidu.hugegraph.store.meta.Store; -import com.baidu.hugegraph.store.meta.TaskManager; -import com.baidu.hugegraph.store.raft.HgStoreStateMachine; -import com.baidu.hugegraph.store.raft.RaftClosure; -import com.baidu.hugegraph.store.raft.RaftOperation; -import com.baidu.hugegraph.store.raft.RaftStateListener; -import com.baidu.hugegraph.store.raft.RaftTaskHandler; -import com.baidu.hugegraph.store.util.HgStoreException; -import com.baidu.hugegraph.store.options.PartitionEngineOptions; -import com.baidu.hugegraph.store.raft.util.RaftUtils; -import com.baidu.hugegraph.store.snapshot.HgSnapshotHandler; -import com.baidu.hugegraph.store.util.HgRaftError; -import com.baidu.hugegraph.store.util.FutureClosure; -import com.baidu.hugegraph.store.util.Lifecycle; import com.google.protobuf.CodedInputStream; + import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.ListUtils; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.atomic.AtomicBoolean; - -import static com.baidu.hugegraph.pd.grpc.MetaTask.TaskType.Clean_Partition; /** @@ -87,14 +105,14 @@ public class PartitionEngine implements Lifecycle, RaftS private TaskManager taskManager; private HgSnapshotHandler snapshotHandler; - private List stateListeners; + private final List stateListeners; private Node raftNode; - private ShardGroup shardGroup; + private final ShardGroup shardGroup; - private AtomicBoolean changingPeer; + private final AtomicBoolean changingPeer; - private AtomicBoolean snapshotFlag; - private final Object leaderChangedEvent = new String("leaderChangedEvent"); + private final AtomicBoolean snapshotFlag; + private final Object leaderChangedEvent = "leaderChangedEvent"; public PartitionEngine(HgStoreEngine storeEngine, ShardGroup shardGroup) { @@ -114,19 +132,21 @@ public class PartitionEngine implements Lifecycle, RaftS partitionManager.loadPartitionFromSnapshot(partition); } - public List loadPartitionsFromLocalDb(){ + public List loadPartitionsFromLocalDb() { return partitionManager.loadPartitionsFromDb(getGroupId()); } - public void removePartition(String graphName){ + public void removePartition(String graphName) { partitionManager.removePartition(graphName, options.getGroupId()); } - public boolean hasPartition(String graphName){ + public boolean hasPartition(String graphName) { return partitionManager.hasPartition(graphName, getGroupId()); } - public Integer getGroupId() { return options.getGroupId(); } + public Integer getGroupId() { + return options.getGroupId(); + } private static ThreadPoolExecutor raftLogWriteExecutor = null; @@ -134,12 +154,14 @@ public class PartitionEngine implements Lifecycle, RaftS public static ThreadPoolExecutor getRaftLogWriteExecutor() { if (raftLogWriteExecutor == null) { synchronized (PartitionEngine.class) { - if (raftLogWriteExecutor == null) + if (raftLogWriteExecutor == null) { raftLogWriteExecutor = RocksDBSegmentLogStorage.createDefaultWriteExecutor(); + } } } return raftLogWriteExecutor; } + /** * Initialize the raft engine * @@ -169,7 +191,7 @@ public class PartitionEngine implements Lifecycle, RaftS new File(options.getRaftDataPath()).mkdirs(); Configuration initConf = opts.getConf(); - if ( initConf == null ) { + if (initConf == null) { String peersList = StringUtils.join(options.getPeerList(), ","); initConf = new Configuration(); initConf.parse(peersList); @@ -192,31 +214,37 @@ public class PartitionEngine implements Lifecycle, RaftS nodeOptions.setSharedVoteTimer(true); nodeOptions.setFilterBeforeCopyRemote(true); - nodeOptions.setServiceFactory(new DefaultJRaftServiceFactory(){ + nodeOptions.setServiceFactory(new DefaultJRaftServiceFactory() { @Override public LogStorage createLogStorage(final String uri, final RaftOptions raftOptions) { - if ( options.getRaftOptions().isUseRocksDBSegmentLogStorage()) + if (options.getRaftOptions().isUseRocksDBSegmentLogStorage()) { return new RocksDBSegmentLogStorage(uri, raftOptions, - RocksDBSegmentLogStorage.DEFAULT_VALUE_SIZE_THRESHOLD, - options.getRaftOptions().getMaxSegmentFileSize(), - options.getRaftOptions().getPreAllocateSegmentCount(), - options.getRaftOptions().getKeepInMemorySegmentCount(), - RocksDBSegmentLogStorage.DEFAULT_CHECKPOINT_INTERVAL_MS, - getRaftLogWriteExecutor()); - else + RocksDBSegmentLogStorage.DEFAULT_VALUE_SIZE_THRESHOLD, + options.getRaftOptions() + .getMaxSegmentFileSize(), + options.getRaftOptions() + .getPreAllocateSegmentCount(), + options.getRaftOptions() + .getKeepInMemorySegmentCount(), + RocksDBSegmentLogStorage.DEFAULT_CHECKPOINT_INTERVAL_MS, + getRaftLogWriteExecutor()); + } else { return new RocksDBLogStorage(uri, raftOptions); + } } }); - // 初始集群 + // 初始集群 nodeOptions.setInitialConf(initConf); // 快照时间间隔 nodeOptions.setSnapshotIntervalSecs(options.getRaftOptions().getSnapshotIntervalSecs()); - //nodeOptions.setSnapshotLogIndexMargin(options.getRaftOptions().getSnapshotLogIndexMargin()); + //nodeOptions.setSnapshotLogIndexMargin(options.getRaftOptions() + // .getSnapshotLogIndexMargin()); nodeOptions.setRpcConnectTimeoutMs(options.getRaftOptions().getRpcConnectTimeoutMs()); nodeOptions.setRpcDefaultTimeout(options.getRaftOptions().getRpcDefaultTimeout()); - nodeOptions.setRpcInstallSnapshotTimeout(options.getRaftOptions().getRpcInstallSnapshotTimeout()); + nodeOptions.setRpcInstallSnapshotTimeout( + options.getRaftOptions().getRpcInstallSnapshotTimeout()); nodeOptions.setElectionTimeoutMs(options.getRaftOptions().getElectionTimeoutMs()); nodeOptions.setChannelWriteBufLowWaterMark(2 * 1024 * 1024); nodeOptions.setChannelWriteBufHighWaterMark(4 * 1024 * 1024); @@ -224,18 +252,19 @@ public class PartitionEngine implements Lifecycle, RaftS RaftOptions raftOptions = nodeOptions.getRaftOptions(); raftOptions.setDisruptorBufferSize(options.getRaftOptions().getDisruptorBufferSize()); raftOptions.setMaxEntriesSize(options.getRaftOptions().getMaxEntriesSize()); - raftOptions.setMaxReplicatorInflightMsgs(options.getRaftOptions().getMaxReplicatorInflightMsgs()); - raftOptions.setMaxByteCountPerRpc(1024*1024); + raftOptions.setMaxReplicatorInflightMsgs( + options.getRaftOptions().getMaxReplicatorInflightMsgs()); + raftOptions.setMaxByteCountPerRpc(1024 * 1024); nodeOptions.setEnableMetrics(true); - final PeerId serverId = JRaftUtils.getPeerId(options.getRaftAddress()); // 构建raft组并启动raft - this.raftGroupService = new RaftGroupService(raftPrefix + options.getGroupId(), serverId, nodeOptions, - storeEngine.getRaftRpcServer(), true); + this.raftGroupService = + new RaftGroupService(raftPrefix + options.getGroupId(), serverId, nodeOptions, + storeEngine.getRaftRpcServer(), true); this.raftNode = raftGroupService.start(false); this.raftNode.addReplicatorStateListener(new ReplicatorStateListener()); // 检查pd返回的peers是否和本地一致，如果不一致，重置peerlist @@ -246,7 +275,8 @@ public class PartitionEngine implements Lifecycle, RaftS started = true; } - log.info("PartitionEngine start successfully: id = {}, peers list = {}", options.getGroupId(), nodeOptions.getInitialConf().getPeers()); + log.info("PartitionEngine start successfully: id = {}, peers list = {}", + options.getGroupId(), nodeOptions.getInitialConf().getPeers()); return this.started; } @@ -255,25 +285,28 @@ public class PartitionEngine implements Lifecycle, RaftS return this.storeEngine; } - public ShardGroup getShardGroup() { return shardGroup; } + public ShardGroup getShardGroup() { + return shardGroup; + } /** - 1、收到PD发送的分区迁移指令，向状态机添加迁移任务，状态为新建 - 2、执行状态机消息，添加到任务队列，并执行任务 - 3、比较新旧peer，查找出新增和删除的peer - 4、如果存在新增的peer - 4.1、对于新增的peer，通知peer创建raft状态机 - 4.2、以learner方式加入raft group - 4.3、监听snapshot同步事件，重复执行步骤3 - 5、不存在新增的peer - 5.1、移除learner，并等待返回 - 5.2、修改learner为peer，加入raft group - 6、存在被删除的peer - 6.1、通知peer，删除状态机并删除数据 + * 1、收到PD发送的分区迁移指令，向状态机添加迁移任务，状态为新建 + * 2、执行状态机消息，添加到任务队列，并执行任务 + * 3、比较新旧peer，查找出新增和删除的peer + * 4、如果存在新增的peer + * 4.1、对于新增的peer，通知peer创建raft状态机 + * 4.2、以learner方式加入raft group + * 4.3、监听snapshot同步事件，重复执行步骤3 + * 5、不存在新增的peer + * 5.1、移除learner，并等待返回 + * 5.2、修改learner为peer，加入raft group + * 6、存在被删除的peer + * 6.1、通知peer，删除状态机并删除数据 + * * @param peers * @param done - * @return true表示完成，false表示未完成 + * @return true表示完成，false表示未完成 */ public Status changePeers(List peers, final Closure done) { if (ListUtils.isEqualList(peers, RaftUtils.getPeerEndpoints(raftNode))) { @@ -282,11 +315,13 @@ public class PartitionEngine implements Lifecycle, RaftS Status result = HgRaftError.TASK_CONTINUE.toStatus(); List oldPeers = RaftUtils.getAllEndpoints(raftNode); - log.info("Raft {} changePeers start, old peer is {}, new peer is {}", getGroupId(), oldPeers, peers); + log.info("Raft {} changePeers start, old peer is {}, new peer is {}", getGroupId(), + oldPeers, peers); // 检查需要新增的peer。 List addPeers = ListUtils.removeAll(peers, oldPeers); // 需要删除的learner。可能peer发生改变 - List removedPeers = ListUtils.removeAll(RaftUtils.getLearnerEndpoints(raftNode), peers); + List removedPeers = + ListUtils.removeAll(RaftUtils.getLearnerEndpoints(raftNode), peers); HgCmdClient rpcClient = storeEngine.getHgCmdClient(); // 生成新的Configuration对象 @@ -304,12 +339,14 @@ public class PartitionEngine implements Lifecycle, RaftS FutureClosure closure = new FutureClosure(addPeers.size()); addPeers.forEach(peer -> Utils.runInThread(() -> { // 1. 创建新peer的raft对象 - rpcClient.createRaftNode(peer, partitionManager.getPartitionList(getGroupId()), conf, status -> { - closure.run(status); - if (!status.isOk()) { - log.error("Raft {} add node {} error {}", options.getGroupId(), peer, status); - } - }); + rpcClient.createRaftNode(peer, partitionManager.getPartitionList(getGroupId()), + conf, status -> { + closure.run(status); + if (!status.isOk()) { + log.error("Raft {} add node {} error {}", options.getGroupId(), + peer, status); + } + }); })); closure.get(); } else { @@ -333,7 +370,7 @@ public class PartitionEngine implements Lifecycle, RaftS conf.removeLearner(peerId); }); result = Status.OK(); - }else { + } else { // 失败了重试 result = HgRaftError.TASK_ERROR.toStatus(); } @@ -345,9 +382,11 @@ public class PartitionEngine implements Lifecycle, RaftS // 同步完成，删除旧peer removedPeers.addAll(ListUtils.removeAll(oldPeers, peers)); // 检查leader是否被删除，如果是，先进行leader迁移 - if (removedPeers.contains(this.getRaftNode().getNodeId().getPeerId().getEndpoint().toString())) { + if (removedPeers.contains( + this.getRaftNode().getNodeId().getPeerId().getEndpoint().toString())) { - log.info("Raft {} leader is removed, needs to transfer leader {}, conf: {}", getGroupId(), peers, conf); + log.info("Raft {} leader is removed, needs to transfer leader {}, conf: {}", + getGroupId(), peers, conf); // only one (that's leader self), should add peer first if (raftNode.listPeers().size() == 1) { FutureClosure closure = new FutureClosure(); @@ -378,20 +417,26 @@ public class PartitionEngine implements Lifecycle, RaftS if (!removedPeers.isEmpty()) { removedPeers.forEach(peer -> Utils.runInThread(() -> { // 6. 停止已被删除的peer - rpcClient.destroyRaftNode(peer, partitionManager.getPartitionList(getGroupId()), - status -> { - if (!status.isOk()) { - // TODO 失败了怎么办？ - log.error("Raft {} destroy node {} error {}", options.getGroupId(), peer, status); - } - }); + rpcClient.destroyRaftNode(peer, + partitionManager.getPartitionList(getGroupId()), + status -> { + if (!status.isOk()) { + // TODO 失败了怎么办？ + log.error( + "Raft {} destroy node {} error " + + "{}", + options.getGroupId(), peer, + status); + } + }); })); } } else { // 失败了重试 result = HgRaftError.TASK_ERROR.toStatus(); } - log.info("Raft {} changePeers result {}, conf is {}", getRaftNode().getGroupId(), closure.get(), conf); + log.info("Raft {} changePeers result {}, conf is {}", getRaftNode().getGroupId(), + closure.get(), conf); } log.info("Raft {} changePeers end. {}, result is {}", getGroupId(), peers, result); return result; @@ -431,7 +476,7 @@ public class PartitionEngine implements Lifecycle, RaftS /** * 重启raft引擎 */ - public void restartRaftNode(){ + public void restartRaftNode() { shutdown(); log.error("Raft {} is restarting !!!", getGroupId()); this.init(this.options); @@ -440,12 +485,13 @@ public class PartitionEngine implements Lifecycle, RaftS /** * 检查是否活跃，如果不活跃，则重启 */ - public void checkActivity(){ + public void checkActivity() { Utils.runInThread(() -> { - if ( !this.raftNode.getNodeState().isActive()){ - log.error("Raft {} is not activity state is ", this.getGroupId(), raftNode.getNodeState()); - restartRaftNode(); - } + if (!this.raftNode.getNodeState().isActive()) { + log.error("Raft {} is not activity state is ", this.getGroupId(), + raftNode.getNodeState()); + restartRaftNode(); + } }); } @@ -456,7 +502,8 @@ public class PartitionEngine implements Lifecycle, RaftS shutdown(); try { FileUtils.deleteDirectory(new File(this.options.getRaftDataPath())); - if (! Objects.equals(this.options.getRaftDataPath(), this.options.getRaftSnapShotPath())) { + if (!Objects.equals(this.options.getRaftDataPath(), + this.options.getRaftSnapShotPath())) { FileUtils.deleteDirectory(new File(this.options.getRaftSnapShotPath())); } } catch (IOException e) { @@ -485,9 +532,11 @@ public class PartitionEngine implements Lifecycle, RaftS public Map getAlivePeers() { Map peers = new HashMap<>(); raftNode.listAlivePeers().forEach(peerId -> { - Shard shard = partitionManager.getShardByRaftEndpoint(shardGroup, peerId.getEndpoint().toString()); - if (shard != null) + Shard shard = partitionManager.getShardByRaftEndpoint(shardGroup, + peerId.getEndpoint().toString()); + if (shard != null) { peers.put(shard.getStoreId(), peerId); + } }); return peers; } @@ -504,7 +553,7 @@ public class PartitionEngine implements Lifecycle, RaftS */ public Endpoint waitForLeader(long timeOut) { Endpoint leader = getLeader(); - if ( leader != null ) { + if (leader != null) { return leader; } @@ -522,18 +571,20 @@ public class PartitionEngine implements Lifecycle, RaftS log.error("Raft {} wait for leader exception", this.options.getGroupId(), e); } leader = getLeader(); - if (leader == null){ - if ( partitionManager.isLocalPartition(this.options.getGroupId())) { - log.error("Raft {} leader not found, try to repair!", this.options.getGroupId()); + if (leader == null) { + if (partitionManager.isLocalPartition(this.options.getGroupId())) { + log.error("Raft {} leader not found, try to repair!", + this.options.getGroupId()); // TODO 判断raft是否本机，如果是，尝试修复Leader，包括检查配置是否正确 - storeEngine.createPartitionGroups(partitionManager.getPartitionList(getGroupId()).get(0)); + storeEngine.createPartitionGroups( + partitionManager.getPartitionList(getGroupId()).get(0)); } - }else{ + } else { log.info("Raft {} wait for leader success, latency time {}", - this.options.getGroupId(), System.currentTimeMillis() - start); + this.options.getGroupId(), System.currentTimeMillis() - start); } } - return leader != null ? leader : null; + return leader; } } @@ -544,19 +595,22 @@ public class PartitionEngine implements Lifecycle, RaftS public Partition getPartition(String graphName) { return partitionManager.getPartition(graphName, getGroupId()); } - public PartitionEngineOptions getOptions(){ + + public PartitionEngineOptions getOptions() { return options; } @Override public String toString() { - return "PartitionEngine{" + "groupId =" + this.options.getGroupId() + ", isLeader=" + isLeader() - + ", options=" + this.options + '}'; + return "PartitionEngine{" + "groupId =" + this.options.getGroupId() + ", isLeader=" + + isLeader() + + ", options=" + this.options + '}'; } /** * update partition leader + * * @param newTerm the new term */ @Override @@ -571,16 +625,18 @@ public class PartitionEngine implements Lifecycle, RaftS } } + @Override - public void onStartFollowing(final PeerId newLeaderId, final long newTerm){ + public void onStartFollowing(final PeerId newLeaderId, final long newTerm) { onConfigurationCommitted(this.getRaftNode().getCurrentConf()); synchronized (leaderChangedEvent) { leaderChangedEvent.notifyAll(); } - }; + } /** * update partition shardList + * * @param conf committed configuration */ @Override @@ -595,16 +651,18 @@ public class PartitionEngine implements Lifecycle, RaftS Store store = getStoreByEndpoint(peer); if (store != null) { peerIds.add(store.getId()); - } else + } else { log.error("Raft {} GetStoreInfo failure, {}", getGroupId(), peer); + } } List learners = new ArrayList<>(); for (String learner : RaftUtils.getLearnerEndpoints(conf)) { Store store = getStoreByEndpoint(learner); if (store != null) { learners.add(store.getId()); - } else + } else { log.error("Raft {} GetStoreInfo failure, {}", getGroupId(), learner); + } } shardGroup.changeShardList(peerIds, learners, partitionManager.getStore().getId()); @@ -618,7 +676,7 @@ public class PartitionEngine implements Lifecycle, RaftS var pdGroup = storeEngine.getPdProvider().getShardGroup(getGroupId()); List peers = partitionManager.shards2Peers(pdGroup.getShardsList()); - if (! ListUtils.isEqualList(peers, RaftUtils.getPeerEndpoints(raftNode))) { + if (!ListUtils.isEqualList(peers, RaftUtils.getPeerEndpoints(raftNode))) { partitionManager.getPdProvider().updateShardGroup(shardGroup.getProtoObj()); } @@ -626,7 +684,8 @@ public class PartitionEngine implements Lifecycle, RaftS throw new RuntimeException(e); } } - log.info("Raft {} onConfigurationCommitted, shardGroup {}", getGroupId(), shardGroup.getMetaPbShard()); + log.info("Raft {} onConfigurationCommitted, shardGroup {}", getGroupId(), + shardGroup.getMetaPbShard()); } catch (Exception e) { log.error("Raft {} onConfigurationCommitted exception {}", getGroupId(), e); @@ -643,7 +702,7 @@ public class PartitionEngine implements Lifecycle, RaftS } @Override - public void onDataCommitted(long index){ + public void onDataCommitted(long index) { } @@ -656,26 +715,26 @@ public class PartitionEngine implements Lifecycle, RaftS return this.raftNode.getNodeMetrics(); } - public long getCommittedIndex(){ + public long getCommittedIndex() { return stateMachine.getCommittedIndex(); } public long getLeaderTerm() { - return stateMachine.getLeaderTerm(); + return stateMachine.getLeaderTerm(); } - public TaskManager getTaskManager(){ + public TaskManager getTaskManager() { return this.taskManager; } /** * 收到PD发送的leader转移指令 - + * * @param graphName * @param shard * @return */ - public Status transferLeader(String graphName, Metapb.Shard shard){ + public Status transferLeader(String graphName, Metapb.Shard shard) { if (!isLeader()) { return new Status(HgRaftError.NOT_LEADER.getNumber(), "Not leader"); } @@ -684,7 +743,6 @@ public class PartitionEngine implements Lifecycle, RaftS } - /** * 接收到pd发来的修改副本指令 * 1. 比较新旧peer，查找出新增和删除的peer @@ -700,7 +758,8 @@ public class PartitionEngine implements Lifecycle, RaftS log.info("Raft {} doChangeShard task is {}", getGroupId(), task); // 如果相同的分区有相同的任务在执行，忽略任务执行 if (taskManager.partitionTaskRepeat(task.getPartition().getId(), - task.getPartition().getGraphName(), task.getType().name())) { + task.getPartition().getGraphName(), + task.getType().name())) { log.error("Raft {} doChangeShard task repeat, type:{}", getGroupId(), task.getType()); return; } @@ -709,11 +768,13 @@ public class PartitionEngine implements Lifecycle, RaftS Utils.runInThread(() -> { try { // 不能在状态机中changePeers - List peers = partitionManager.shards2Peers(task.getChangeShard().getShardList()); + List peers = + partitionManager.shards2Peers(task.getChangeShard().getShardList()); HashSet hashSet = new HashSet<>(peers); // 任务中有相同的peers，说明任务本身有错误，任务忽略 if (peers.size() != hashSet.size()) { - log.info("Raft {} doChangeShard peer is repeat， peers：{}", getGroupId(), peers); + log.info("Raft {} doChangeShard peer is repeat， peers：{}", getGroupId(), + peers); } Status result; if (changingPeer.compareAndSet(false, true)) { @@ -723,39 +784,50 @@ public class PartitionEngine implements Lifecycle, RaftS } if (result.getCode() != HgRaftError.TASK_CONTINUE.getNumber()) { - log.info("Raft {} doChangeShard is finished, status is {}", getGroupId(), result); + log.info("Raft {} doChangeShard is finished, status is {}", getGroupId(), + result); // 任务完成，同步任务状态 MetaTask.Task newTask; if (result.isOk()) { - newTask = task.toBuilder().setState(MetaTask.TaskState.Task_Success).build(); + newTask = task.toBuilder().setState(MetaTask.TaskState.Task_Success) + .build(); } else { - log.warn("Raft {} doChangeShard is failure, need to retry, status is {}", + log.warn( + "Raft {} doChangeShard is failure, need to retry, status is {}", getGroupId(), result); - try{ + try { // 减少发送次数 Thread.sleep(1000); - }catch (Exception e){ - log.error("wait 1s to resend retry task. got error:{}", e.getMessage()); + } catch (Exception e) { + log.error("wait 1s to resend retry task. got error:{}", + e.getMessage()); } - newTask = task.toBuilder().setState(MetaTask.TaskState.Task_Ready).build(); + newTask = task.toBuilder().setState(MetaTask.TaskState.Task_Ready) + .build(); } try { // 等待过程中，可能已经shut down了 if (isLeader()) { storeEngine.addRaftTask(newTask.getPartition().getGraphName(), - newTask.getPartition().getId(), - RaftOperation.create(RaftOperation.SYNC_PARTITION_TASK, newTask), - status -> { - if (!status.isOk()) { - log.error("Raft {} addRaftTask error, status is {}", - newTask.getPartition().getId(), status); - } - } + newTask.getPartition().getId(), + RaftOperation.create( + RaftOperation.SYNC_PARTITION_TASK, + newTask), + status -> { + if (!status.isOk()) { + log.error( + "Raft {} addRaftTask " + + "error, status is {}", + newTask.getPartition() + .getId(), status); + } + } ); } } catch (Exception e) { log.error("Partition {}-{} update task state exception {}", - task.getPartition().getGraphName(), task.getPartition().getId(), e); + task.getPartition().getGraphName(), + task.getPartition().getId(), e); } // db 可能被销毁了，就不要更新了 if (this.started) { @@ -782,20 +854,22 @@ public class PartitionEngine implements Lifecycle, RaftS * 2. 从源机器拷贝数据到目标机器 * 3. 迁移成功后，通知PD修改分区信息 * 4. 删除源分区 + * * @return */ public Status moveData(MetaTask.Task task) { synchronized (this) { if (taskManager.taskExists(task.getPartition().getId(), - task.getPartition().getGraphName(), - task.getType().name())) { + task.getPartition().getGraphName(), + task.getType().name())) { log.info("task : {}-{}-{} repeat", task.getPartition().getGraphName(), - task.getPartition().getId(), - task.getType().name()); + task.getPartition().getId(), + task.getType().name()); return Status.OK(); } - taskManager.updateTask(task.toBuilder().setState(MetaTask.TaskState.Task_Doing).build()); + taskManager.updateTask( + task.toBuilder().setState(MetaTask.TaskState.Task_Doing).build()); } Status status = Status.OK(); @@ -810,22 +884,25 @@ public class PartitionEngine implements Lifecycle, RaftS default: break; } - log.info("moveData {}-{}-{} result:{}", task.getPartition().getGraphName(), task.getPartition().getId(), - task.getType().name(), status); + log.info("moveData {}-{}-{} result:{}", task.getPartition().getGraphName(), + task.getPartition().getId(), + task.getType().name(), status); if (status.isOk()) { // 向PD汇报任务执行结果 - partitionManager.reportTask(task.toBuilder().setState(MetaTask.TaskState.Task_Success).build()); + partitionManager.reportTask( + task.toBuilder().setState(MetaTask.TaskState.Task_Success).build()); // 更新本地任务状态 - taskManager.updateTask(task.toBuilder().setState(MetaTask.TaskState.Task_Success).build()); + taskManager.updateTask( + task.toBuilder().setState(MetaTask.TaskState.Task_Success).build()); } else { partitionManager.reportTask(task.toBuilder() - .setState(MetaTask.TaskState.Task_Failure) - .setMessage(status.getErrorMsg()).build()); + .setState(MetaTask.TaskState.Task_Failure) + .setMessage(status.getErrorMsg()).build()); // 更新本地任务状态 taskManager.updateTask(task.toBuilder() - .setState(MetaTask.TaskState.Task_Failure) - .setMessage(status.getErrorMsg()).build()); + .setState(MetaTask.TaskState.Task_Failure) + .setMessage(status.getErrorMsg()).build()); } return status; @@ -837,11 +914,12 @@ public class PartitionEngine implements Lifecycle, RaftS * @param task split partition task * @return task execution result */ - private Status handleSplitTask(MetaTask.Task task){ + private Status handleSplitTask(MetaTask.Task task) { - log.info("Partition {}-{} moveData {}", task.getPartition().getGraphName(), task.getPartition().getId(), task); + log.info("Partition {}-{} moveData {}", task.getPartition().getGraphName(), + task.getPartition().getId(), task); - Status status ; + Status status; List targets = task.getSplitPartition().getNewPartitionList(); List newPartitions = targets.subList(1, targets.size()); @@ -854,12 +932,13 @@ public class PartitionEngine implements Lifecycle, RaftS if (status.isOk()) { var source = Metapb.Partition.newBuilder(targets.get(0)) - .setState(Metapb.PartitionState.PState_Normal) - .build(); + .setState(Metapb.PartitionState.PState_Normal) + .build(); // 更新本地key range，并同步follower partitionManager.updatePartition(source, true); storeEngine.getDataMover().updatePartitionRange(source, - (int) source.getStartKey(), (int) source.getEndKey()); + (int) source.getStartKey(), + (int) source.getEndKey()); } if (!status.isOk()) { @@ -867,7 +946,7 @@ public class PartitionEngine implements Lifecycle, RaftS } } catch (Exception e) { log.error("Partition {}-{} moveData exception {}", - task.getPartition().getGraphName(), task.getPartition().getId(), e); + task.getPartition().getGraphName(), task.getPartition().getId(), e); status = new Status(-1, e.getMessage()); } @@ -880,14 +959,17 @@ public class PartitionEngine implements Lifecycle, RaftS * @param task move partition task * @return task execution result */ - private Status handleMoveTask(MetaTask.Task task){ + private Status handleMoveTask(MetaTask.Task task) { Status status; try { - log.info("handleMoveTask: start to copy {} data from {} to {}", task.getPartition().getGraphName(), - task.getPartition().getId(), task.getMovePartition().getTargetPartition().getId()); + log.info("handleMoveTask: start to copy {} data from {} to {}", + task.getPartition().getGraphName(), + task.getPartition().getId(), + task.getMovePartition().getTargetPartition().getId()); status = storeEngine.getDataMover().moveData(task.getPartition(), - task.getMovePartition().getTargetPartition()); - } catch (Exception e){ + task.getMovePartition() + .getTargetPartition()); + } catch (Exception e) { log.error("handleMoveTask got exception: {}", e); status = new Status(-1, e.getMessage()); } @@ -898,30 +980,34 @@ public class PartitionEngine implements Lifecycle, RaftS * 对于整个图删除的清空，删除分区， 如果没有其他的图，销毁raft group. * 需要放到调用move data 之后 * - * @param graphName graph name + * @param graphName graph name * @param partitionId partition id - * @param keyStart key start 用于验证 - * @param keyEnd key end 用于验证 - * @param isLeader 是否leader，避免leader漂移，采取move data时候的leader状态 + * @param keyStart key start 用于验证 + * @param keyEnd key end 用于验证 + * @param isLeader 是否leader，避免leader漂移，采取move data时候的leader状态 */ private synchronized void destroyPartitionIfGraphsNull(String graphName, int partitionId, - long keyStart, long keyEnd, boolean isLeader) { + long keyStart, long keyEnd, + boolean isLeader) { Partition partition = partitionManager.getPartition(graphName, partitionId); // key range 校验 - if (partition != null && partition.getEndKey() == keyEnd && partition.getStartKey() == keyStart) { - log.info("remove partition id :{}, graph:{}", partition.getId(), partition.getGraphName()); + if (partition != null && partition.getEndKey() == keyEnd && + partition.getStartKey() == keyStart) { + log.info("remove partition id :{}, graph:{}", partition.getId(), + partition.getGraphName()); storeEngine.deletePartition(partitionId, graphName); } // 没有partition engine的情况 - if (isLeader && partition == null){ + if (isLeader && partition == null) { partitionManager.deletePartition(graphName, partitionId); } if (isLeader && partition != null) { if (partitionManager.getPartitions(partitionId).size() == 0) { - log.info("destroyPartitionIfGraphsNull, destroy raft group id:{}", partition.getId()); + log.info("destroyPartitionIfGraphsNull, destroy raft group id:{}", + partition.getId()); storeEngine.destroyPartitionGroups(partition); try { // delete shard group from pd @@ -976,18 +1062,21 @@ public class PartitionEngine implements Lifecycle, RaftS private void handleCleanOp(CleanDataRequest request) { // 避免清理数据过程中的leader漂移 boolean isLeader = isLeader(); - var partition = partitionManager.getPartition(request.getGraphName(), request.getPartitionId()); + var partition = + partitionManager.getPartition(request.getGraphName(), request.getPartitionId()); if (partition != null) { storeEngine.getDataMover().doCleanData(request); - storeEngine.getBusinessHandler().dbCompaction(partition.getGraphName(), partition.getId()); + storeEngine.getBusinessHandler() + .dbCompaction(partition.getGraphName(), partition.getId()); if (request.isDeletePartition()) { destroyPartitionIfGraphsNull(request.getGraphName(), request.getPartitionId(), - request.getKeyStart(), request.getKeyEnd(), isLeader); + request.getKeyStart(), request.getKeyEnd(), isLeader); } } else { - log.info("handleCleanOp, partition is null. {}-{}", request.getGraphName(), request.getPartitionId()); + log.info("handleCleanOp, partition is null. {}-{}", request.getGraphName(), + request.getPartitionId()); partition = new Partition() {{ setId(request.getPartitionId()); setGraphName(request.getGraphName()); @@ -1001,18 +1090,20 @@ public class PartitionEngine implements Lifecycle, RaftS // report task if (isLeader) { MetaTask.Task task = MetaTask.Task.newBuilder() - .setType(Clean_Partition) - .setId(request.getTaskId()) - .setPartition(partition.getProtoObj()) - .setCleanPartition(CleanDataRequest.toCleanPartitionTask(request)) - .setState(MetaTask.TaskState.Task_Success) - .build(); + .setType(Clean_Partition) + .setId(request.getTaskId()) + .setPartition(partition.getProtoObj()) + .setCleanPartition( + CleanDataRequest.toCleanPartitionTask( + request)) + .setState(MetaTask.TaskState.Task_Success) + .build(); partitionManager.reportTask(task); } } - class ReplicatorStateListener implements Replicator.ReplicatorStateListener{ + class ReplicatorStateListener implements Replicator.ReplicatorStateListener { @Override public void onCreated(PeerId peer) { @@ -1021,7 +1112,7 @@ public class PartitionEngine implements Lifecycle, RaftS @Override public void onError(PeerId peer, Status status) { - // log.info("Raft {} Replicator onError {} {}", getGroupId(), peer, status); + // log.info("Raft {} Replicator onError {} {}", getGroupId(), peer, status); } @Override @@ -1039,18 +1130,22 @@ public class PartitionEngine implements Lifecycle, RaftS if (newState == ReplicatorState.ONLINE) { MetaTask.Task task = taskManager .getOneTask(getGroupId(), MetaTask.TaskType.Change_Shard.name()); - if (task != null) + if (task != null) { doChangeShard(task, status -> { - log.info("Raft {} (replicator state changed) changeShard task {}, status is {}", + log.info( + "Raft {} (replicator state changed) changeShard task {}, status " + + "is {}", getGroupId(), task, status); }); + } } } } class TaskHandler implements RaftTaskHandler { @Override - public boolean invoke(final int groupId, byte[] request, RaftClosure response) throws HgStoreException { + public boolean invoke(final int groupId, byte[] request, RaftClosure response) throws + HgStoreException { try { CodedInputStream input = CodedInputStream.newInstance(request); byte methodId = input.readRawByte(); @@ -1081,7 +1176,8 @@ public class PartitionEngine implements Lifecycle, RaftS } @Override - public boolean invoke(final int groupId, byte methodId, Object req, RaftClosure response) throws HgStoreException { + public boolean invoke(final int groupId, byte methodId, Object req, + RaftClosure response) throws HgStoreException { switch (methodId) { case RaftOperation.SYNC_PARTITION_TASK: { @@ -1089,7 +1185,8 @@ public class PartitionEngine implements Lifecycle, RaftS taskManager.updateTask(task); switch (task.getType()) { case Change_Shard: - log.info("change shard task: id {}, change shard:{} ", task.getId(), task.getChangeShard()); + log.info("change shard task: id {}, change shard:{} ", task.getId(), + task.getChangeShard()); doChangeShard(task, response); break; default: @@ -1109,20 +1206,22 @@ public class PartitionEngine implements Lifecycle, RaftS doSnapshot(response); break; case RaftOperation.IN_WRITE_OP: - storeEngine.getDataMover().doWriteData((BatchPutRequest)(req)); + storeEngine.getDataMover().doWriteData((BatchPutRequest) (req)); break; case RaftOperation.IN_CLEAN_OP: handleCleanOp((CleanDataRequest) req); break; case RaftOperation.RAFT_UPDATE_PARTITION: log.info("Raft {}, receive raft updatePartitionRangeOrState {}", - getGroupId(), req); + getGroupId(), req); partitionManager.updatePartitionRangeOrState((UpdatePartitionRequest) (req)); break; case RaftOperation.DB_COMPACTION: DbCompactionRequest dbCompactionRequest = (DbCompactionRequest) (req); - storeEngine.getBusinessHandler().dbCompaction(dbCompactionRequest.getGraphName(), - dbCompactionRequest.getPartitionId(), dbCompactionRequest.getTableName()); + storeEngine.getBusinessHandler() + .dbCompaction(dbCompactionRequest.getGraphName(), + dbCompactionRequest.getPartitionId(), + dbCompactionRequest.getTableName()); break; default: return false; diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/PartitionInstructionProcessor.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/PartitionInstructionProcessor.java new file mode 100644 index 000000000..715af2d75 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/PartitionInstructionProcessor.java @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import org.apache.hugegraph.store.cmd.CleanDataRequest; +import org.apache.hugegraph.store.cmd.DbCompactionRequest; +import org.apache.hugegraph.store.meta.MetadataKeyHelper; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.pd.PartitionInstructionListener; +import org.apache.hugegraph.store.raft.RaftClosure; +import org.apache.hugegraph.store.raft.RaftOperation; +import org.slf4j.Logger; + +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.util.Utils; +import com.baidu.hugegraph.pd.common.PDException; +import com.baidu.hugegraph.pd.grpc.MetaTask; +import com.baidu.hugegraph.pd.grpc.Metapb; +import com.baidu.hugegraph.pd.grpc.pulse.ChangeShard; +import com.baidu.hugegraph.pd.grpc.pulse.CleanPartition; +import com.baidu.hugegraph.pd.grpc.pulse.DbCompaction; +import com.baidu.hugegraph.pd.grpc.pulse.MovePartition; +import com.baidu.hugegraph.pd.grpc.pulse.PartitionKeyRange; +import com.baidu.hugegraph.pd.grpc.pulse.SplitPartition; +import com.baidu.hugegraph.pd.grpc.pulse.TransferLeader; +import com.baidu.hugegraph.util.Log; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * PD发给Store的分区指令处理器 + */ +public class PartitionInstructionProcessor implements PartitionInstructionListener { + private static final Logger LOG = Log.logger(PartitionInstructionProcessor.class); + private final HgStoreEngine storeEngine; + private final ExecutorService threadPool; + + public PartitionInstructionProcessor(HgStoreEngine storeEngine) { + this.storeEngine = storeEngine; + ThreadFactory namedThreadFactory = + new ThreadFactoryBuilder().setNameFormat("instruct-process-pool-%d").build(); + threadPool = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors(), + 1000000, + 180L, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(1000000), + namedThreadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + @Override + public void onChangeShard(long taskId, Partition partition, ChangeShard changeShard, + Consumer consumer) { + PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId()); + + if (engine != null) { + // 清理所有的任务，有失败的情况 + engine.getTaskManager() + .deleteTask(partition.getId(), MetaTask.TaskType.Change_Shard.name()); + } + + if (engine != null && engine.isLeader()) { + LOG.info("Partition {}-{} Receive change shard message, {}", partition.getGraphName(), + partition.getId(), changeShard); + String graphName = partition.getGraphName(); + int partitionId = partition.getId(); + MetaTask.Task task = MetaTask.Task.newBuilder() + .setId(taskId) + .setPartition(partition.getProtoObj()) + .setType(MetaTask.TaskType.Change_Shard) + .setState(MetaTask.TaskState.Task_Ready) + .setChangeShard(changeShard) + .build(); + try { + storeEngine.addRaftTask(graphName, partitionId, + RaftOperation.create(RaftOperation.SYNC_PARTITION_TASK, + task), + new RaftClosure() { + @Override + public void run(Status status) { + LOG.info( + "Partition {}-{} onChangeShard complete, " + + "status is {}", + graphName, partitionId, status); + consumer.accept(0); + } + }); + } catch (Exception e) { + LOG.error("Partition {}-{} onSplitPartition exception {}", + graphName, partitionId, e); + } + } + } + + @Override + public void onTransferLeader(long taskId, Partition partition, TransferLeader transferLeader, + Consumer consumer) { + PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId()); + if (engine != null && engine.isLeader()) { + consumer.accept(0); + Utils.runInThread(() -> { + LOG.info("Partition {}-{} receive TransferLeader instruction, new leader is {}" + , partition.getGraphName(), partition.getId(), transferLeader.getShard()); + engine.transferLeader(partition.getGraphName(), transferLeader.getShard()); + }); + } + } + + /** + * Leader接收到PD发送的分区分裂任务 + * 添加到raft任务队列，由raft进行任务分发。 + */ + @Override + public void onSplitPartition(long taskId, Partition partition, SplitPartition splitPartition, + Consumer consumer) { + PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId()); + + if (preCheckTaskId(taskId, partition.getId())) { + return; + } + + if (engine != null && engine.isLeader()) { + // 先应答，避免超时造成pd重复发送 + consumer.accept(0); + + String graphName = partition.getGraphName(); + int partitionId = partition.getId(); + MetaTask.Task task = MetaTask.Task.newBuilder() + .setId(taskId) + .setPartition(partition.getProtoObj()) + .setType(MetaTask.TaskType.Split_Partition) + .setState(MetaTask.TaskState.Task_Ready) + .setSplitPartition(splitPartition) + .build(); + try { + threadPool.submit(() -> { + engine.moveData(task); + }); + } catch (Exception e) { + LOG.error("Partition {}-{} onSplitPartition exception {}", + graphName, partitionId, e); + } + } + } + + /** + * Leader接收到PD发送的rocksdb compaction任务 + * 添加到raft任务队列，由raft进行任务分发。 + */ + @Override + public void onDbCompaction(long taskId, Partition partition, DbCompaction dbCompaction, + Consumer consumer) { + PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId()); + if (engine != null && engine.isLeader()) { + try { + DbCompactionRequest dbCompactionRequest = new DbCompactionRequest(); + dbCompactionRequest.setPartitionId(partition.getId()); + dbCompactionRequest.setTableName(dbCompaction.getTableName()); + dbCompactionRequest.setGraphName(partition.getGraphName()); + engine.addRaftTask(RaftOperation.create(RaftOperation.DB_COMPACTION, + dbCompactionRequest), + new RaftClosure() { + @Override + public void run(Status status) { + LOG.info( + "onRocksdbCompaction {}-{} sync partition " + + "status is {}", + partition.getGraphName(), partition.getId(), + status); + } + } + ); + } finally { + consumer.accept(0); + } + } + } + + @Override + public void onMovePartition(long taskId, Partition partition, MovePartition movePartition, + Consumer consumer) { + PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId()); + + if (preCheckTaskId(taskId, partition.getId())) { + return; + } + + if (engine != null && engine.isLeader()) { + // 先应答，避免超时造成pd重复发送 + consumer.accept(0); + + String graphName = partition.getGraphName(); + int partitionId = partition.getId(); + MetaTask.Task task = MetaTask.Task.newBuilder() + .setId(taskId) + .setPartition(partition.getProtoObj()) + .setType(MetaTask.TaskType.Move_Partition) + .setState(MetaTask.TaskState.Task_Ready) + .setMovePartition(movePartition) + .build(); + try { + threadPool.submit(() -> { + engine.moveData(task); + }); + } catch (Exception e) { + LOG.error("Partition {}-{} onMovePartition exception {}", + graphName, partitionId, e); + } + } + } + + @Override + public void onCleanPartition(long taskId, Partition partition, CleanPartition cleanPartition, + Consumer consumer) { + + if (preCheckTaskId(taskId, partition.getId())) { + return; + } + + PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId()); + if (engine != null && engine.isLeader()) { + consumer.accept(0); + + CleanDataRequest + request = + CleanDataRequest.fromCleanPartitionTask(cleanPartition, partition, taskId); + + storeEngine.addRaftTask(partition.getGraphName(), partition.getId(), + RaftOperation.create(RaftOperation.IN_CLEAN_OP, request), + status -> { + LOG.info( + "onCleanPartition {}-{}, cleanType: {}, " + + "range:{}-{}, status:{}", + partition.getGraphName(), + partition.getId(), + cleanPartition.getCleanType(), + cleanPartition.getKeyStart(), + cleanPartition.getKeyEnd(), + status); + }); + } + + } + + @Override + public void onPartitionKeyRangeChanged(long taskId, Partition partition, + PartitionKeyRange partitionKeyRange, + Consumer consumer) { + PartitionEngine engine = storeEngine.getPartitionEngine(partition.getId()); + if (engine != null && engine.isLeader()) { + consumer.accept(0); + var partitionManager = storeEngine.getPartitionManager(); + var localPartition = + partitionManager.getPartition(partition.getGraphName(), partition.getId()); + + if (localPartition == null) { + // 如果分区数据为空，本地不会存储 + localPartition = partitionManager.getPartitionFromPD(partition.getGraphName(), + partition.getId()); + LOG.info("onPartitionKeyRangeChanged, get from pd:{}-{} -> {}", + partition.getGraphName(), partition.getId(), localPartition); + if (localPartition == null) { + return; + } + } + + var newPartition = localPartition.getProtoObj().toBuilder() + .setStartKey(partitionKeyRange.getKeyStart()) + .setEndKey(partitionKeyRange.getKeyEnd()) + .setState(Metapb.PartitionState.PState_Normal) + .build(); + partitionManager.updatePartition(newPartition, true); + + try { + engine.addRaftTask(RaftOperation.create(RaftOperation.SYNC_PARTITION, newPartition), + status -> { + LOG.info( + "onPartitionKeyRangeChanged, {}-{},key range: " + + "{}-{} status{}", + newPartition.getGraphName(), + newPartition.getId(), + partitionKeyRange.getKeyStart(), + partitionKeyRange.getKeyEnd(), + status); + }); + LOG.info("onPartitionKeyRangeChanged: {}, update to pd", newPartition); + partitionManager.updatePartitionToPD(List.of(newPartition)); + } catch (IOException e) { + LOG.error("Partition {}-{} onPartitionKeyRangeChanged exception {}", + newPartition.getGraphName(), newPartition.getId(), e); + } catch (PDException e) { + throw new RuntimeException(e); + } + } + } + + /** + * is the task exists + * + * @param taskId task id + * @param partId partition id + * @return true if exists, false otherwise + */ + private boolean preCheckTaskId(long taskId, int partId) { + + if (storeEngine.getPartitionEngine(partId) == null) { + return false; + } + + byte[] key = MetadataKeyHelper.getInstructionIdKey(taskId); + var wrapper = storeEngine.getPartitionManager().getWrapper(); + byte[] value = wrapper.get(partId, key); + + if (value != null) { + return true; + } + + wrapper.put(partId, key, new byte[0]); + return false; + } +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/PartitionStateListener.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/PartitionStateListener.java new file mode 100644 index 000000000..5a7fa98a5 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/PartitionStateListener.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import java.util.List; + +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.meta.PartitionRole; + +import com.baidu.hugegraph.pd.grpc.Metapb; + +public interface PartitionStateListener { + // 分区角色发生改变 + void partitionRoleChanged(Partition partition, PartitionRole newRole); + + // 分区发生改变 + void partitionShardChanged(Partition partition, List oldShards, + List newShards); +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/AbstractSelectIterator.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/AbstractSelectIterator.java similarity index 67% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/business/AbstractSelectIterator.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/business/AbstractSelectIterator.java index 4f15af3f5..5a5c92a4a 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/AbstractSelectIterator.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/AbstractSelectIterator.java @@ -1,12 +1,29 @@ -package com.baidu.hugegraph.store.business; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hugegraph.store.business; + +import org.apache.hugegraph.rocksdb.access.ScanIterator; import org.apache.tinkerpop.gremlin.structure.Edge; import com.baidu.hugegraph.backend.serializer.AbstractSerializer; import com.baidu.hugegraph.backend.serializer.BinarySerializer; import com.baidu.hugegraph.backend.store.BackendEntry; import com.baidu.hugegraph.iterator.CIter; -import org.apache.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.structure.HugeElement; import com.baidu.hugegraph.util.Bytes; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandler.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandler.java similarity index 73% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandler.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandler.java index 68dbe1603..8be42deeb 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandler.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandler.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.business; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.business; import java.util.List; import java.util.Map; @@ -7,22 +24,22 @@ import java.util.function.Supplier; import javax.annotation.concurrent.NotThreadSafe; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.meta.base.DBSessionBuilder; +import org.apache.hugegraph.store.metric.HgStoreMetric; +import org.apache.hugegraph.store.raft.HgStoreStateMachine; +import org.apache.hugegraph.store.term.HgPair; +import org.apache.hugegraph.store.util.HgStoreException; import org.rocksdb.Cache; import org.rocksdb.MemoryUsageType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.baidu.hugegraph.pd.grpc.pulse.CleanType; -import org.apache.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.store.grpc.Graphpb; import com.baidu.hugegraph.store.grpc.common.Key; import com.baidu.hugegraph.store.grpc.common.OpType; import com.baidu.hugegraph.store.grpc.session.BatchEntry; -import com.baidu.hugegraph.store.meta.base.DBSessionBuilder; -import com.baidu.hugegraph.store.metric.HgStoreMetric; -import com.baidu.hugegraph.store.raft.HgStoreStateMachine; -import com.baidu.hugegraph.store.term.HgPair; -import com.baidu.hugegraph.store.util.HgStoreException; /** * @projectName: hugegraph-store @@ -49,7 +66,8 @@ public interface BusinessHandler extends DBSessionBuilder { tableUnknown, tableVertex, tableOutEdge, tableInEdge, tableIndex, tableTask, tableOlap }; - void doPut(String graph, int code, String table, byte[] key, byte[] value) throws HgStoreException; + void doPut(String graph, int code, String table, byte[] key, byte[] value) throws + HgStoreException; byte[] doGet(String graph, int code, String table, byte[] key) throws HgStoreException; @@ -69,11 +87,14 @@ public interface BusinessHandler extends DBSessionBuilder { ScanIterator scanOriginal(Graphpb.ScanPartitionRequest request); - ScanIterator scanPrefix(String graph, int code, String table, byte[] prefix, int scanType) throws HgStoreException; + ScanIterator scanPrefix(String graph, int code, String table, byte[] prefix, + int scanType) throws HgStoreException; - ScanIterator scanPrefix(String graph, int code, String table, byte[] prefix) throws HgStoreException; + ScanIterator scanPrefix(String graph, int code, String table, byte[] prefix) throws + HgStoreException; - HgStoreMetric.Partition getPartitionMetric(String graph, int partId, boolean accurateCount) throws HgStoreException; + HgStoreMetric.Partition getPartitionMetric(String graph, int partId, + boolean accurateCount) throws HgStoreException; void batchGet(String graph, String table, Supplier> s, Consumer> c) throws HgStoreException; @@ -93,7 +114,8 @@ public interface BusinessHandler extends DBSessionBuilder { void saveSnapshot(String snapshotPath, String graph, int partId) throws HgStoreException; - void loadSnapshot(String snapshotPath, String graph, int partId, long version) throws HgStoreException; + void loadSnapshot(String snapshotPath, String graph, int partId, long version) throws + HgStoreException; long getLatestSequenceNumber(String graph, int partId); @@ -101,7 +123,8 @@ public interface BusinessHandler extends DBSessionBuilder { // 扫描分区从seqnum开始的kv ScanIterator scanRaw(String graph, int partId, long seqNum) throws HgStoreException; - void ingestSstFile(String graph, int partId, Map> sstFiles) throws HgStoreException; + void ingestSstFile(String graph, int partId, Map> sstFiles) throws + HgStoreException; //提交分区分裂，删除旧数据 // 删除分区数据 @@ -110,7 +133,8 @@ public interface BusinessHandler extends DBSessionBuilder { //清理分区，删除多余的数据 boolean cleanPartition(String graph, int partId); - boolean cleanPartition(String graph, int partId, long startKey, long endKey, CleanType cleanType); + boolean cleanPartition(String graph, int partId, long startKey, long endKey, + CleanType cleanType); //所有指定分区图的所有table名 List getTableNames(String graph, int partId); @@ -127,7 +151,8 @@ public interface BusinessHandler extends DBSessionBuilder { TxBuilder delPrefix(int code, String table, byte[] prefix) throws HgStoreException; - TxBuilder delRange(int code, String table, byte[] start, byte[] end) throws HgStoreException; + TxBuilder delRange(int code, String table, byte[] start, byte[] end) throws + HgStoreException; TxBuilder merge(int code, String table, byte[] key, byte[] value) throws HgStoreException; @@ -159,22 +184,26 @@ public interface BusinessHandler extends DBSessionBuilder { builder.delPrefix(start.getCode(), table, startKey); continue; case OpType.OP_TYPE_DEL_RANGE_VALUE: - builder.delRange(start.getCode(), table, startKey, b.getEndKey().getKey().toByteArray()); + builder.delRange(start.getCode(), table, startKey, + b.getEndKey().getKey().toByteArray()); continue; case OpType.OP_TYPE_DEL_SINGLE_VALUE: builder.delSingle(start.getCode(), table, startKey); continue; case OpType.OP_TYPE_MERGE_VALUE: - builder.merge(start.getCode(), table, startKey, b.getValue().toByteArray()); + builder.merge(start.getCode(), table, startKey, + b.getValue().toByteArray()); continue; default: - throw new IllegalArgumentException("unsupported batch-op-type: " + b.getOpType().name()); + throw new IllegalArgumentException( + "unsupported batch-op-type: " + b.getOpType().name()); } } } builder.build().commit(); } catch (Throwable e) { - String msg = String.format("graph data %s-%s do batch insert with error:", graph, partId); + String msg = + String.format("graph data %s-%s do batch insert with error:", graph, partId); log.error(msg, e); builder.build().rollback(); throw e; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandlerImpl.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandlerImpl.java similarity index 85% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandlerImpl.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandlerImpl.java index 767f29e13..a108fe9ba 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandlerImpl.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandlerImpl.java @@ -1,8 +1,25 @@ -package com.baidu.hugegraph.store.business; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.business; -import static com.baidu.hugegraph.store.util.HgStoreConst.EMPTY_BYTES; -import static com.baidu.hugegraph.store.util.HgStoreConst.SCAN_ALL_PARTITIONS_ID; +import static org.apache.hugegraph.store.util.HgStoreConst.EMPTY_BYTES; +import static org.apache.hugegraph.store.util.HgStoreConst.SCAN_ALL_PARTITIONS_ID; import java.io.IOException; import java.util.ArrayList; @@ -20,13 +37,6 @@ import javax.annotation.concurrent.NotThreadSafe; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; -import org.rocksdb.Cache; -import org.rocksdb.MemoryUsageType; - -import com.alipay.sofa.jraft.util.Utils; -import com.baidu.hugegraph.config.HugeConfig; -import com.baidu.hugegraph.config.OptionSpace; -import com.baidu.hugegraph.pd.grpc.pulse.CleanType; import org.apache.hugegraph.rocksdb.access.DBStoreException; import org.apache.hugegraph.rocksdb.access.RocksDBFactory; import org.apache.hugegraph.rocksdb.access.RocksDBFactory.RocksdbChangedListener; @@ -34,20 +44,27 @@ import org.apache.hugegraph.rocksdb.access.RocksDBOptions; import org.apache.hugegraph.rocksdb.access.RocksDBSession; import org.apache.hugegraph.rocksdb.access.ScanIterator; import org.apache.hugegraph.rocksdb.access.SessionOperator; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.cmd.CleanDataRequest; +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.cmd.CleanDataRequest; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.meta.PartitionManager; +import org.apache.hugegraph.store.meta.asynctask.AsyncTaskState; +import org.apache.hugegraph.store.meta.asynctask.CleanTask; +import org.apache.hugegraph.store.metric.HgStoreMetric; +import org.apache.hugegraph.store.pd.PdProvider; +import org.apache.hugegraph.store.term.Bits; +import org.apache.hugegraph.store.term.HgPair; +import org.apache.hugegraph.store.util.HgStoreException; +import org.rocksdb.Cache; +import org.rocksdb.MemoryUsageType; + +import com.alipay.sofa.jraft.util.Utils; +import com.baidu.hugegraph.config.HugeConfig; +import com.baidu.hugegraph.config.OptionSpace; +import com.baidu.hugegraph.pd.grpc.pulse.CleanType; import com.baidu.hugegraph.store.grpc.Graphpb.ScanPartitionRequest; import com.baidu.hugegraph.store.grpc.Graphpb.ScanPartitionRequest.Request; import com.baidu.hugegraph.store.grpc.Graphpb.ScanPartitionRequest.ScanType; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.meta.PartitionManager; -import com.baidu.hugegraph.store.meta.asynctask.AsyncTaskState; -import com.baidu.hugegraph.store.meta.asynctask.CleanTask; -import com.baidu.hugegraph.store.metric.HgStoreMetric; -import com.baidu.hugegraph.store.pd.PdProvider; -import com.baidu.hugegraph.store.term.Bits; -import com.baidu.hugegraph.store.term.HgPair; -import com.baidu.hugegraph.store.util.HgStoreException; import lombok.extern.slf4j.Slf4j; @@ -63,16 +80,16 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class BusinessHandlerImpl implements BusinessHandler { - private static int batchSize = 10000; - private PartitionManager partitionManager; - private PdProvider provider; - private InnerKeyCreator keyCreator; - private static RocksDBFactory factory = RocksDBFactory.getInstance(); - private static HashMap tableMapping = new HashMap<>() {{ + private static final int batchSize = 10000; + private final PartitionManager partitionManager; + private final PdProvider provider; + private final InnerKeyCreator keyCreator; + private static final RocksDBFactory factory = RocksDBFactory.getInstance(); + private static final HashMap tableMapping = new HashMap<>() {{ put(ScanType.SCAN_VERTEX, "vertex"); put(ScanType.SCAN_EDGE, "out_edge"); }}; - private static Map dbNames = new ConcurrentHashMap<>(); + private static final Map dbNames = new ConcurrentHashMap<>(); static { int code = tableUnknown.hashCode(); @@ -86,7 +103,8 @@ public class BusinessHandlerImpl implements BusinessHandler { } - public static HugeConfig initRocksdb(Map rocksdbConfig, RocksdbChangedListener listener) { + public static HugeConfig initRocksdb(Map rocksdbConfig, + RocksdbChangedListener listener) { // 注册rocksdb配置 OptionSpace.register("rocksdb", "org.apache.hugegraph.rocksdb.access.RocksDBOptions"); @@ -123,7 +141,8 @@ public class BusinessHandlerImpl implements BusinessHandler { } @Override - public void doPut(String graph, int code, String table, byte[] key, byte[] value) throws HgStoreException { + public void doPut(String graph, int code, String table, byte[] key, byte[] value) throws + HgStoreException { int partId = provider.getPartitionByCode(graph, code).getId(); try (RocksDBSession dbSession = getSession(graph, table, partId)) { @@ -163,9 +182,13 @@ public class BusinessHandlerImpl implements BusinessHandler { try (RocksDBSession dbSession = getSession(graph, table, id)) { return new InnerKeyFilter(dbSession.sessionOp().scan(table, position == null ? - keyCreator.getStartKey(id, graph) : - keyCreator.getStartKey(id, graph, position), - keyCreator.getEndKey(id, graph), + keyCreator.getStartKey(id, + graph) : + keyCreator.getStartKey(id, + graph, + position), + keyCreator.getEndKey(id, + graph), ScanIterator.Trait.SCAN_LT_END)); } }; @@ -178,7 +201,8 @@ public class BusinessHandlerImpl implements BusinessHandler { } @Override - public ScanIterator scan(String graph, int code, String table, byte[] start, byte[] end, int scanType) + public ScanIterator scan(String graph, int code, String table, byte[] start, byte[] end, + int scanType) throws HgStoreException { List ids; if (code == SCAN_ALL_PARTITIONS_ID) { @@ -200,7 +224,9 @@ public class BusinessHandlerImpl implements BusinessHandler { try (RocksDBSession dbSession = getSession(graph, table, id)) { return new InnerKeyFilter( dbSession.sessionOp().scan(table, - keyCreator.getStartKey(id, graph, toPosition(start, position)), + keyCreator.getStartKey(id, graph, + toPosition(start, + position)), endKey, type)); } }; @@ -232,7 +258,7 @@ public class BusinessHandlerImpl implements BusinessHandler { } byte[] endKey = keyCreator.getEndKey(id, graph); ScanIterator iterator = dbSession.sessionOp().scan(table, startKey, endKey, - ScanIterator.Trait.SCAN_LT_END); + ScanIterator.Trait.SCAN_LT_END); return new InnerKeyFilter(iterator, codeFrom, codeTo); } }; @@ -307,7 +333,8 @@ public class BusinessHandlerImpl implements BusinessHandler { } @Override - public ScanIterator scanPrefix(String graph, int code, String table, byte[] prefix, int scanType) + public ScanIterator scanPrefix(String graph, int code, String table, byte[] prefix, + int scanType) throws HgStoreException { List ids; if (code == SCAN_ALL_PARTITIONS_ID) { @@ -320,7 +347,9 @@ public class BusinessHandlerImpl implements BusinessHandler { try (RocksDBSession dbSession = getSession(graph, table, id)) { return new InnerKeyFilter( dbSession.sessionOp().scan(table, - keyCreator.getPrefixKey(id, graph, toPosition(prefix, position)), + keyCreator.getPrefixKey(id, graph, + toPosition(prefix, + position)), scanType)); } }; @@ -341,7 +370,8 @@ public class BusinessHandlerImpl implements BusinessHandler { @Override public HgStoreMetric.Partition getPartitionMetric(String graph, int partId, - boolean accurateCount) throws HgStoreException { + boolean accurateCount) throws + HgStoreException { // get key count Map countMap = null; Map sizeMap = null; @@ -466,7 +496,8 @@ public class BusinessHandlerImpl implements BusinessHandler { } @Override - public void saveSnapshot(String snapshotPath, String graph, int partId) throws HgStoreException { + public void saveSnapshot(String snapshotPath, String graph, int partId) throws + HgStoreException { try (RocksDBSession dbSession = getSession(graph, partId)) { dbSession.saveSnapshot(snapshotPath); } catch (DBStoreException e) { @@ -475,7 +506,8 @@ public class BusinessHandlerImpl implements BusinessHandler { } @Override - public void loadSnapshot(String snapshotPath, String graph, int partId, long v1) throws HgStoreException { + public void loadSnapshot(String snapshotPath, String graph, int partId, long v1) throws + HgStoreException { try (RocksDBSession dbSession = getSession(graph, partId)) { dbSession.loadSnapshot(snapshotPath, v1); keyCreator.clearCache(partId); @@ -502,7 +534,8 @@ public class BusinessHandlerImpl implements BusinessHandler { } @Override - public void ingestSstFile(String graph, int partId, Map> sstFiles) throws HgStoreException { + public void ingestSstFile(String graph, int partId, Map> sstFiles) throws + HgStoreException { try (RocksDBSession dbSession = getSession(graph, partId)) { dbSession.ingestSstFile(sstFiles); } @@ -511,12 +544,14 @@ public class BusinessHandlerImpl implements BusinessHandler { @Override public boolean cleanPartition(String graph, int partId) { Partition partition = partitionManager.getPartitionFromPD(graph, partId); - cleanPartition(graph, partId, partition.getStartKey(), partition.getEndKey(), CleanType.CLEAN_TYPE_KEEP_RANGE); + cleanPartition(graph, partId, partition.getStartKey(), partition.getEndKey(), + CleanType.CLEAN_TYPE_KEEP_RANGE); return true; } @Override - public boolean cleanPartition(String graph, int partId, long startKey, long endKey, CleanType cleanType) { + public boolean cleanPartition(String graph, int partId, long startKey, long endKey, + CleanType cleanType) { Partition partition = partitionManager.getPartition(graph, partId); if (partition == null) { return true; @@ -539,11 +574,12 @@ public class BusinessHandlerImpl implements BusinessHandler { cleanPartition(partition, code -> { // in range boolean flag = code >= startKey && code < endKey; - return cleanType == CleanType.CLEAN_TYPE_KEEP_RANGE ? flag : !flag; + return (cleanType == CleanType.CLEAN_TYPE_KEEP_RANGE) == flag; }); // 可能被 destroy了 if (HgStoreEngine.getInstance().getPartitionEngine(partId) != null) { - taskManager.updateAsyncTaskState(partId, graph, cleanTask.getId(), AsyncTaskState.SUCCESS); + taskManager.updateAsyncTaskState(partId, graph, cleanTask.getId(), + AsyncTaskState.SUCCESS); } }); return true; @@ -553,8 +589,10 @@ public class BusinessHandlerImpl implements BusinessHandler { * 清理分区数据，删除非本分区的数据 * 遍历partId的所有key，读取code，if code >= splitKey 生成新的key，写入newPartId */ - private boolean cleanPartition(Partition partition, Function belongsFunction) { - log.info("Partition {}-{} cleanPartition begin... {}", partition.getGraphName(), partition.getId(), partition); + private boolean cleanPartition(Partition partition, + Function belongsFunction) { + log.info("Partition {}-{} cleanPartition begin... {}", partition.getGraphName(), + partition.getId(), partition); int counter = 0; SessionOperator op = getSession(partition.getGraphName(), partition.getId()).sessionOp(); @@ -588,7 +626,8 @@ public class BusinessHandlerImpl implements BusinessHandler { } cfIterator.close(); } catch (Exception e) { - log.error("Partition {}-{} cleanPartition exception {}", partition.getGraphName(), partition.getId(), e); + log.error("Partition {}-{} cleanPartition exception {}", partition.getGraphName(), + partition.getId(), e); op.rollback(); throw e; } finally { @@ -626,7 +665,8 @@ public class BusinessHandlerImpl implements BusinessHandler { } } - private RocksDBSession getSession(String graph, String table, int partId) throws HgStoreException { + private RocksDBSession getSession(String graph, String table, int partId) throws + HgStoreException { RocksDBSession dbSession = getSession(partId); dbSession.checkTable(table); return dbSession; @@ -646,7 +686,9 @@ public class BusinessHandlerImpl implements BusinessHandler { RocksDBSession dbSession = factory.queryGraphDB(dbName); if (dbSession == null) { long version = HgStoreEngine.getInstance().getCommittedIndex(partId); - dbSession = factory.createGraphDB(partitionManager.getDbDataPath(partId, dbName), dbName, version); + dbSession = + factory.createGraphDB(partitionManager.getDbDataPath(partId, dbName), dbName, + version); if (dbSession == null) { log.info("failed to create a new graph db: {}", dbName); throw new HgStoreException(HgStoreException.EC_RKDB_CREATE_FAIL, @@ -697,7 +739,8 @@ public class BusinessHandlerImpl implements BusinessHandler { } @Override - public TxBuilder put(int code, String table, byte[] key, byte[] value) throws HgStoreException { + public TxBuilder put(int code, String table, byte[] key, byte[] value) throws + HgStoreException { try { byte[] targetKey = keyCreator.getKey(this.partId, graph, code, key); this.op.put(table, targetKey, value); @@ -726,7 +769,8 @@ public class BusinessHandlerImpl implements BusinessHandler { byte[] targetKey = keyCreator.getKey(this.partId, graph, code, key); op.deleteSingle(table, targetKey); } catch (DBStoreException e) { - throw new HgStoreException(HgStoreException.EC_RDKDB_DOSINGLEDEL_FAIL, e.toString()); + throw new HgStoreException(HgStoreException.EC_RDKDB_DOSINGLEDEL_FAIL, + e.toString()); } return this; @@ -745,7 +789,8 @@ public class BusinessHandlerImpl implements BusinessHandler { } @Override - public TxBuilder delRange(int code, String table, byte[] start, byte[] end) throws HgStoreException { + public TxBuilder delRange(int code, String table, byte[] start, byte[] end) throws + HgStoreException { try { this.op.deleteRange(table, keyCreator.getStartKey(this.partId, graph, start), @@ -758,7 +803,8 @@ public class BusinessHandlerImpl implements BusinessHandler { } @Override - public TxBuilder merge(int code, String table, byte[] key, byte[] value) throws HgStoreException { + public TxBuilder merge(int code, String table, byte[] key, byte[] value) throws + HgStoreException { try { byte[] targetKey = keyCreator.getKey(this.partId, graph, code, key); diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/DataMover.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/DataMover.java similarity index 53% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/business/DataMover.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/business/DataMover.java index 919ac856f..fc4bea951 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/DataMover.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/DataMover.java @@ -1,13 +1,31 @@ -package com.baidu.hugegraph.store.business; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.business; + +import java.util.List; + +import org.apache.hugegraph.store.cmd.BatchPutRequest; +import org.apache.hugegraph.store.cmd.CleanDataRequest; +import org.apache.hugegraph.store.cmd.HgCmdClient; +import org.apache.hugegraph.store.cmd.UpdatePartitionResponse; import com.alipay.sofa.jraft.Status; import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.cmd.BatchPutRequest; -import com.baidu.hugegraph.store.cmd.CleanDataRequest; -import com.baidu.hugegraph.store.cmd.HgCmdClient; -import com.baidu.hugegraph.store.cmd.UpdatePartitionResponse; - -import java.util.List; /** * 数据转移接口，实现分区分裂和合并，支持跨机器转移数据 @@ -18,12 +36,11 @@ public interface DataMover { void setCmdClient(HgCmdClient client); - /** * 拷贝分区source内的数据到其他分区targets * 一个分区，迁移到多个分区 * - * @param source source partition + * @param source source partition * @param targets target partitions * @return execution status * @throws Exception execution exception @@ -42,15 +59,18 @@ public interface DataMover { Status moveData(Metapb.Partition source, Metapb.Partition target) throws Exception; // 同步副本之间的分区状态 - UpdatePartitionResponse updatePartitionState(Metapb.Partition partition, Metapb.PartitionState state); + UpdatePartitionResponse updatePartitionState(Metapb.Partition partition, + Metapb.PartitionState state); // 同步副本之间分区的范围 - UpdatePartitionResponse updatePartitionRange(Metapb.Partition partition, int startKey, int endKey); + UpdatePartitionResponse updatePartitionRange(Metapb.Partition partition, int startKey, + int endKey); // 清理分区partition内的无效数据 void cleanData(Metapb.Partition partition); // 写入数据 void doWriteData(BatchPutRequest request); + void doCleanData(CleanDataRequest request); } diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/DefaultDataMover.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/DefaultDataMover.java similarity index 66% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/business/DefaultDataMover.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/business/DefaultDataMover.java index afc8d2ba7..074e46883 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/DefaultDataMover.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/DefaultDataMover.java @@ -1,31 +1,52 @@ -package com.baidu.hugegraph.store.business; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.alipay.sofa.jraft.Status; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.pd.grpc.pulse.CleanType; -import org.apache.hugegraph.rocksdb.access.RocksDBSession; -import org.apache.hugegraph.rocksdb.access.ScanIterator; -import com.baidu.hugegraph.store.cmd.BatchPutRequest; -import com.baidu.hugegraph.store.cmd.BatchPutResponse; -import com.baidu.hugegraph.store.cmd.CleanDataRequest; -import com.baidu.hugegraph.store.cmd.HgCmdClient; -import com.baidu.hugegraph.store.cmd.UpdatePartitionRequest; -import com.baidu.hugegraph.store.cmd.UpdatePartitionResponse; -import com.baidu.hugegraph.store.term.Bits; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.business; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.function.BiFunction; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.cmd.BatchPutRequest; +import org.apache.hugegraph.store.cmd.BatchPutResponse; +import org.apache.hugegraph.store.cmd.CleanDataRequest; +import org.apache.hugegraph.store.cmd.HgCmdClient; +import org.apache.hugegraph.store.cmd.UpdatePartitionRequest; +import org.apache.hugegraph.store.cmd.UpdatePartitionResponse; +import org.apache.hugegraph.store.term.Bits; + +import com.alipay.sofa.jraft.Status; +import com.baidu.hugegraph.pd.grpc.Metapb; +import com.baidu.hugegraph.pd.grpc.pulse.CleanType; + +import lombok.extern.slf4j.Slf4j; + @Slf4j -public class DefaultDataMover implements DataMover{ +public class DefaultDataMover implements DataMover { public static int Batch_Put_Size = 2000; private BusinessHandler businessHandler; private HgCmdClient client; + @Override public void setBusinessHandler(BusinessHandler handler) { this.businessHandler = handler; @@ -37,24 +58,29 @@ public class DefaultDataMover implements DataMover{ } @Override - public Status moveData(Metapb.Partition source, List targets) throws Exception { + public Status moveData(Metapb.Partition source, List targets) throws + Exception { Status status = Status.OK(); // 开始移动数据之前，先把分区下线 - UpdatePartitionResponse response = updatePartitionState(source, Metapb.PartitionState.PState_Offline); + UpdatePartitionResponse response = + updatePartitionState(source, Metapb.PartitionState.PState_Offline); if (response.getStatus().isOK()) { status = moveData(source, targets, DefaultDataMover::findPartition); // 数据迁移成功后，设置新分区范围和上线新分区 - for (var target : targets){ + for (var target : targets) { if (status.isOk()) { - if (!(updatePartitionRange(target, (int) target.getStartKey(), (int) target.getEndKey()) - .getStatus().isOK() - && updatePartitionState(target, Metapb.PartitionState.PState_Normal).getStatus().isOK())){ + if (!(updatePartitionRange(target, (int) target.getStartKey(), + (int) target.getEndKey()) + .getStatus().isOK() + && updatePartitionState(target, + Metapb.PartitionState.PState_Normal).getStatus() + .isOK())) { status.setError(-3, "new partition online fail"); } } } - }else { + } else { status.setError(-1, "source partition offline fail"); } @@ -66,29 +92,30 @@ public class DefaultDataMover implements DataMover{ @Override public Status moveData(Metapb.Partition source, Metapb.Partition target) throws Exception { // 只写入 target - return moveData(source, Arrays.asList(target), (partitions, integer) -> target); + return moveData(source, Collections.singletonList(target), (partitions, integer) -> target); } /** * move data from partition to targets * - * @param source source partition - * @param targets target partitions + * @param source source partition + * @param targets target partitions * @param partitionSelector the key of source partition belongs which target * @return execution result * @throws Exception exception when put data */ private Status moveData(Metapb.Partition source, List targets, - BiFunction, Integer, Metapb.Partition > partitionSelector) + BiFunction, Integer, Metapb.Partition> partitionSelector) throws Exception { Status status = Status.OK(); String graphName = source.getGraphName(); List tables = businessHandler.getTableNames(graphName, source.getId()); - log.info("moveData, graph:{}, partition id:{} tables:{}, {}-{}", source.getGraphName(), source.getId(), tables, - source.getStartKey(), source.getEndKey()); + log.info("moveData, graph:{}, partition id:{} tables:{}, {}-{}", source.getGraphName(), + source.getId(), tables, + source.getStartKey(), source.getEndKey()); WriteBatch batch = new WriteBatch(graphName); // target partition : count Map moveCount = new HashMap<>(); @@ -99,7 +126,7 @@ public class DefaultDataMover implements DataMover{ try (ScanIterator iterator = businessHandler.scan(graphName, table, (int) source.getStartKey(), - (int) source.getEndKey())) { + (int) source.getEndKey())) { int count = 0; while (iterator.hasNext() && status.isOk()) { total += 1; @@ -109,10 +136,12 @@ public class DefaultDataMover implements DataMover{ int code = Bits.getShort(innerKey, innerKey.length - Short.BYTES); Metapb.Partition partition = partitionSelector.apply(targets, code); if (partition != null) { - moveCount.put(partition.getId(), moveCount.getOrDefault(partition.getId(), 0L) + 1); - batch.add(partition.getId(), BatchPutRequest.KV.of(table, code, key, entry.value)); + moveCount.put(partition.getId(), + moveCount.getOrDefault(partition.getId(), 0L) + 1); + batch.add(partition.getId(), + BatchPutRequest.KV.of(table, code, key, entry.value)); if (++count >= Batch_Put_Size) { - if (!batch.sync()){ + if (!batch.sync()) { status.setError(-2, "move data fail"); } count = 0; @@ -120,14 +149,15 @@ public class DefaultDataMover implements DataMover{ } } if (count > 0) { - if (!batch.sync()){ + if (!batch.sync()) { status.setError(-2, "move data fail"); } } for (var pair : moveCount.entrySet()) { log.info("{}-{}, table: {}, move to partition id {}, count:{}, total:{}", - source.getGraphName(), source.getId(), table, pair.getKey(), pair.getValue(), total); + source.getGraphName(), source.getId(), table, pair.getKey(), + pair.getValue(), total); } } } @@ -135,7 +165,8 @@ public class DefaultDataMover implements DataMover{ return status; } - public UpdatePartitionResponse updatePartitionState(Metapb.Partition partition, Metapb.PartitionState state){ + public UpdatePartitionResponse updatePartitionState(Metapb.Partition partition, + Metapb.PartitionState state) { // 分区分裂时，主动需要查找leader进行同步信息 UpdatePartitionRequest request = new UpdatePartitionRequest(); request.setWorkState(state); @@ -144,7 +175,8 @@ public class DefaultDataMover implements DataMover{ return client.raftUpdatePartition(request); } - public UpdatePartitionResponse updatePartitionRange(Metapb.Partition partition, int startKey, int endKey){ + public UpdatePartitionResponse updatePartitionRange(Metapb.Partition partition, int startKey, + int endKey) { // 分区分裂时，主动需要查找leader进行同步信息 UpdatePartitionRequest request = new UpdatePartitionRequest(); request.setStartKey(startKey); @@ -186,26 +218,29 @@ public class DefaultDataMover implements DataMover{ public void doCleanData(CleanDataRequest request) { // raft 执行真实数据的清理 businessHandler.cleanPartition(request.getGraphName(), request.getPartitionId(), - request.getKeyStart(), request.getKeyEnd(), request.getCleanType()); + request.getKeyStart(), request.getKeyEnd(), + request.getCleanType()); } private static Metapb.Partition findPartition(List partitions, int code) { for (Metapb.Partition partition : partitions) { - if (code >= partition.getStartKey() && code < partition.getEndKey()) + if (code >= partition.getStartKey() && code < partition.getEndKey()) { return partition; + } } return null; } - class WriteBatch{ + class WriteBatch { private final Map> data = new HashMap<>(); private final String graphName; - public WriteBatch(String graphName){ + + public WriteBatch(String graphName) { this.graphName = graphName; } - public WriteBatch add(int partition, BatchPutRequest.KV kv){ - if (!data.containsKey(partition)){ + public WriteBatch add(int partition, BatchPutRequest.KV kv) { + if (!data.containsKey(partition)) { data.put(partition, new LinkedList<>()); } data.get(partition).add(kv); @@ -217,21 +252,22 @@ public class DefaultDataMover implements DataMover{ for (Map.Entry> entry : data.entrySet()) { ret = ret && sendData(entry.getKey(), entry.getValue()); } - for (List list : data.values()) + for (List list : data.values()) { list.clear(); + } return ret; } - public Boolean sendData(Integer partId, List kvs) throws Exception{ + public Boolean sendData(Integer partId, List kvs) throws Exception { BatchPutRequest request = new BatchPutRequest(); request.setGraphName(graphName); request.setPartitionId(partId); request.setEntries(kvs); BatchPutResponse response = client.batchPut(request); - if (response == null || !response.getStatus().isOK()){ + if (response == null || !response.getStatus().isOK()) { log.error("sendData moveData error, pId:{} status:{}", partId, - response != null ? response.getStatus() : "EMPTY_RESPONSE"); + response != null ? response.getStatus() : "EMPTY_RESPONSE"); return false; } return true; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/FilterIterator.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/FilterIterator.java similarity index 79% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/business/FilterIterator.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/business/FilterIterator.java index 62aed5159..309da32cc 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/FilterIterator.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/FilterIterator.java @@ -1,12 +1,29 @@ -package com.baidu.hugegraph.store.business; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.business; import org.apache.commons.lang3.ArrayUtils; +import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; +import org.apache.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.backend.query.ConditionQuery; import com.baidu.hugegraph.backend.serializer.BinaryBackendEntry; import com.baidu.hugegraph.backend.store.BackendEntry; -import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; -import org.apache.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.structure.HugeElement; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/GraphStoreIterator.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/GraphStoreIterator.java similarity index 89% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/business/GraphStoreIterator.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/business/GraphStoreIterator.java index 83bb44cda..b4d3ea511 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/GraphStoreIterator.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/GraphStoreIterator.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.business; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.business; import java.util.ArrayList; import java.util.Date; @@ -13,6 +30,8 @@ import javax.script.ScriptEngineManager; import javax.script.ScriptException; import org.apache.commons.lang.StringUtils; +import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; +import org.apache.hugegraph.rocksdb.access.ScanIterator; import org.apache.tinkerpop.gremlin.structure.Property; import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.codehaus.groovy.jsr223.GroovyScriptEngineImpl; @@ -20,8 +39,6 @@ import org.codehaus.groovy.jsr223.GroovyScriptEngineImpl; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.serializer.BinaryBackendEntry; import com.baidu.hugegraph.backend.store.BackendEntry; -import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; -import org.apache.hugegraph.rocksdb.access.ScanIterator; import com.baidu.hugegraph.schema.EdgeLabel; import com.baidu.hugegraph.schema.PropertyKey; import com.baidu.hugegraph.schema.VertexLabel; @@ -51,15 +68,15 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class GraphStoreIterator extends AbstractSelectIterator implements ScanIterator { - private ScanPartitionRequest scanRequest; - private ScanIterator iter; - private Request request; - private boolean isVertex; + private final ScanPartitionRequest scanRequest; + private final ScanIterator iter; + private final Request request; + private final boolean isVertex; private Vertex.Builder vertex; private Edge.Builder edge; - private HugeType type; + private final HugeType type; private ArrayList data; - private Set properties; + private final Set properties; private GroovyScriptEngineImpl engine; private CompiledScript script; private HugeElement current; @@ -103,7 +120,7 @@ public class GraphStoreIterator extends AbstractSelectIterator BackendEntry.BackendColumn column = BackendEntry.BackendColumn.of( next.name, next.value); if (entry == null || !belongToMe(entry, column) || !isVertex) { - try{ + try { entry = new BinaryBackendEntry(type, next.name); } catch (Exception e) { log.error("using core to new entry with error:", e); @@ -147,7 +164,7 @@ public class GraphStoreIterator extends AbstractSelectIterator log.error("get next with error:", e); } } - }else { + } else { return true; } return false; @@ -193,7 +210,7 @@ public class GraphStoreIterator extends AbstractSelectIterator Iterator

    eps) { int pSize = properties.size(); List props = new ArrayList<>(pSize > 0 ? - pSize : size); + pSize : size); Graphpb.Property.Builder pb = Graphpb.Property.newBuilder(); while (eps.hasNext()) { HugeProperty property = (HugeProperty) eps.next(); @@ -290,9 +307,9 @@ public class GraphStoreIterator extends AbstractSelectIterator } } - private static Descriptors.FieldDescriptor propertiesDesEdge = + private static final Descriptors.FieldDescriptor propertiesDesEdge = Edge.getDescriptor().findFieldByNumber(6); - private static Descriptors.FieldDescriptor propertiesDesVertex = + private static final Descriptors.FieldDescriptor propertiesDesVertex = Vertex.getDescriptor().findFieldByNumber(3); private Edge parseEdge(HugeElement element) { diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/InnerKeyCreator.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/InnerKeyCreator.java similarity index 73% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/business/InnerKeyCreator.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/business/InnerKeyCreator.java index 869ee1469..6d48a65c2 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/InnerKeyCreator.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/InnerKeyCreator.java @@ -1,12 +1,29 @@ -package com.baidu.hugegraph.store.business; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.business; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import com.baidu.hugegraph.store.meta.GraphIdManager; -import com.baidu.hugegraph.store.term.Bits; -import com.baidu.hugegraph.store.util.HgStoreException; +import org.apache.hugegraph.store.meta.GraphIdManager; +import org.apache.hugegraph.store.term.Bits; +import org.apache.hugegraph.store.util.HgStoreException; import lombok.extern.slf4j.Slf4j; @@ -19,7 +36,7 @@ public class InnerKeyCreator { this.businessHandler = businessHandler; } - private volatile Map graphIdCache = new ConcurrentHashMap<>(); + private final Map graphIdCache = new ConcurrentHashMap<>(); public int getGraphId(Integer partId, String graphName) throws HgStoreException { diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/InnerKeyFilter.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/InnerKeyFilter.java similarity index 58% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/business/InnerKeyFilter.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/business/InnerKeyFilter.java index 6a116fbe0..ac97b4294 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/InnerKeyFilter.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/InnerKeyFilter.java @@ -1,10 +1,27 @@ -package com.baidu.hugegraph.store.business; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.business; + +import java.util.Arrays; import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; import org.apache.hugegraph.rocksdb.access.ScanIterator; -import com.baidu.hugegraph.store.term.Bits; - -import java.util.Arrays; +import org.apache.hugegraph.store.term.Bits; public class InnerKeyFilter implements ScanIterator { ScanIterator iterator; @@ -14,7 +31,8 @@ public class InnerKeyFilter implements ScanIterator { //是否进行code过滤，启动该选项，返回key的尾部包含code final boolean codeFilter; - public InnerKeyFilter(ScanIterator iterator){ + + public InnerKeyFilter(ScanIterator iterator) { this.iterator = iterator; this.codeFrom = Integer.MIN_VALUE; this.codeTo = Integer.MAX_VALUE; @@ -22,7 +40,7 @@ public class InnerKeyFilter implements ScanIterator { moveNext(); } - public InnerKeyFilter(ScanIterator iterator, int codeFrom, int codeTo){ + public InnerKeyFilter(ScanIterator iterator, int codeFrom, int codeTo) { this.iterator = iterator; this.codeFrom = codeFrom; this.codeTo = codeTo; @@ -30,7 +48,7 @@ public class InnerKeyFilter implements ScanIterator { moveNext(); } - private void moveNext(){ + private void moveNext() { current = null; if (codeFilter) { while (iterator.hasNext()) { @@ -42,13 +60,15 @@ public class InnerKeyFilter implements ScanIterator { } } } else { - if (iterator.hasNext()) + if (iterator.hasNext()) { current = iterator.next(); + } } } + @Override public boolean hasNext() { - return current!= null; + return current != null; } @Override @@ -60,12 +80,15 @@ public class InnerKeyFilter implements ScanIterator { public T next() { T column = current; if (!codeFilter) - // 去掉图ID和hash后缀 + // 去掉图ID和hash后缀 + { column.name = Arrays.copyOfRange(column.name, Short.BYTES, - column.name.length - Short.BYTES); - else// 去掉图ID + column.name.length - Short.BYTES); + } else// 去掉图ID + { column.name = Arrays.copyOfRange(column.name, Short.BYTES, - column.name.length); + column.name.length); + } moveNext(); return column; } diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/MultiPartitionIterator.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/MultiPartitionIterator.java similarity index 75% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/business/MultiPartitionIterator.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/business/MultiPartitionIterator.java index c979589f2..0b2f683a9 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/MultiPartitionIterator.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/MultiPartitionIterator.java @@ -1,11 +1,32 @@ -package com.baidu.hugegraph.store.business; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import org.apache.hugegraph.rocksdb.access.ScanIterator; +package org.apache.hugegraph.store.business; import java.nio.ByteBuffer; -import java.util.*; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Queue; import java.util.function.BiFunction; +import org.apache.hugegraph.rocksdb.access.ScanIterator; + import lombok.extern.slf4j.Slf4j; /** @@ -15,18 +36,20 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class MultiPartitionIterator implements ScanIterator { public final static byte[] EMPTY_BYTES = new byte[0]; - private Queue partitions; - private BiFunction supplier; + private final Queue partitions; + private final BiFunction supplier; private ScanIterator iterator; private Integer curPartitionId; private Integer positionPartitionId; private byte[] positionKey; - public static MultiPartitionIterator of(List partitionIdList, BiFunction supplier) { + public static MultiPartitionIterator of(List partitionIdList, + BiFunction supplier) { return new MultiPartitionIterator(partitionIdList, supplier); } - private MultiPartitionIterator(List partitionIds, BiFunction supplier) { + private MultiPartitionIterator(List partitionIds, + BiFunction supplier) { /***************************************************************************** ** CAUTION: MAKE SURE IT SORTED IN A FIXED ORDER! TO DO THIS IS FOR PAGING. ** *****************************************************************************/ @@ -72,21 +95,13 @@ public class MultiPartitionIterator implements ScanIterator { @Override public boolean hasNext() { this.init(); - if (this.iterator == null) { - return false; - } else { - return true; - } + return this.iterator != null; } @Override public boolean isValid() { this.init(); - if (this.iterator == null) { - return false; - } else { - return true; - } + return this.iterator != null; } @Override @@ -146,8 +161,7 @@ public class MultiPartitionIterator implements ScanIterator { private boolean inPosition(int partitionId) { if (this.positionPartitionId == null) return true; - if (partitionId >= this.positionPartitionId) return true; - return false; + return partitionId >= this.positionPartitionId; } private byte[] getPositionKey(int partitionId) { diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/SelectIterator.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/SelectIterator.java similarity index 75% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/business/SelectIterator.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/business/SelectIterator.java index f0feb9744..014f854e2 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/business/SelectIterator.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/business/SelectIterator.java @@ -1,13 +1,31 @@ -package com.baidu.hugegraph.store.business; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.business; import java.util.HashSet; import java.util.List; import java.util.Set; -import com.baidu.hugegraph.backend.id.Id; -import com.baidu.hugegraph.backend.serializer.BytesBuffer; import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn; import org.apache.hugegraph.rocksdb.access.ScanIterator; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.serializer.BytesBuffer; import com.baidu.hugegraph.type.define.DataType; import com.baidu.hugegraph.type.define.SerialEnum; diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/BatchPutRequest.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/BatchPutRequest.java new file mode 100644 index 000000000..53c3b9188 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/BatchPutRequest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +import java.io.Serializable; +import java.util.List; + +import lombok.Data; + +@Data +public class BatchPutRequest extends HgCmdBase.BaseRequest { + private List entries; + + @Override + public byte magic() { + return HgCmdBase.BATCH_PUT; + } + + @Data + public static class KV implements Serializable { + private String table; + private int code; + private byte[] key; + private byte[] value; + + public static KV of(String table, int code, byte[] key, byte[] value) { + KV kv = new KV(); + kv.table = table; + kv.code = code; + kv.key = key; + kv.value = value; + return kv; + } + } +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/BatchPutResponse.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/BatchPutResponse.java new file mode 100644 index 000000000..a197c261c --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/BatchPutResponse.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +public class BatchPutResponse extends HgCmdBase.BaseResponse { +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CleanDataRequest.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CleanDataRequest.java new file mode 100644 index 000000000..7ed27ab01 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CleanDataRequest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +import org.apache.hugegraph.store.meta.Partition; + +import com.baidu.hugegraph.pd.grpc.pulse.CleanPartition; +import com.baidu.hugegraph.pd.grpc.pulse.CleanType; + +import lombok.Data; + + +@Data +public class CleanDataRequest extends HgCmdBase.BaseRequest { + + private long keyStart; + private long keyEnd; + + private CleanType cleanType; + + private boolean deletePartition; + + private long taskId; + + @Override + public byte magic() { + return HgCmdBase.CLEAN_DATA; + } + + public static CleanDataRequest fromCleanPartitionTask(CleanPartition task, Partition partition, + long taskId) { + return fromCleanPartitionTask(partition.getGraphName(), partition.getId(), taskId, task); + } + + public static CleanDataRequest fromCleanPartitionTask(String graphName, int partitionId, + long taskId, + CleanPartition task) { + CleanDataRequest request = new CleanDataRequest(); + request.setGraphName(graphName); + request.setPartitionId(partitionId); + request.setCleanType(task.getCleanType()); + request.setKeyStart(task.getKeyStart()); + request.setKeyEnd(task.getKeyEnd()); + request.setDeletePartition(task.getDeletePartition()); + request.setTaskId(taskId); + return request; + } + + public static CleanPartition toCleanPartitionTask(CleanDataRequest request) { + return CleanPartition.newBuilder() + .setKeyStart(request.keyStart) + .setKeyEnd(request.keyEnd) + .setDeletePartition(request.deletePartition) + .setCleanType(request.cleanType) + .build(); + } +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CleanDataResponse.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CleanDataResponse.java new file mode 100644 index 000000000..e280494dd --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CleanDataResponse.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +public class CleanDataResponse extends HgCmdBase.BaseResponse { +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/CreateRaftRequest.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CreateRaftRequest.java similarity index 54% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/CreateRaftRequest.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CreateRaftRequest.java index 4884260ca..32d5ba762 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/CreateRaftRequest.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CreateRaftRequest.java @@ -1,46 +1,68 @@ -package com.baidu.hugegraph.store.cmd; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.alipay.sofa.jraft.conf.Configuration; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.google.protobuf.InvalidProtocolBufferException; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.cmd; import java.util.ArrayList; import java.util.List; +import com.alipay.sofa.jraft.conf.Configuration; +import com.baidu.hugegraph.pd.grpc.Metapb; +import com.google.protobuf.InvalidProtocolBufferException; + +import lombok.extern.slf4j.Slf4j; + @Slf4j -public class CreateRaftRequest extends HgCmdBase.BaseRequest{ +public class CreateRaftRequest extends HgCmdBase.BaseRequest { public List getPartitions() { try { List partitions = new ArrayList<>(); - for (byte[] partition : values) + for (byte[] partition : values) { partitions.add(Metapb.Partition.parseFrom(partition)); + } return partitions; } catch (InvalidProtocolBufferException e) { log.error("CreateRaftNodeProcessor parse partition exception }", e); } return new ArrayList<>(); } + public void addPartition(Metapb.Partition partition) { values.add(partition.toByteArray()); } public void setConf(Configuration conf) { - if ( conf != null ){ + if (conf != null) { this.peers = conf.toString(); } } public Configuration getConf() { Configuration conf = null; - if ( peers != null) { + if (peers != null) { conf = new Configuration(); conf.parse(this.peers); } return conf; } + List values = new ArrayList<>(); String peers; + @Override public byte magic() { return HgCmdBase.CREATE_RAFT; diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CreateRaftResponse.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CreateRaftResponse.java new file mode 100644 index 000000000..77990e282 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/CreateRaftResponse.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +public class CreateRaftResponse extends HgCmdBase.BaseResponse { +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DbCompactionRequest.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DbCompactionRequest.java new file mode 100644 index 000000000..0cc32721b --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DbCompactionRequest.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +import lombok.Data; + +@Data +public class DbCompactionRequest extends HgCmdBase.BaseRequest { + private String tableName; + + @Override + public byte magic() { + return HgCmdBase.ROCKSDB_COMPACTION; + } +} + diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DbCompactionResponse.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DbCompactionResponse.java new file mode 100644 index 000000000..d5c3418b9 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DbCompactionResponse.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +public class DbCompactionResponse extends HgCmdBase.BaseResponse { +} \ No newline at end of file diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DestroyRaftRequest.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DestroyRaftRequest.java new file mode 100644 index 000000000..5e17f27fa --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DestroyRaftRequest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +import java.util.ArrayList; +import java.util.List; + +import lombok.Data; + +@Data +public class DestroyRaftRequest extends HgCmdBase.BaseRequest { + private final List graphNames = new ArrayList<>(); + + public void addGraphName(String graphName) { + graphNames.add(graphName); + } + + @Override + public byte magic() { + return HgCmdBase.DESTROY_RAFT; + } +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DestroyRaftResponse.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DestroyRaftResponse.java new file mode 100644 index 000000000..4e6633af5 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/DestroyRaftResponse.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +public class DestroyRaftResponse extends HgCmdBase.BaseResponse { + +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/FutureClosureAdapter.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/FutureClosureAdapter.java new file mode 100644 index 000000000..ce0627d44 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/FutureClosureAdapter.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +import java.util.concurrent.CompletableFuture; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.Status; + +public class FutureClosureAdapter implements Closure { + public final CompletableFuture future = new CompletableFuture<>(); + private T resp; + + public T getResponse() { + return this.resp; + } + + public void setResponse(T resp) { + this.resp = resp; + future.complete(resp); + run(Status.OK()); + } + + public void failure(Throwable t) { + future.completeExceptionally(t); + run(new Status(-1, t.getMessage())); + } + + @Override + public void run(Status status) { + + } +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/GetStoreInfoRequest.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/GetStoreInfoRequest.java new file mode 100644 index 000000000..accbc2890 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/GetStoreInfoRequest.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +public class GetStoreInfoRequest extends HgCmdBase.BaseRequest { + @Override + public byte magic() { + return HgCmdBase.GET_STORE_INFO; + } +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/GetStoreInfoResponse.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/GetStoreInfoResponse.java new file mode 100644 index 000000000..12c6f4fc1 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/GetStoreInfoResponse.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +import org.apache.hugegraph.store.meta.Store; + +import com.baidu.hugegraph.pd.grpc.Metapb; +import com.google.protobuf.InvalidProtocolBufferException; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class GetStoreInfoResponse extends HgCmdBase.BaseResponse { + private byte[] store; + + public void setStore(Store store) { + this.store = store.getProtoObj().toByteArray(); + } + + public Store getStore() { + try { + return new Store(Metapb.Store.parseFrom(this.store)); + } catch (InvalidProtocolBufferException e) { + log.error("GetStoreResponse parse exception {}", e); + } + return null; + } +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/HgCmdBase.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/HgCmdBase.java new file mode 100644 index 000000000..88b315961 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/HgCmdBase.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import lombok.Data; + +public class HgCmdBase { + + public static final byte GET_STORE_INFO = 0x01; + public static final byte BATCH_PUT = 0x02; + public static final byte CLEAN_DATA = 0x03; + public static final byte RAFT_UPDATE_PARTITION = 0x04; + public static final byte ROCKSDB_COMPACTION = 0x05; + public static final byte CREATE_RAFT = 0x06; + public static final byte DESTROY_RAFT = 0x07; + + @Data + public abstract static class BaseRequest implements Serializable { + private String graphName; + private int partitionId; + + public abstract byte magic(); + } + + @Data + public abstract static class BaseResponse implements Serializable { + private HgCmdProcessor.Status status; + List partitionLeaders; + + public static class PartitionLeader implements Serializable { + private final Integer partId; + private final Long storeId; + + public PartitionLeader(Integer partId, Long storeId) { + this.partId = partId; + this.storeId = storeId; + } + + public Long getStoreId() { + return storeId; + } + + public Integer getPartId() { + return partId; + } + } + + public synchronized BaseResponse addPartitionLeader(PartitionLeader ptLeader) { + if (partitionLeaders == null) { + partitionLeaders = new ArrayList<>(); + } + partitionLeaders.add(ptLeader); + return this; + } + } +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/HgCmdClient.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/HgCmdClient.java similarity index 81% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/HgCmdClient.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/HgCmdClient.java index 0e84385c3..66f1ec606 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/HgCmdClient.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/HgCmdClient.java @@ -1,4 +1,30 @@ -package com.baidu.hugegraph.store.cmd; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.meta.Store; import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.JRaftUtils; @@ -11,17 +37,8 @@ import com.alipay.sofa.jraft.rpc.RaftRpcFactory; import com.alipay.sofa.jraft.rpc.RpcClient; import com.alipay.sofa.jraft.util.Endpoint; import com.alipay.sofa.jraft.util.RpcFactoryHelper; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.meta.Store; -import com.baidu.hugegraph.store.pd.PdProvider; -import lombok.extern.slf4j.Slf4j; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import lombok.extern.slf4j.Slf4j; @Slf4j public class HgCmdClient { @@ -39,12 +56,14 @@ public class HgCmdClient { this.ptAgent = ptAgent; this.rpcOptions = rpcOptions; final RaftRpcFactory factory = RpcFactoryHelper.rpcFactory(); - this.rpcClient = factory.createRpcClient(factory.defaultJRaftClientConfigHelper(this.rpcOptions)); + this.rpcClient = + factory.createRpcClient(factory.defaultJRaftClientConfigHelper(this.rpcOptions)); return this.rpcClient.init(rpcOptions); } - public Future createRaftNode(final String address, final List partitions, final Closure done) { + public Future createRaftNode(final String address, final List partitions, + final Closure done) { CreateRaftRequest request = new CreateRaftRequest(); partitions.forEach(partition -> { request.addPartition(partition.getProtoObj()); @@ -66,7 +85,8 @@ public class HgCmdClient { return internalCallAsyncWithRpc(JRaftUtils.getEndPoint(address), request, done); } - public Future destroyRaftNode(final String peer, final List partitions, final Closure done) { + public Future destroyRaftNode(final String peer, final List partitions, + final Closure done) { DestroyRaftRequest request = new DestroyRaftRequest(); partitions.forEach(partition -> { @@ -74,7 +94,8 @@ public class HgCmdClient { request.addGraphName(partition.getGraphName()); }); - log.info("Send to {} DestroyRaftNode rpc call partitionId={} ", peer, request.getPartitionId()); + log.info("Send to {} DestroyRaftNode rpc call partitionId={} ", peer, + request.getPartitionId()); return internalCallAsyncWithRpc(JRaftUtils.getEndPoint(peer), request, done); } @@ -133,9 +154,11 @@ public class HgCmdClient { for (int i = 0; i < MAX_RETRY_TIMES; i++) { try { - Endpoint leader = ptAgent.getPartitionLeader(request.getGraphName(), request.getPartitionId()); + Endpoint leader = ptAgent.getPartitionLeader(request.getGraphName(), + request.getPartitionId()); if (leader == null) { - log.error("get leader of graph {} - {} is null", request.getGraphName(), request.getPartitionId()); + log.error("get leader of graph {} - {} is null", request.getGraphName(), + request.getPartitionId()); Thread.sleep(i * 1000); continue; } @@ -145,11 +168,12 @@ public class HgCmdClient { if (response.getStatus().isOK()) { break; } else if (HgCmdProcessor.Status.LEADER_REDIRECT == response.getStatus() - && response.partitionLeaders != null + && response.partitionLeaders != null ) { // 当返回leader 漂移，并且partitionLeaders 不为空时，需要重新设置leader } else { - log.error("HgCmdClient tryInternalCallSyncWithRpc error msg {} leaders is {}", + log.error( + "HgCmdClient tryInternalCallSyncWithRpc error msg {} leaders is {}", response.getStatus().getMsg(), response.getPartitionLeaders()); } } @@ -163,7 +187,8 @@ public class HgCmdClient { return response; } - private V internalCallSyncWithRpc(final Endpoint endpoint, final HgCmdBase.BaseRequest request) + private V internalCallSyncWithRpc(final Endpoint endpoint, + final HgCmdBase.BaseRequest request) throws ExecutionException, InterruptedException, TimeoutException { FutureClosureAdapter response = new FutureClosureAdapter<>(); internalCallAsyncWithRpc(endpoint, request, response); @@ -189,7 +214,8 @@ public class HgCmdClient { return response.future; } - private void internalCallAsyncWithRpc(final Endpoint endpoint, final HgCmdBase.BaseRequest request, + private void internalCallAsyncWithRpc(final Endpoint endpoint, + final HgCmdBase.BaseRequest request, final FutureClosureAdapter closure) { final InvokeContext invokeCtx = null; int[] retryCount = new int[]{0}; @@ -210,7 +236,7 @@ public class HgCmdClient { }; try { this.rpcClient.invokeAsync(endpoint, request, invokeCtx, invokeCallback, - this.rpcOptions.getRpcDefaultTimeout()); + this.rpcOptions.getRpcDefaultTimeout()); } catch (final Throwable err) { tryWithThrowable(endpoint, request, closure, invokeCtx, retryCount, err); } diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/HgCmdProcessor.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/HgCmdProcessor.java similarity index 83% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/HgCmdProcessor.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/HgCmdProcessor.java index c5424c2ce..fff649591 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/cmd/HgCmdProcessor.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/HgCmdProcessor.java @@ -1,19 +1,38 @@ -package com.baidu.hugegraph.store.cmd; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.alipay.sofa.jraft.rpc.RpcContext; -import com.alipay.sofa.jraft.rpc.RpcProcessor; -import com.alipay.sofa.jraft.rpc.RpcServer; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.raft.RaftClosure; -import com.baidu.hugegraph.store.raft.RaftOperation; -import com.baidu.hugegraph.store.util.HgRaftError; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.cmd; import java.io.Serializable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.raft.RaftClosure; +import org.apache.hugegraph.store.raft.RaftOperation; +import org.apache.hugegraph.store.util.HgRaftError; + +import com.alipay.sofa.jraft.rpc.RpcContext; +import com.alipay.sofa.jraft.rpc.RpcProcessor; +import com.alipay.sofa.jraft.rpc.RpcServer; + +import lombok.extern.slf4j.Slf4j; + /** * 快照同步rpc处理器，leader批量入库完成后，基于seqnum读取新增的kv,批量发送给follower. * @@ -60,7 +79,8 @@ public class HgCmdProcessor implements RpcProce } case HgCmdBase.RAFT_UPDATE_PARTITION: { response = new UpdatePartitionResponse(); - handleUpdatePartition((UpdatePartitionRequest) request, (UpdatePartitionResponse) response); + handleUpdatePartition((UpdatePartitionRequest) request, + (UpdatePartitionResponse) response); break; } case HgCmdBase.CREATE_RAFT: { @@ -90,7 +110,8 @@ public class HgCmdProcessor implements RpcProce response.setStatus(Status.OK); } - public void handleUpdatePartition(UpdatePartitionRequest request, UpdatePartitionResponse response) { + public void handleUpdatePartition(UpdatePartitionRequest request, + UpdatePartitionResponse response) { raftSyncTask(request, response, RaftOperation.RAFT_UPDATE_PARTITION); } @@ -103,7 +124,8 @@ public class HgCmdProcessor implements RpcProce } public void handleCreateRaft(CreateRaftRequest request, CreateRaftResponse response) { - log.info("CreateRaftNode rpc call received, {}, {}", request.getPartitions(), request.getConf()); + log.info("CreateRaftNode rpc call received, {}, {}", request.getPartitions(), + request.getConf()); request.getPartitions().forEach(partition -> { engine.createPartitionEngine(new Partition(partition), request.getConf()); }); @@ -123,10 +145,11 @@ public class HgCmdProcessor implements RpcProce * @param response * @param op */ - private void raftSyncTask(HgCmdBase.BaseRequest request, HgCmdBase.BaseResponse response, final byte op) { + private void raftSyncTask(HgCmdBase.BaseRequest request, HgCmdBase.BaseResponse response, + final byte op) { CountDownLatch latch = new CountDownLatch(1); engine.addRaftTask(request.getGraphName(), request.getPartitionId(), - RaftOperation.create(op, request), new RaftClosure() { + RaftOperation.create(op, request), new RaftClosure() { @Override public void run(com.alipay.sofa.jraft.Status status) { Status responseStatus = Status.UNKNOWN; @@ -178,7 +201,7 @@ public class HgCmdProcessor implements RpcProce ABORT(100, "Transmission aborted"); - private int code; + private final int code; private String msg; Status(int code, String msg) { diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/UpdatePartitionRequest.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/UpdatePartitionRequest.java new file mode 100644 index 000000000..2d7b405fd --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/UpdatePartitionRequest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +import com.baidu.hugegraph.pd.grpc.Metapb; + +import lombok.Data; + + +@Data +public class UpdatePartitionRequest extends HgCmdBase.BaseRequest { + private int startKey; + private int endKey; + + private Metapb.PartitionState workState; + + @Override + public byte magic() { + return HgCmdBase.RAFT_UPDATE_PARTITION; + } +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/UpdatePartitionResponse.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/UpdatePartitionResponse.java new file mode 100644 index 000000000..98bc247b2 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/cmd/UpdatePartitionResponse.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cmd; + +public class UpdatePartitionResponse extends HgCmdBase.BaseResponse { +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/DeletedFileManager.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/DeletedFileManager.java similarity index 59% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/DeletedFileManager.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/meta/DeletedFileManager.java index 75033d53b..27a099a2d 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/DeletedFileManager.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/DeletedFileManager.java @@ -1,16 +1,34 @@ -package com.baidu.hugegraph.store.meta; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import org.apache.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.store.meta.base.GlobalMetaStore; -import com.baidu.hugegraph.store.options.MetadataOptions; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; +package org.apache.hugegraph.store.meta; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.meta.base.GlobalMetaStore; +import org.apache.hugegraph.store.options.MetadataOptions; + +import lombok.extern.slf4j.Slf4j; + @Slf4j public class DeletedFileManager extends GlobalMetaStore { public DeletedFileManager(MetadataOptions options) { diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/Graph.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Graph.java similarity index 52% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/Graph.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Graph.java index c2c1b61bc..85d5b31f8 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/Graph.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Graph.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.meta; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta; import com.baidu.hugegraph.pd.grpc.Metapb; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/GraphIdManager.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/GraphIdManager.java similarity index 80% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/GraphIdManager.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/meta/GraphIdManager.java index 5e38920c4..49c4778ba 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/GraphIdManager.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/GraphIdManager.java @@ -1,13 +1,31 @@ -package com.baidu.hugegraph.store.meta; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta; import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import com.baidu.hugegraph.store.meta.base.DBSessionBuilder; -import com.baidu.hugegraph.store.meta.base.PartitionMetaStore; -import com.baidu.hugegraph.store.util.HgStoreException; +import org.apache.hugegraph.store.meta.base.DBSessionBuilder; +import org.apache.hugegraph.store.meta.base.PartitionMetaStore; +import org.apache.hugegraph.store.util.HgStoreException; + import com.google.protobuf.Int64Value; /** @@ -17,13 +35,14 @@ public class GraphIdManager extends PartitionMetaStore { final DBSessionBuilder sessionBuilder; final int partitionId; + public GraphIdManager(DBSessionBuilder sessionBuilder, int partitionId) { super(sessionBuilder, partitionId); this.sessionBuilder = sessionBuilder; this.partitionId = partitionId; } - private Map graphIdCache = new ConcurrentHashMap<>(); + private final Map graphIdCache = new ConcurrentHashMap<>(); static Object graphIdLock = new Object(); @@ -41,7 +60,8 @@ public class GraphIdManager extends PartitionMetaStore { id = Int64Value.of(getCId(GRAPH_ID_PREFIX, maxGraphID)); if (id.getValue() == -1) { throw new HgStoreException(HgStoreException.EC_FAIL, - "The number of graphs exceeds the maximum 65535"); + "The number of graphs exceeds the maximum " + + "65535"); } put(key, id); flush(); @@ -121,7 +141,8 @@ public class GraphIdManager extends PartitionMetaStore { if (current == max) { current = 0; - ids = scan(Int64Value.parser(), genCIDSlotKey(key, current), genCIDSlotKey(key, last)); + ids = scan(Int64Value.parser(), genCIDSlotKey(key, current), + genCIDSlotKey(key, last)); for (Int64Value id : ids) { if (current == id.getValue()) { current++; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/GraphManager.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/GraphManager.java similarity index 53% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/GraphManager.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/meta/GraphManager.java index c259d9361..8ba12e71c 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/GraphManager.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/GraphManager.java @@ -1,18 +1,36 @@ -package com.baidu.hugegraph.store.meta; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.hugegraph.store.meta.base.GlobalMetaStore; +import org.apache.hugegraph.store.options.MetadataOptions; +import org.apache.hugegraph.store.pd.PdProvider; + import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.meta.base.GlobalMetaStore; -import com.baidu.hugegraph.store.options.MetadataOptions; -import com.baidu.hugegraph.store.pd.PdProvider; public class GraphManager extends GlobalMetaStore { - private PdProvider pdProvider; + private final PdProvider pdProvider; - private Map graphs; + private final Map graphs; public GraphManager(MetadataOptions options, PdProvider pdProvider) { super(options); @@ -24,6 +42,7 @@ public class GraphManager extends GlobalMetaStore { /** * 修改图 * 此处不加锁，要求graph是被克隆的，进制修改原始对象 + * * @param graph * @return */ @@ -52,13 +71,14 @@ public class GraphManager extends GlobalMetaStore { return graphs.get(graphName); } - public Graph getCloneGraph(String graphName){ - if ( graphs.containsKey(graphName)) - return graphs.get(graphName).clone(); + public Graph getCloneGraph(String graphName) { + if (graphs.containsKey(graphName)) { + return graphs.get(graphName).clone(); + } return new Graph(); } - public Graph removeGraph(String graphName){ + public Graph removeGraph(String graphName) { byte[] key = MetadataKeyHelper.getGraphKey(graphName); delete(key); return graphs.remove(graphName); diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/MetadataKeyHelper.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/MetadataKeyHelper.java new file mode 100644 index 000000000..c7b3dce65 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/MetadataKeyHelper.java @@ -0,0 +1,315 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta; + +import java.nio.charset.StandardCharsets; + +public class MetadataKeyHelper { + private static final char DELIMITER = '/'; + private static final String HUGEGRAPH = "HUGEGRAPH"; + private static final String STORE = "STORE"; + private static final String PARTITION = "PARTITION"; + private static final String TASK = "TASK"; + private static final String ASYNC_TASK = "A_TASK"; + private static final String INSTRUCTION_TASK = "INS_TASK"; + private static final String TASK_DONE = "TASK_DONE"; + private static final String GRAPH = "GRAPH"; + private static final String PARTITION_STORE = "PARTITION_STORE"; + private static final String PARTITION_RAFT = "PARTITION_Raft"; + private static final String DELETED_FILE = "DELETED_FILE"; + private static final String SHARD_GROUP = "SHARDGROUP"; + + private static final String CID_PREFIX = "CID"; + private static final String CID_SLOT_PREFIX = "CID_SLOT"; + private static final String GRAPH_ID_PREFIX = "GRAPH_ID"; + + public static byte[] getPartitionKey(String graph, Integer partId) { + // HUGEGRAPH/Partition/{graph}/partId + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(PARTITION).append(DELIMITER) + .append(graph).append(DELIMITER) + .append(partId) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getPartitionPrefixKey(String graph) { + // HUGEGRAPH/Partition/{graph}/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(PARTITION).append(DELIMITER) + .append(graph).append(DELIMITER) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + /** + * 查询分区内的所有partition prefix， 不包含 graph name + * + * @return + */ + public static byte[] getPartitionPrefixKey() { + // HUGEGRAPH/Partition/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(PARTITION).append(DELIMITER) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getShardGroupKey(int partitionId) { + // HUGEGRAPH/SHARDGROUP/{partition_id} + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(SHARD_GROUP).append(DELIMITER) + .append(partitionId) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getGraphKey(String graph) { + // HUGEGRAPH/Graph/{graph} + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(GRAPH).append(DELIMITER) + .append(graph) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getGraphKeyPrefix() { + // HUGEGRAPH/Graph/{graph} + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(GRAPH).append(DELIMITER) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getStoreKey() { + // HUGEGRAPH/STORE/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(STORE) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getTaskKey(int partId, String type, long taskId) { + // HUGEGRAPH/TASK/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(TASK).append(DELIMITER) + .append(partId).append(DELIMITER) + .append(type).append(DELIMITER) + .append(String.format("%016x", taskId)) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getInstructionIdKey(long taskId) { + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(INSTRUCTION_TASK).append(DELIMITER) + .append(taskId).append(DELIMITER) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getAsyncTaskKey(int partId, String graphName, String taskId) { + // HUGEGRAPH/A_TASK/ part id / graphName / task id + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(ASYNC_TASK).append(DELIMITER) + .append(partId).append(DELIMITER) + .append(graphName).append(DELIMITER) + .append(taskId) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getAsyncTaskPrefix(int partId, String graphName) { + // HUGEGRAPH/A_TASK/ part id / graphName / task id + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(ASYNC_TASK).append(DELIMITER) + .append(partId).append(DELIMITER) + .append(graphName).append(DELIMITER) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getTaskPrefix(int partId, String type) { + // HUGEGRAPH/TASK/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(TASK).append(DELIMITER) + .append(partId).append(DELIMITER) + .append(type).append(DELIMITER) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getTaskPrefix(int partId) { + // HUGEGRAPH/TASK/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(TASK).append(DELIMITER) + .append(partId).append(DELIMITER) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getTaskPrefix() { + // HUGEGRAPH/TASK/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(TASK).append(DELIMITER) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getDoneTaskKey(long taskId) { + // HUGEGRAPH/TASK/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(TASK_DONE).append(DELIMITER) + .append(String.format("%016x", taskId)) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getPartitionStoreKey(int partId) { + // HUGEGRAPH/TASK/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(PARTITION_STORE).append(DELIMITER) + .append(partId) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getPartitionStorePrefix() { + // HUGEGRAPH/TASK/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(PARTITION_STORE).append(DELIMITER) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getPartitionRaftKey(int partId) { + // HUGEGRAPH/TASK/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(PARTITION_RAFT).append(DELIMITER) + .append(partId) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getPartitionRaftPrefix() { + // HUGEGRAPH/TASK/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(PARTITION_RAFT).append(DELIMITER) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getDeletedFileKey(String filePath) { + // HUGEGRAPH/TASK/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(DELETED_FILE).append(DELIMITER) + .append(filePath) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getDeletedFilePrefix() { + // HUGEGRAPH/TASK/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(DELETED_FILE).append(DELIMITER) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getCidKey(String name) { + // HUGEGRAPH/CID/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(CID_PREFIX).append(DELIMITER) + .append(name) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getCidSlotKeyPrefix(String name) { + // HUGEGRAPH/CID_SLOT/ + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(CID_SLOT_PREFIX).append(DELIMITER) + .append(name).append(DELIMITER) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] getGraphIDKey(String graph) { + // HUGEGRAPH/Graph/{graph} + String key = StringBuilderHelper.get() + .append(HUGEGRAPH).append(DELIMITER) + .append(GRAPH_ID_PREFIX).append(DELIMITER) + .append(graph) + .toString(); + return key.getBytes(StandardCharsets.UTF_8); + } + + static class StringBuilderHelper { + private static final int DISCARD_LIMIT = 1024 << 3; // 8k + + private static final ThreadLocal holderThreadLocal = ThreadLocal + .withInitial(StringBuilderHolder::new); + + public static StringBuilder get() { + final StringBuilderHolder holder = holderThreadLocal.get(); + return holder.getStringBuilder(); + } + + public static void truncate() { + final StringBuilderHolder holder = holderThreadLocal.get(); + holder.truncate(); + } + + private static class StringBuilderHolder { + + private final StringBuilder buf = new StringBuilder(); + + private StringBuilder getStringBuilder() { + truncate(); + return buf; + } + + private void truncate() { + buf.setLength(0); + } + } + } +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Partition.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Partition.java new file mode 100644 index 000000000..b9401f383 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Partition.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta; + +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.PartitionEngine; + +import com.baidu.hugegraph.pd.grpc.Metapb; + +import lombok.Data; + +/** + * @projectName: hugegraph-store + * @package: com.baidu.hugegraph.store.partitions + * @className: Partition + * @author: tyzer + * @description: TODO + * @date: 2021/10/19 18:16 + * @version: 1.0 + */ +@Data +public class Partition implements Cloneable { + private int id; // region id + private String graphName; + // Region key range [startKey, endKey) + private long startKey; + private long endKey; + private long version; + // shardlist版本，shardlist每次改变加1 + // private long confVer; + + private Metapb.PartitionState workState; + // private PartitionRole role; + // private List shardsList; + // exclusive + + public Partition() { + workState = Metapb.PartitionState.PState_Normal; + } + + public Partition(Metapb.Partition protoObj) { + id = protoObj.getId(); + graphName = protoObj.getGraphName(); + startKey = protoObj.getStartKey(); + endKey = protoObj.getEndKey(); + // shardsList = protoObj.getShardsList(); + workState = protoObj.getState(); + version = protoObj.getVersion(); + // confVer = protoObj.getConfVer(); + if (workState == Metapb.PartitionState.UNRECOGNIZED || + workState == Metapb.PartitionState.PState_None) { + workState = Metapb.PartitionState.PState_Normal; + } + } + + public boolean isLeader() { + PartitionEngine engine = HgStoreEngine.getInstance().getPartitionEngine(id); + return engine != null && engine.isLeader(); + } + + public Metapb.Partition getProtoObj() { + return Metapb.Partition.newBuilder() + .setId(id) + .setVersion(version) + // .setConfVer(confVer) + .setGraphName(graphName) + .setStartKey(startKey) + .setEndKey(endKey) + .setState(workState) + // .addAllShards(shardsList) + .build(); + } + + @Override + public Partition clone() { + Partition obj = null; + try { + obj = (Partition) super.clone(); + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + } + return obj; + } + + @Override + public String toString() { + return getProtoObj().toString(); + } +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/PartitionManager.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionManager.java similarity index 77% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/PartitionManager.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionManager.java index d66d707aa..a8e3452cb 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/PartitionManager.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionManager.java @@ -1,21 +1,21 @@ -package com.baidu.hugegraph.store.meta; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.alipay.sofa.jraft.core.ElectionPriority; -import com.baidu.hugegraph.pd.common.PDException; -import com.baidu.hugegraph.pd.grpc.MetaTask; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.business.BusinessHandlerImpl; -import com.baidu.hugegraph.store.cmd.UpdatePartitionRequest; -import com.baidu.hugegraph.store.cmd.UpdatePartitionResponse; -import com.baidu.hugegraph.store.meta.base.GlobalMetaStore; -import com.baidu.hugegraph.store.options.HgStoreEngineOptions; -import com.baidu.hugegraph.store.options.MetadataOptions; -import com.baidu.hugegraph.store.pd.PdProvider; -import com.baidu.hugegraph.store.util.PartitionMetaStoreWrapper; -import com.baidu.hugegraph.util.Log; -import lombok.extern.slf4j.Slf4j; -import org.slf4j.Logger; +package org.apache.hugegraph.store.meta; import java.io.File; import java.nio.file.Paths; @@ -32,6 +32,25 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.business.BusinessHandlerImpl; +import org.apache.hugegraph.store.cmd.UpdatePartitionRequest; +import org.apache.hugegraph.store.cmd.UpdatePartitionResponse; +import org.apache.hugegraph.store.meta.base.GlobalMetaStore; +import org.apache.hugegraph.store.options.HgStoreEngineOptions; +import org.apache.hugegraph.store.options.MetadataOptions; +import org.apache.hugegraph.store.pd.PdProvider; +import org.apache.hugegraph.store.util.PartitionMetaStoreWrapper; +import org.slf4j.Logger; + +import com.alipay.sofa.jraft.core.ElectionPriority; +import com.baidu.hugegraph.pd.common.PDException; +import com.baidu.hugegraph.pd.grpc.MetaTask; +import com.baidu.hugegraph.pd.grpc.Metapb; +import com.baidu.hugegraph.util.Log; + +import lombok.extern.slf4j.Slf4j; + /** * Partition对象管理策略，每次修改需要克隆一份，并且版本号递增 @@ -39,24 +58,24 @@ import java.util.function.Consumer; @Slf4j public class PartitionManager extends GlobalMetaStore { private static final Logger LOG = Log.logger(PartitionManager.class); - private PdProvider pdProvider; - private GraphManager graphManager; - private StoreMetadata storeMetadata; - private DeletedFileManager deletedFileManager; - private boolean useRaft; + private final PdProvider pdProvider; + private final GraphManager graphManager; + private final StoreMetadata storeMetadata; + private final DeletedFileManager deletedFileManager; + private final boolean useRaft; private final HgStoreEngineOptions options; - private List partitionChangedListeners; + private final List partitionChangedListeners; // 读写锁对象 - private ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); - private PartitionMetaStoreWrapper wrapper = new PartitionMetaStoreWrapper(); + private final PartitionMetaStoreWrapper wrapper = new PartitionMetaStoreWrapper(); // 记录本机所有的分区信息，与rocksdb存储保持一致 private Map> partitions; public PartitionManager(PdProvider pdProvider, HgStoreEngineOptions options) { - super(new MetadataOptions(){{ + super(new MetadataOptions() {{ setDataPath(options.getDataPath()); setRaftPath(options.getRaftPath()); }}); @@ -77,36 +96,42 @@ public class PartitionManager extends GlobalMetaStore { deletedFileManager.load(); } - public void loadPartition(){ + public void loadPartition() { loadPartitions(); } - public DeletedFileManager getDeletedFileManager() { return deletedFileManager; } + public DeletedFileManager getDeletedFileManager() { + return deletedFileManager; + } - public PdProvider getPdProvider() { return pdProvider; } + public PdProvider getPdProvider() { + return pdProvider; + } - public StoreMetadata getStoreMetadata(){ return storeMetadata;} + public StoreMetadata getStoreMetadata() { + return storeMetadata; + } - public void addPartitionChangedListener(PartitionChangedListener listener){ + public void addPartitionChangedListener(PartitionChangedListener listener) { partitionChangedListeners.add(listener); } /** * 判断存储路径为分区id或者分区id_开头 - * @param detections dir list + * + * @param detections dir list * @param partitionId partition id * @param checkLogDir : 是否包含子目录 log ( raft snapshot 和log 分离，需要进一步检查) * @return true if contains partition id, otherwise false - * */ - private Boolean checkPathContains(File[] detections, int partitionId, boolean checkLogDir){ + private Boolean checkPathContains(File[] detections, int partitionId, boolean checkLogDir) { String partitionDirectory = String.format("%05d", partitionId); - for (int x=0; x < detections.length; x++){ + for (int x = 0; x < detections.length; x++) { // 一定是以分区id命名的文件夹下 - if (detections[x].isDirectory()){ + if (detections[x].isDirectory()) { String tmp = detections[x].getName(); - if (tmp.equals(partitionDirectory) || tmp.startsWith(partitionDirectory + "_")){ + if (tmp.equals(partitionDirectory) || tmp.startsWith(partitionDirectory + "_")) { if (checkLogDir) { String logDir = detections[x].getAbsolutePath() + "/log"; if (new File(logDir).exists()) { @@ -126,7 +151,7 @@ public class PartitionManager extends GlobalMetaStore { * 根据约定db数据在dataPath/db/分区id目录，raft数据在dataPath/raft/分区id目录 * 检测分区存储文件夹是否存在 */ - private Boolean resetPartitionPath(int partitionId){ + private Boolean resetPartitionPath(int partitionId) { List dataPaths = Arrays.asList(this.options.getDataPath().split(",")); List raftPaths = Arrays.asList(this.options.getRaftPath().split(",")); @@ -134,19 +159,21 @@ public class PartitionManager extends GlobalMetaStore { boolean isRaftOk = false; // 检查 db 目录 - for (int i=0; i < dataPaths.size(); i++){ + for (int i = 0; i < dataPaths.size(); i++) { String dbPath = Paths.get(dataPaths.get(i), - HgStoreEngineOptions.DB_Path_Prefix).toAbsolutePath().toString(); + HgStoreEngineOptions.DB_Path_Prefix).toAbsolutePath() + .toString(); File dbFile = new File(dbPath); if (dbFile.exists()) { File[] dbFiles = dbFile.listFiles(); - if (this.checkPathContains(dbFiles, partitionId, false)){ + if (this.checkPathContains(dbFiles, partitionId, false)) { Metapb.PartitionStore location = storeMetadata.getPartitionStore(partitionId); - if (!location.getStoreLocation().equals(dataPaths.get(i))){ + if (!location.getStoreLocation().equals(dataPaths.get(i))) { Metapb.PartitionStore newLocation = location.toBuilder() - .setStoreLocation(dataPaths.get(i)) - .build(); + .setStoreLocation( + dataPaths.get(i)) + .build(); storeMetadata.savePartitionStore(newLocation); } isDataOk = true; @@ -156,22 +183,27 @@ public class PartitionManager extends GlobalMetaStore { } // 检查 raft目录 - for (int i = 0 ; i < raftPaths.size(); i ++){ + for (int i = 0; i < raftPaths.size(); i++) { String raftPath = Paths.get(raftPaths.get(i), - HgStoreEngineOptions.Raft_Path_Prefix).toAbsolutePath().toString(); + HgStoreEngineOptions.Raft_Path_Prefix).toAbsolutePath() + .toString(); File raftFile = new File(raftPath); if (raftFile.exists()) { File[] raftFiles = raftFile.listFiles(); - if (this.checkPathContains(raftFiles, partitionId, true)){ + if (this.checkPathContains(raftFiles, partitionId, true)) { Metapb.PartitionRaft location = storeMetadata.getPartitionRaft(partitionId); // 兼容版本升级 - if (location == null || !Objects.equals(location.getRaftLocation(), raftPaths.get(i))) { + if (location == null || + !Objects.equals(location.getRaftLocation(), raftPaths.get(i))) { Metapb.PartitionRaft newLocation = Metapb.PartitionRaft.newBuilder() - .setPartitionId(partitionId) - .setRaftLocation(raftPaths.get(i)) - .build(); + .setPartitionId( + partitionId) + .setRaftLocation( + raftPaths.get( + i)) + .build(); storeMetadata.savePartitionRaft(newLocation); } isRaftOk = true; @@ -186,25 +218,26 @@ public class PartitionManager extends GlobalMetaStore { /** * 从本地storage中读取分区 */ - private void loadPartitions(){ + private void loadPartitions() { byte[] key = MetadataKeyHelper.getPartitionPrefixKey(); long storeId = getStore().getId(); // 从data path中读取 partition // 记录有哪些分区 var partIds = new HashSet(); - for (String path: this.options.getDataPath().split(",")){ + for (String path : this.options.getDataPath().split(",")) { File[] dirs = new File(path + "/" + HgStoreEngineOptions.DB_Path_Prefix).listFiles(); - if (dirs == null || dirs.length == 0){ + if (dirs == null) { continue; } - for (File f : dirs){ + for (File f : dirs) { if (f.isDirectory()) { try { partIds.add(Integer.parseInt(f.getName().split("_")[0])); - }catch (Exception e){ - log.error("find illegal dir {} in data path, error:{}", f.getName(), e.getMessage()); + } catch (Exception e) { + log.error("find illegal dir {} in data path, error:{}", f.getName(), + e.getMessage()); } } } @@ -213,12 +246,13 @@ public class PartitionManager extends GlobalMetaStore { // 一次按照分区读取 for (int partId : partIds) { if (!resetPartitionPath(partId)) { - log.error("partition " + partId + " Directory not exists,options " + this.options.getDataPath()); + log.error("partition " + partId + " Directory not exists,options " + + this.options.getDataPath()); continue; } for (var metaPart : wrapper.scan(partId, Metapb.Partition.parser(), key)) { - var graph = metaPart.getGraphName(); + var graph = metaPart.getGraphName(); var pdPartition = pdProvider.getPartitionByID(graph, metaPart.getId()); boolean isLegeal = false; @@ -226,7 +260,7 @@ public class PartitionManager extends GlobalMetaStore { if (pdPartition != null) { // 判断是否包含本store id - if (shards.stream().anyMatch(s -> s.getStoreId() == storeId)){ + if (shards.stream().anyMatch(s -> s.getStoreId() == storeId)) { isLegeal = true; } } @@ -239,24 +273,25 @@ public class PartitionManager extends GlobalMetaStore { Partition partition = new Partition(metaPart); partition.setWorkState(Metapb.PartitionState.PState_Normal); // 启动恢复工作状态 partitions.get(graph).put(partition.getId(), partition); - log.info("load partition : {} -{}", partition.getGraphName(), partition.getId()); - }else { + log.info("load partition : {} -{}", partition.getGraphName(), + partition.getId()); + } else { // 无效 // removePartitionFromLocalDb(graph, partId); // var businessHandler = HgStoreEngine.getInstance().getBusinessHandler(); // businessHandler.truncate(graph, partId); // businessHandler.dbCompaction(graph, partId); log.error("partition {}-{} is illegal. store id {} not in valid shard group:{}", - graph, partId, getStore().getId(), shards2Peers(shards)); + graph, partId, getStore().getId(), shards2Peers(shards)); System.exit(0); } } } } - public List loadPartitionsFromDb(int partitionId){ + public List loadPartitionsFromDb(int partitionId) { byte[] key = MetadataKeyHelper.getPartitionPrefixKey(); - return wrapper.scan(partitionId , Metapb.Partition.parser(), key); + return wrapper.scan(partitionId, Metapb.Partition.parser(), key); } /** @@ -268,14 +303,17 @@ public class PartitionManager extends GlobalMetaStore { writeLock.lock(); try { - List partListFrPD = pdProvider.getPartitionsByStore(storeMetadata.getStore().getId()); + List partListFrPD = + pdProvider.getPartitionsByStore(storeMetadata.getStore().getId()); Map> graphPtFrpd = new HashMap<>(); partListFrPD.forEach(partition -> { - if (!graphPtFrpd.containsKey(partition.getGraphName())) + if (!graphPtFrpd.containsKey(partition.getGraphName())) { graphPtFrpd.put(partition.getGraphName(), new HashMap<>()); - if (isLocalPartition(partition)) + } + if (isLocalPartition(partition)) { graphPtFrpd.get(partition.getGraphName()).put(partition.getId(), partition); + } }); // 遍历本地图，删除本地多余，追加新的。 @@ -294,7 +332,7 @@ public class PartitionManager extends GlobalMetaStore { } }); - if ( partitionsFrpd != null) { + if (partitionsFrpd != null) { partitionsFrpd.forEach((id, pt) -> { if (!v.containsKey(id)) { // 新增的分区 @@ -304,7 +342,7 @@ public class PartitionManager extends GlobalMetaStore { } }); partitions = graphPtFrpd; - }finally { + } finally { writeLock.unlock(); } } @@ -315,12 +353,12 @@ public class PartitionManager extends GlobalMetaStore { writeLock.lock(); try { storeMetadata.save(store); - }finally { + } finally { writeLock.unlock(); } } - public Partition changeState(Partition partition, Metapb.PartitionState state){ + public Partition changeState(Partition partition, Metapb.PartitionState state) { Lock writeLock = readWriteLock.writeLock(); writeLock.lock(); try { @@ -333,7 +371,7 @@ public class PartitionManager extends GlobalMetaStore { return partition; } - public Partition changeKeyRange(Partition partition, int startKey, int endKey){ + public Partition changeKeyRange(Partition partition, int startKey, int endKey) { Lock writeLock = readWriteLock.writeLock(); writeLock.lock(); try { @@ -347,12 +385,13 @@ public class PartitionManager extends GlobalMetaStore { return partition; } - public Partition updatePartition(Metapb.Partition partition, boolean updateRange){ + public Partition updatePartition(Metapb.Partition partition, boolean updateRange) { return updatePartition(new Partition(partition), updateRange); } /** * 增加partition对象 + * * @param partition * @return */ @@ -373,19 +412,21 @@ public class PartitionManager extends GlobalMetaStore { try { Partition partition = findPartition(req.getGraphName(), req.getPartitionId()); if (req.getStartKey() >= 0 && req.getEndKey() > 0 - && partition.getStartKey() != req.getStartKey() && partition.getEndKey() != req.getEndKey()){ + && partition.getStartKey() != req.getStartKey() && + partition.getEndKey() != req.getEndKey()) { changeKeyRange(partition, req.getStartKey(), req.getEndKey()); } - if (req.getWorkState() != null){ + if (req.getWorkState() != null) { changeState(partition, req.getWorkState()); } - }finally { + } finally { writeLock.unlock(); } } /** * 强制更新 partition，不校验 version + * * @param partition * @return */ @@ -410,8 +451,9 @@ public class PartitionManager extends GlobalMetaStore { */ public Partition findPartition(String graph, Integer partId) { Partition partition = null; - if (partitions.containsKey(graph)) + if (partitions.containsKey(graph)) { partition = partitions.get(graph).get(partId); + } if (partition == null) { partition = pdProvider.getPartitionByID(graph, partId); @@ -427,11 +469,12 @@ public class PartitionManager extends GlobalMetaStore { writeLock.unlock(); } } else { - LOG.error("Partition {}-{} does not belong to local store! store id{} \n {}", graph, partId, - storeMetadata.getStore().getId(), partition.getProtoObj()); + LOG.error("Partition {}-{} does not belong to local store! store id{} \n {}", + graph, partId, + storeMetadata.getStore().getId(), partition.getProtoObj()); return null; } - }else { + } else { LOG.error("Partition {}-{} is not Found! ", graph, partId); return null; } @@ -439,14 +482,14 @@ public class PartitionManager extends GlobalMetaStore { return partitions.get(graph).get(partId); } - public int getPartitionIdByCode(String graph, int code){ + public int getPartitionIdByCode(String graph, int code) { return pdProvider.getPartitionByCode(graph, code).getId(); } /** * 从pd获取拉取分区信息，并和本地的分区信息进行合并。leader和shardList取自本地 */ - public Partition getPartitionFromPD(String graph, int partId){ + public Partition getPartitionFromPD(String graph, int partId) { pdProvider.invalidPartitionCache(graph, partId); Partition partition = pdProvider.getPartitionByID(graph, partId); Lock writeLock = readWriteLock.writeLock(); @@ -454,7 +497,7 @@ public class PartitionManager extends GlobalMetaStore { try { if (partitions.containsKey(graph)) { Partition local = partitions.get(graph).get(partId); - if ( local != null){ + if (local != null) { //更新本地的key范围，保证pd和本地分区信息的一致性 local.setStartKey(partition.getStartKey()); local.setEndKey(partition.getEndKey()); @@ -467,9 +510,11 @@ public class PartitionManager extends GlobalMetaStore { } return partition; } + /** * 是否是本地的分区 * 对于批处理入库，只有leader才属于本地 + * * @param partition * @return */ @@ -477,7 +522,7 @@ public class PartitionManager extends GlobalMetaStore { boolean isLocal = false; var shardGroup = getShardGroup(partition.getId()); if (shardGroup != null) { - for (Shard shard : shardGroup.getShards()){ + for (Shard shard : shardGroup.getShards()) { if (shard.getStoreId() == storeMetadata.getStore().getId()) { isLocal = true; break; @@ -490,6 +535,7 @@ public class PartitionManager extends GlobalMetaStore { /** * 是否是本地的分区 * 对于批处理入库，只有leader才属于本地 + * * @return */ public boolean isLocalPartition(int partId) { @@ -509,10 +555,10 @@ public class PartitionManager extends GlobalMetaStore { /** * 保存partition 信息 * - * @param partition partition + * @param partition partition * @param changeLeader is change leader - * @param changeRange update start and end key if yes. - * using key range in local if no and partition key exists + * @param changeRange update start and end key if yes. + * using key range in local if no and partition key exists */ private void savePartition(Partition partition, Boolean changeLeader, Boolean changeRange) { @@ -520,9 +566,9 @@ public class PartitionManager extends GlobalMetaStore { Integer partId = partition.getId(); byte[] key = MetadataKeyHelper.getPartitionKey(graphName, partId); - if (! changeRange) { + if (!changeRange) { var local = wrapper.get(partId, key, Metapb.Partition.parser()); - if (local != null){ + if (local != null) { partition.setStartKey(local.getStartKey()); partition.setEndKey(local.getEndKey()); } @@ -550,38 +596,42 @@ public class PartitionManager extends GlobalMetaStore { /** * 更新shard group到db, 同时更新shardGroups对象 + * * @param shardGroup */ - public void updateShardGroup(ShardGroup shardGroup){ + public void updateShardGroup(ShardGroup shardGroup) { Lock writeLock = readWriteLock.writeLock(); writeLock.lock(); wrapper.put(shardGroup.getId(), - MetadataKeyHelper.getShardGroupKey(shardGroup.getId()), - shardGroup.getProtoObj().toByteArray()); + MetadataKeyHelper.getShardGroupKey(shardGroup.getId()), + shardGroup.getProtoObj().toByteArray()); writeLock.unlock(); } /** * 查找 partition id对应的shard group。 * 依次从 raft node/local db/ pd 读取。 + * * @param partitionId * @return */ - public ShardGroup getShardGroup(int partitionId){ + public ShardGroup getShardGroup(int partitionId) { var partitionEngine = HgStoreEngine.getInstance().getPartitionEngine(partitionId); - if (partitionEngine != null){ + if (partitionEngine != null) { return partitionEngine.getShardGroup(); } - Metapb.ShardGroup shardGroup = wrapper.get(partitionId, MetadataKeyHelper.getShardGroupKey(partitionId), - Metapb.ShardGroup.parser()); + Metapb.ShardGroup shardGroup = + wrapper.get(partitionId, MetadataKeyHelper.getShardGroupKey(partitionId), + Metapb.ShardGroup.parser()); if (shardGroup == null) { shardGroup = pdProvider.getShardGroup(partitionId); if (shardGroup != null) { // local not found, write back to db from pd - wrapper.put(partitionId, MetadataKeyHelper.getShardGroupKey(partitionId), shardGroup.toByteArray()); + wrapper.put(partitionId, MetadataKeyHelper.getShardGroupKey(partitionId), + shardGroup.toByteArray()); } else { log.error("get shard group {} from pd failed", partitionId); } @@ -605,7 +655,7 @@ public class PartitionManager extends GlobalMetaStore { return null; } - private void removePartitionFromLocalDb(String graphName, Integer partId){ + private void removePartitionFromLocalDb(String graphName, Integer partId) { byte[] key = MetadataKeyHelper.getPartitionKey(graphName, partId); // delete(key); wrapper.delete(partId, key); @@ -621,7 +671,9 @@ public class PartitionManager extends GlobalMetaStore { // 获取本地Store信息 - public Store getStore(){ return storeMetadata.getStore(); } + public Store getStore() { + return storeMetadata.getStore(); + } public Store getStore(Long storeId) { return pdProvider.getStoreByID(storeId); @@ -631,24 +683,25 @@ public class PartitionManager extends GlobalMetaStore { return partitions; } - public Map getPartitions(int partitionId){ + public Map getPartitions(int partitionId) { Map result = new HashMap<>(); - this.partitions.forEach((k,v) -> { - v.forEach((k1, v1) -> { - if (k1 == partitionId){ - result.put(k, v1); - } - }); - } + this.partitions.forEach((k, v) -> { + v.forEach((k1, v1) -> { + if (k1 == partitionId) { + result.put(k, v1); + } + }); + } ); return result; } public Partition getPartition(String graphName, int partitionId) { - return this.partitions.getOrDefault(graphName, new HashMap<>()).getOrDefault(partitionId, null); + return this.partitions.getOrDefault(graphName, new HashMap<>()) + .getOrDefault(partitionId, null); } - public List getPartitionList(int partitionId){ + public List getPartitionList(int partitionId) { List pts = new ArrayList<>(); getPartitions(partitionId).forEach((k, v) -> { pts.add(findPartition(k, v.getId())); @@ -656,7 +709,7 @@ public class PartitionManager extends GlobalMetaStore { return pts; } - public boolean hasPartition(String graphName, int partitionId){ + public boolean hasPartition(String graphName, int partitionId) { return this.partitions.getOrDefault(graphName, new HashMap<>()).containsKey(partitionId); } @@ -668,17 +721,20 @@ public class PartitionManager extends GlobalMetaStore { */ public List getLeaderPartitionIds(String graph) { List ids = new ArrayList<>(); - if (partitions.containsKey(graph)) + if (partitions.containsKey(graph)) { partitions.get(graph).forEach((k, v) -> { - if (!useRaft || v.isLeader()) + if (!useRaft || v.isLeader()) { ids.add(k); + } }); + } return ids; } /** * 生成分区peer字符串，包含优先级信息 * + * * @param shardGroup * @return */ @@ -719,9 +775,10 @@ public class PartitionManager extends GlobalMetaStore { return storeMetadata.getStore().getId() == store.getId(); } - public PartitionRole getLocalRoleFromShard(Partition partition){ + public PartitionRole getLocalRoleFromShard(Partition partition) { return partition.isLeader() ? PartitionRole.LEADER : PartitionRole.FOLLOWER; } + /** * 修改分区角色 */ @@ -736,7 +793,7 @@ public class PartitionManager extends GlobalMetaStore { savePartition(partition, true); } return partition; - }finally { + } finally { writeLock.unlock(); } } @@ -763,7 +820,8 @@ public class PartitionManager extends GlobalMetaStore { /** * 拆分partition对象 */ - public List updatePartitionToPD(List partitions) throws PDException { + public List updatePartitionToPD(List partitions) throws + PDException { // 更新本地分区信息，以及cache信息 return pdProvider.updatePartition(partitions); } @@ -792,21 +850,24 @@ public class PartitionManager extends GlobalMetaStore { }); return result[0]; } + /** * raft存储路径 + * * @param groupId - * @return location/raft/groupId/ + * @return location/raft/groupId/ */ public String getRaftDataPath(int groupId) { String location = storeMetadata.getPartitionRaftLocation(groupId); location = Paths.get(location, - HgStoreEngineOptions.Raft_Path_Prefix, - String.format("%05d", groupId)).toAbsolutePath().toString(); + HgStoreEngineOptions.Raft_Path_Prefix, + String.format("%05d", groupId)).toAbsolutePath().toString(); return location; } /** * raft snapshot 的路径，要和 db同一个盘上，便于hard link + * * @param groupId raft group id * @return location/snapshot/0000x/ */ @@ -821,7 +882,8 @@ public class PartitionManager extends GlobalMetaStore { /** * db存储路径 - * @return location/db + * + * @return location/db */ public String getDbDataPath(int partitionId, String dbName) { String location = storeMetadata.getPartitionStoreLocation(partitionId, dbName); @@ -831,7 +893,7 @@ public class PartitionManager extends GlobalMetaStore { } - public void reportTask(MetaTask.Task task){ + public void reportTask(MetaTask.Task task) { try { pdProvider.reportTask(task); } catch (Exception e) { @@ -842,15 +904,16 @@ public class PartitionManager extends GlobalMetaStore { /** * Partition对象被修改消息 */ - public interface PartitionChangedListener{ + public interface PartitionChangedListener { void onChanged(Partition partition); + UpdatePartitionResponse rangeOrStateChanged(UpdatePartitionRequest request); } /** * 修改partion的state状态 */ - public List changePartitionToOnLine(List partitions){ + public List changePartitionToOnLine(List partitions) { List newPartitions = new ArrayList<>(); partitions.forEach(e -> { newPartitions.add(e.toBuilder().setState(Metapb.PartitionState.PState_Normal).build()); @@ -858,7 +921,7 @@ public class PartitionManager extends GlobalMetaStore { return newPartitions; } - public PartitionMetaStoreWrapper getWrapper(){ + public PartitionMetaStoreWrapper getWrapper() { return wrapper; } diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/PartitionRole.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionRole.java similarity index 52% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/PartitionRole.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionRole.java index 7f8f124d7..a2ab70bbb 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/PartitionRole.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionRole.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.meta; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta; import com.baidu.hugegraph.pd.grpc.Metapb; @@ -12,10 +29,13 @@ import com.baidu.hugegraph.pd.grpc.Metapb; * @version: 1.0 */ public enum PartitionRole { - UNKNOWN(0, "unknown"), LEADER(1, "leader"), FOLLOWER(2, "follower"), - LEARNER(3, "learner"), CANDIDATE(4, "candidate"); - private int role; - private String name; + UNKNOWN(0, "unknown"), + LEADER(1, "leader"), + FOLLOWER(2, "follower"), + LEARNER(3, "learner"), + CANDIDATE(4, "candidate"); + private final int role; + private final String name; PartitionRole(int role, String name) { this.role = role; @@ -27,11 +47,13 @@ public enum PartitionRole { return this.ordinal() + "_" + this.name; } - public String getName(){ return this.name; } + public String getName() { + return this.name; + } - public Metapb.ShardRole toShardRole(){ + public Metapb.ShardRole toShardRole() { Metapb.ShardRole shardRole = Metapb.ShardRole.None; - switch (this){ + switch (this) { case LEADER: shardRole = Metapb.ShardRole.Leader; break; @@ -45,9 +67,9 @@ public enum PartitionRole { return shardRole; } - public static PartitionRole fromShardRole(Metapb.ShardRole shard){ + public static PartitionRole fromShardRole(Metapb.ShardRole shard) { PartitionRole role = PartitionRole.FOLLOWER; - switch (shard){ + switch (shard) { case Leader: role = PartitionRole.LEADER; break; diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionStats.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionStats.java new file mode 100644 index 000000000..f7922359c --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionStats.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta; + +import java.util.ArrayList; +import java.util.List; + +import com.baidu.hugegraph.pd.grpc.Metapb; + +import lombok.Data; + +@Data +public class PartitionStats { + // 分区ID + private int id; + private String namespace; + private String graphName; + // 分区leader所在shard + Metapb.Shard leader; + // 分区离线的shard + List offlineShards = new ArrayList<>(); + long committedIndex; + long leaderTerm; + long approximateSize; + long approximateKeys; + + public PartitionStats addOfflineShard(Metapb.Shard shard) { + offlineShards.add(shard); + return this; + } + + public Metapb.PartitionStats getProtoObj() { + return Metapb.PartitionStats.newBuilder() + .setId(id) + .addGraphName(graphName) + .setLeader(leader) + .addAllShard(offlineShards) + .setApproximateKeys(approximateKeys) + .setApproximateSize(approximateSize) + .setLeaderTerm(leaderTerm) + .build(); + } +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Shard.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Shard.java new file mode 100644 index 000000000..48248bd24 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Shard.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta; + +import com.baidu.hugegraph.pd.grpc.Metapb; + +import lombok.Data; + +/** + * 一个分片 + */ +@Data +public class Shard { + private long storeId; + private Metapb.ShardRole role; + + public Metapb.Shard toMetaPbShard() { + return Metapb.Shard.newBuilder().setStoreId(storeId).setRole(role).build(); + } + + public static Shard fromMetaPbShard(Metapb.Shard shard) { + Shard s = new Shard(); + s.setRole(shard.getRole()); + s.setStoreId(shard.getStoreId()); + return s; + } +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/ShardGroup.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/ShardGroup.java similarity index 55% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/ShardGroup.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/meta/ShardGroup.java index 4756e3e64..c61f4fd8c 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/ShardGroup.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/ShardGroup.java @@ -1,13 +1,31 @@ -package com.baidu.hugegraph.store.meta; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.pd.grpc.Metapb; -import lombok.Data; +package org.apache.hugegraph.store.meta; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.stream.Collectors; +import com.baidu.hugegraph.pd.grpc.Metapb; + +import lombok.Data; + /** * 分片副本组 */ @@ -24,14 +42,14 @@ public class ShardGroup { */ private long confVersion; - private List shards = new CopyOnWriteArrayList<>(); + private final List shards = new CopyOnWriteArrayList<>(); - public ShardGroup addShard(Shard shard){ + public ShardGroup addShard(Shard shard) { this.shards.add(shard); return this; } - public static ShardGroup from(Metapb.ShardGroup meta){ + public static ShardGroup from(Metapb.ShardGroup meta) { if (meta == null) { return null; } @@ -39,18 +57,21 @@ public class ShardGroup { shardGroup.setId(meta.getId()); shardGroup.setVersion(meta.getVersion()); shardGroup.setConfVersion(meta.getConfVer()); - shardGroup.setShards(meta.getShardsList().stream().map(Shard::fromMetaPbShard).collect(Collectors.toList())); + shardGroup.setShards(meta.getShardsList().stream().map(Shard::fromMetaPbShard) + .collect(Collectors.toList())); return shardGroup; } - public synchronized ShardGroup changeLeader(long storeId){ + public synchronized ShardGroup changeLeader(long storeId) { shards.forEach(shard -> { - shard.setRole(shard.getStoreId() == storeId ? Metapb.ShardRole.Leader : Metapb.ShardRole.Follower); + shard.setRole(shard.getStoreId() == storeId ? Metapb.ShardRole.Leader : + Metapb.ShardRole.Follower); }); return this; } - public synchronized ShardGroup changeShardList(List peerIds, List learners, long leaderId) { + public synchronized ShardGroup changeShardList(List peerIds, List learners, + long leaderId) { if (!peerIds.isEmpty()) { shards.clear(); peerIds.forEach(id -> { @@ -71,7 +92,7 @@ public class ShardGroup { return this; } - public synchronized List getMetaPbShard(){ + public synchronized List getMetaPbShard() { List shardList = new ArrayList<>(); shards.forEach(shard -> { shardList.add(shard.toMetaPbShard()); @@ -80,20 +101,20 @@ public class ShardGroup { } @Override - public String toString(){ + public String toString() { StringBuilder builder = new StringBuilder(); - shards.forEach(e->{ + shards.forEach(e -> { builder.append(String.format("{ id:%s,role:%s },", e.getStoreId(), e.getRole())); }); return builder.length() > 0 ? builder.substring(0, builder.length() - 1) : ""; } - public Metapb.ShardGroup getProtoObj(){ + public Metapb.ShardGroup getProtoObj() { return Metapb.ShardGroup.newBuilder() - .setId(this.id) - .setVersion(this.version) - .setConfVer(this.confVersion) - .addAllShards(getMetaPbShard()) - .build(); + .setId(this.id) + .setVersion(this.version) + .setConfVer(this.confVersion) + .addAllShards(getMetaPbShard()) + .build(); } } diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Store.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Store.java new file mode 100644 index 000000000..0eb7b76a8 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/Store.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta; + +import java.util.Map; + +import org.apache.hugegraph.store.util.Asserts; +import org.apache.hugegraph.store.util.Version; + +import com.baidu.hugegraph.pd.grpc.Metapb; + +import lombok.Data; + +/** + * @projectName: hugegraph-store + * @package: com.baidu.hugegraph.store.core + * @className: HgStoreNode + * @author: tyzer + * @description: TODO + * @date: 2021/10/19 19:44 + * @version: 1.0 + */ +@Data +public class Store { + private long id = 0; + private String storeAddress; + private String pdAddress; + private String raftAddress; + private final String version; + private String deployPath; + private String dataPath; // 数据存储路径 + private int dataVersion; + private int partitionCount; + private int startTime; + private int usedSize; //rocksdb存储大小 + private int pdHeartbeatInterval; + private Metapb.StoreState state; + private Map labels; + private int cores; + + public Store() { + this.id = 0; + this.version = Version.getVersion(); + } + + public Store(int dataVersion) { + this.id = 0; + this.dataVersion = dataVersion; + this.version = Version.getVersion(); + } + + public Store(Metapb.Store protoObj) { + if (protoObj != null) { + this.id = protoObj.getId(); + this.raftAddress = protoObj.getRaftAddress(); + this.storeAddress = protoObj.getAddress(); + this.dataVersion = protoObj.getDataVersion(); + } else { + this.id = 0; + } + this.version = Version.getVersion(); + } + + + public Metapb.Store getProtoObj() { + Asserts.isNonNull(storeAddress); + Asserts.isNonNull(raftAddress); + Metapb.Store.Builder builder = Metapb.Store.newBuilder() + .setId(id).setVersion(version) + .setDataVersion(dataVersion) + .setAddress(storeAddress) + .setRaftAddress(raftAddress) + .setState(Metapb.StoreState.Up) + .setCores(cores) + .setDeployPath(deployPath) + .setDataPath(dataPath); + + if (labels != null) { + labels.forEach((k, v) -> { + builder.addLabels(Metapb.StoreLabel.newBuilder().setKey(k).setValue(v).build()); + }); + } + + return builder.build(); + + } + + public boolean checkState(Metapb.StoreState state) { + return this.state == state; + } +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/StoreMetadata.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/StoreMetadata.java similarity index 66% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/StoreMetadata.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/meta/StoreMetadata.java index aadf33d05..44934b745 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/StoreMetadata.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/StoreMetadata.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.meta; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta; import java.io.File; import java.nio.file.Paths; @@ -7,20 +24,21 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.hugegraph.store.meta.base.GlobalMetaStore; +import org.apache.hugegraph.store.options.MetadataOptions; +import org.apache.hugegraph.store.util.HgStoreException; + import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.meta.base.GlobalMetaStore; -import com.baidu.hugegraph.store.options.MetadataOptions; -import com.baidu.hugegraph.store.util.HgStoreException; import lombok.extern.slf4j.Slf4j; @Slf4j public class StoreMetadata extends GlobalMetaStore { - protected final static int Store_DataFmt_Version = 1; + protected final static int Store_DataFmt_Version = 1; private Store store = null; - private List dataLocations; + private final List dataLocations; - private List raftLocations; + private final List raftLocations; public List getDataLocations() { return dataLocations; @@ -30,7 +48,7 @@ public class StoreMetadata extends GlobalMetaStore { return raftLocations; } - public StoreMetadata(MetadataOptions options){ + public StoreMetadata(MetadataOptions options) { super(options); dataLocations = Arrays.asList(options.getDataPath().split(",")); raftLocations = Arrays.asList(options.getRaftPath().split(",")); @@ -38,7 +56,7 @@ public class StoreMetadata extends GlobalMetaStore { public Store load() { // 针对多目录存储的情况下，预先创建文件夹，方便pd端统计文件存储 - dataLocations.forEach(path ->{ + dataLocations.forEach(path -> { String strPath = Paths.get(path).toAbsolutePath().toString(); File dbFile = new File(strPath); if (!dbFile.exists()) { @@ -46,7 +64,7 @@ public class StoreMetadata extends GlobalMetaStore { } }); - raftLocations.forEach(path ->{ + raftLocations.forEach(path -> { String strPath = Paths.get(path).toAbsolutePath().toString(); File dbFile = new File(strPath); if (!dbFile.exists()) { @@ -59,19 +77,23 @@ public class StoreMetadata extends GlobalMetaStore { if (value != null) { try { Metapb.Store protoObj = Metapb.Store.parseFrom(value); - if (protoObj != null) + if (protoObj != null) { store = new Store(protoObj); + } } catch (Exception e) { throw new HgStoreException(HgStoreException.EC_FAIL, e); } } - if ( store == null ) - store = new Store(Store_DataFmt_Version); + if (store == null) { + store = new Store(Store_DataFmt_Version); + } checkDataFmtCompatible(); return store; } - public Store getStore() { return store; } + public Store getStore() { + return store; + } public void save(Store store) { byte[] key = MetadataKeyHelper.getStoreKey(); @@ -79,39 +101,42 @@ public class StoreMetadata extends GlobalMetaStore { this.store = store; } - public void checkDataFmtCompatible(){ - if ( store == null || store.getDataVersion() != Store_DataFmt_Version) + public void checkDataFmtCompatible() { + if (store == null || store.getDataVersion() != Store_DataFmt_Version) { throw new HgStoreException(HgStoreException.EC_DATAFMT_NOT_SUPPORTED, - String.format("Incompatible data format, data format version is %d, supported version is %d", - store.getDataVersion(), Store_DataFmt_Version)); + String.format( + "Incompatible data format, data format version is " + + "%d, supported version is %d", + store.getDataVersion(), Store_DataFmt_Version)); + } } - public Metapb.PartitionStore getPartitionStore(int partitionId){ + public Metapb.PartitionStore getPartitionStore(int partitionId) { byte[] key = MetadataKeyHelper.getPartitionStoreKey(partitionId); return get(Metapb.PartitionStore.parser(), key); } - public List getPartitionStores(){ + public List getPartitionStores() { byte[] key = MetadataKeyHelper.getPartitionStorePrefix(); return scan(Metapb.PartitionStore.parser(), key); } - public void savePartitionStore(Metapb.PartitionStore partitionStore){ + public void savePartitionStore(Metapb.PartitionStore partitionStore) { byte[] key = MetadataKeyHelper.getPartitionStoreKey(partitionStore.getPartitionId()); put(key, partitionStore.toByteArray()); } - public Metapb.PartitionRaft getPartitionRaft(int partitionId){ + public Metapb.PartitionRaft getPartitionRaft(int partitionId) { byte[] key = MetadataKeyHelper.getPartitionRaftKey(partitionId); return get(Metapb.PartitionRaft.parser(), key); } - public List getPartitionRafts(){ + public List getPartitionRafts() { byte[] key = MetadataKeyHelper.getPartitionRaftPrefix(); return scan(Metapb.PartitionRaft.parser(), key); } - public void savePartitionRaft(Metapb.PartitionRaft partitionRaft){ + public void savePartitionRaft(Metapb.PartitionRaft partitionRaft) { byte[] key = MetadataKeyHelper.getPartitionRaftKey(partitionRaft.getPartitionId()); put(key, partitionRaft.toByteArray()); } @@ -119,11 +144,13 @@ public class StoreMetadata extends GlobalMetaStore { private String getMinDataLocation() { Map counter = new HashMap<>(); dataLocations.forEach(l -> { - counter.put(l, new Integer(0)); + counter.put(l, Integer.valueOf(0)); }); getPartitionStores().forEach(ptStore -> { - if (counter.containsKey(ptStore.getStoreLocation())) - counter.put(ptStore.getStoreLocation(), counter.get(ptStore.getStoreLocation()) + 1); + if (counter.containsKey(ptStore.getStoreLocation())) { + counter.put(ptStore.getStoreLocation(), + counter.get(ptStore.getStoreLocation()) + 1); + } }); int min = Integer.MAX_VALUE; String location = ""; @@ -139,7 +166,7 @@ public class StoreMetadata extends GlobalMetaStore { private String getMinRaftLocation() { Map counter = new HashMap<>(); raftLocations.forEach(l -> { - counter.put(l, new Integer(0)); + counter.put(l, Integer.valueOf(0)); }); getPartitionRafts().forEach(ptRaft -> { @@ -161,13 +188,14 @@ public class StoreMetadata extends GlobalMetaStore { /** * 获取分区数据存储的位置，如果分布数据不存在，自动创建新的位置 + * * @param partitionId * @return */ public String getPartitionStoreLocation(int partitionId, String dbName) { Metapb.PartitionStore location = getPartitionStore(partitionId); - if (location == null){ - synchronized (this){ + if (location == null) { + synchronized (this) { location = getPartitionStore(partitionId); if (location == null) { // 查找分区数最少的存储 @@ -183,17 +211,17 @@ public class StoreMetadata extends GlobalMetaStore { return location.getStoreLocation(); } - public String getPartitionRaftLocation(int partitionId){ + public String getPartitionRaftLocation(int partitionId) { Metapb.PartitionRaft location = getPartitionRaft(partitionId); - if ( location == null){ - synchronized (this){ + if (location == null) { + synchronized (this) { location = getPartitionRaft(partitionId); - if ( location == null){ + if (location == null) { // 查找分区数最少的存储 location = Metapb.PartitionRaft.newBuilder() - .setPartitionId(partitionId) - .setRaftLocation(getMinRaftLocation()) - .build(); + .setPartitionId(partitionId) + .setRaftLocation(getMinRaftLocation()) + .build(); // TODO 选择分区数最小的路径 savePartitionRaft(location); } diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/TaskManager.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/TaskManager.java similarity index 62% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/TaskManager.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/meta/TaskManager.java index 39577c692..7e9545f1e 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/TaskManager.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/TaskManager.java @@ -1,16 +1,35 @@ -package com.baidu.hugegraph.store.meta; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.pd.grpc.MetaTask; -import com.baidu.hugegraph.store.meta.asynctask.AbstractAsyncTask; -import com.baidu.hugegraph.store.meta.asynctask.AsyncTask; -import com.baidu.hugegraph.store.meta.asynctask.AsyncTaskState; -import com.baidu.hugegraph.store.meta.base.DBSessionBuilder; -import com.baidu.hugegraph.store.meta.base.PartitionMetaStore; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.meta; import java.util.ArrayList; import java.util.List; +import org.apache.hugegraph.store.meta.asynctask.AbstractAsyncTask; +import org.apache.hugegraph.store.meta.asynctask.AsyncTask; +import org.apache.hugegraph.store.meta.asynctask.AsyncTaskState; +import org.apache.hugegraph.store.meta.base.DBSessionBuilder; +import org.apache.hugegraph.store.meta.base.PartitionMetaStore; + +import com.baidu.hugegraph.pd.grpc.MetaTask; + +import lombok.extern.slf4j.Slf4j; + @Slf4j public class TaskManager extends PartitionMetaStore { @@ -26,9 +45,8 @@ public class TaskManager extends PartitionMetaStore { public void updateTask(MetaTask.Task task) { if (task.getState().compareTo(MetaTask.TaskState.Task_Stop) < 0) { updateTask(task.getPartition().getId(), - task.getType().name(), task.getId(), task.toByteArray()); - } - else { + task.getType().name(), task.getId(), task.toByteArray()); + } else { deleteTask(task); } } @@ -52,7 +70,7 @@ public class TaskManager extends PartitionMetaStore { public void deleteTask(MetaTask.Task task) { byte[] key = MetadataKeyHelper.getTaskKey(task.getPartition().getId(), - task.getType().name(), task.getId()); + task.getType().name(), task.getId()); delete(key); } @@ -60,11 +78,11 @@ public class TaskManager extends PartitionMetaStore { deletePrefix(MetadataKeyHelper.getTaskPrefix(partId, type)); } - public boolean taskExists(int partId, String graphName, String taskTypeName){ + public boolean taskExists(int partId, String graphName, String taskTypeName) { return partitionTaskRepeat(partId, graphName, taskTypeName, 0); } - public boolean taskExists(MetaTask.Task task){ + public boolean taskExists(MetaTask.Task task) { return null != getOneTask(task); } @@ -75,17 +93,18 @@ public class TaskManager extends PartitionMetaStore { * graphName */ public boolean partitionTaskRepeat(int partId, String graphName, String taskTypeName) { - return partitionTaskRepeat(partId, graphName, taskTypeName, 1) ; + return partitionTaskRepeat(partId, graphName, taskTypeName, 1); } - private boolean partitionTaskRepeat(int partId, String graphName, String taskTypeName, int checkCount) { + private boolean partitionTaskRepeat(int partId, String graphName, String taskTypeName, + int checkCount) { byte[] key = MetadataKeyHelper.getTaskPrefix(partId, taskTypeName); List tasks = scan(MetaTask.Task.parser(), key); if (tasks.size() > 1) { int graphCount = 0; for (MetaTask.Task task : tasks) { if (task.getPartition().getGraphName().equals(graphName) && - task.getState().getNumber() < MetaTask.TaskState.Task_Stop_VALUE) { + task.getState().getNumber() < MetaTask.TaskState.Task_Stop_VALUE) { graphCount++; } } @@ -95,12 +114,12 @@ public class TaskManager extends PartitionMetaStore { return false; } - public void putAsyncTask(AsyncTask task){ - put(MetadataKeyHelper.getAsyncTaskKey(task.getPartitionId(), - task.getGraphName(), task.getId()), task.toBytes()); + public void putAsyncTask(AsyncTask task) { + put(MetadataKeyHelper.getAsyncTaskKey(task.getPartitionId(), + task.getGraphName(), task.getId()), task.toBytes()); } - public AsyncTask getOneAsyncTask(int partId, String graphName, String taskId){ + public AsyncTask getOneAsyncTask(int partId, String graphName, String taskId) { var bytes = get(MetadataKeyHelper.getAsyncTaskKey(partId, graphName, taskId)); if (bytes != null) { return AbstractAsyncTask.fromBytes(bytes); @@ -108,7 +127,8 @@ public class TaskManager extends PartitionMetaStore { return null; } - public void updateAsyncTaskState(int partId, String graphName, String taskId, AsyncTaskState state){ + public void updateAsyncTaskState(int partId, String graphName, String taskId, + AsyncTaskState state) { var task = getOneAsyncTask(partId, graphName, taskId); if (task != null) { task.setState(state); @@ -116,9 +136,9 @@ public class TaskManager extends PartitionMetaStore { } } - public List scanAsyncTasks(int partitionId, String graphName){ + public List scanAsyncTasks(int partitionId, String graphName) { var list = new ArrayList(); - for (var task : scan(MetadataKeyHelper.getAsyncTaskPrefix(partitionId, graphName))){ + for (var task : scan(MetadataKeyHelper.getAsyncTaskPrefix(partitionId, graphName))) { list.add(AbstractAsyncTask.fromBytes(task.value)); } return list; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/asynctask/AbstractAsyncTask.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/AbstractAsyncTask.java similarity index 68% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/asynctask/AbstractAsyncTask.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/AbstractAsyncTask.java index 0560fc35d..fc8b161ca 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/asynctask/AbstractAsyncTask.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/AbstractAsyncTask.java @@ -1,7 +1,21 @@ -package com.baidu.hugegraph.store.meta.asynctask; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.scan.safesdk.SafeObjectInputStream; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.meta.asynctask; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -10,21 +24,26 @@ import java.io.ObjectOutputStream; import java.io.Serializable; import java.util.UUID; +import com.baidu.scan.safesdk.SafeObjectInputStream; + +import lombok.extern.slf4j.Slf4j; + @Slf4j public abstract class AbstractAsyncTask implements AsyncTask, Serializable { - private String id; - private int partitionId; + private final String id; + private final int partitionId; - private String graphName; + private final String graphName; private AsyncTaskState state; - private String type; + private final String type; /** * 任务额外需要的参数 */ - private Object extra; + private final Object extra; - public AbstractAsyncTask(int partitionId, String graphName, AsyncTaskState state, Object extra) { + public AbstractAsyncTask(int partitionId, String graphName, AsyncTaskState state, + Object extra) { this.id = getNextId(); this.partitionId = partitionId; this.graphName = graphName; @@ -117,12 +136,12 @@ public abstract class AbstractAsyncTask implements AsyncTask, Serializable { @Override public String toString() { return "AbstractAsyncTask{" + - "id='" + id + '\'' + - ", partitionId=" + partitionId + - ", graphName='" + graphName + '\'' + - ", state=" + state + - ", type='" + type + '\'' + - ", extra=" + getExtra() + - '}'; + "id='" + id + '\'' + + ", partitionId=" + partitionId + + ", graphName='" + graphName + '\'' + + ", state=" + state + + ", type='" + type + '\'' + + ", extra=" + getExtra() + + '}'; } } diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/AsyncTask.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/AsyncTask.java new file mode 100644 index 000000000..d758899c9 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/AsyncTask.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta.asynctask; + +public interface AsyncTask { + + /** + * 需要检查异步任务时候，检查当前的状态，根据状态去做对应的处理 + */ + void handleTask(); + + /** + * 任务ID + */ + String getId(); + + /** + * 针对哪个图的 + */ + String getGraphName(); + + /** + * 针对哪个分区的 + */ + int getPartitionId(); + + /** + * 用来进行序列化 + * + * @return + */ + byte[] toBytes(); + + /** + * 设置执行状态 + * + * @param newState + */ + void setState(AsyncTaskState newState); +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/AsyncTaskState.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/AsyncTaskState.java new file mode 100644 index 000000000..c89497487 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/AsyncTaskState.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta.asynctask; + +public enum AsyncTaskState { + START, + SUCCESS, + FAILED +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/CleanTask.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/CleanTask.java new file mode 100644 index 000000000..fc9737e0a --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/asynctask/CleanTask.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta.asynctask; + +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.cmd.CleanDataRequest; + +import com.baidu.hugegraph.pd.grpc.pulse.CleanType; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CleanTask extends AbstractAsyncTask { + + public CleanTask(int partitionId, String graphName, AsyncTaskState state, Object attach) { + super(partitionId, graphName, state, attach); + } + + @Override + public String getType() { + return "CLEAN_TYPE"; + } + + @Override + protected void onError() { + cleanTask(); + } + + @Override + protected void onNotFinished() { + cleanTask(); + } + + private void cleanTask() { + log.info("CleanTask begin to run:{}", this); + var storeEngine = HgStoreEngine.getInstance(); + if (storeEngine != null) { + if (getExtra() != null) { + CleanDataRequest request = (CleanDataRequest) getExtra(); + var partition = storeEngine.getPartitionManager() + .getPartition(getGraphName(), getPartitionId()); + // 只允许清理本分区之外的数据。 缩容等任务会造成干扰, 而且不能删除分区 + if (request.getKeyEnd() == partition.getStartKey() && + request.getKeyEnd() == partition.getEndKey() && + request.getCleanType() == CleanType.CLEAN_TYPE_EXCLUDE_RANGE && + !request.isDeletePartition()) { + storeEngine.getBusinessHandler() + .cleanPartition(getGraphName(), getPartitionId(), + request.getKeyStart(), request.getKeyEnd(), + request.getCleanType()); + } + } else { + storeEngine.getBusinessHandler().cleanPartition(getGraphName(), getPartitionId()); + } + + storeEngine.getPartitionEngine(getPartitionId()).getTaskManager() + .updateAsyncTaskState(getPartitionId(), getGraphName(), getId(), + AsyncTaskState.SUCCESS); + } + } +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/DBSessionBuilder.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/DBSessionBuilder.java new file mode 100644 index 000000000..fed6b1f61 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/DBSessionBuilder.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta.base; + +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.util.HgStoreException; + +public interface DBSessionBuilder { + RocksDBSession getSession(int partId) throws HgStoreException; +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/GlobalMetaStore.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/GlobalMetaStore.java similarity index 51% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/GlobalMetaStore.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/GlobalMetaStore.java index bf067e562..54966f733 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/GlobalMetaStore.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/GlobalMetaStore.java @@ -1,10 +1,27 @@ -package com.baidu.hugegraph.store.meta.base; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta.base; + +import java.util.Arrays; import org.apache.hugegraph.rocksdb.access.RocksDBFactory; import org.apache.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.store.options.MetadataOptions; - -import java.util.Arrays; +import org.apache.hugegraph.store.options.MetadataOptions; public class GlobalMetaStore extends MetaStoreBase { public static final String HSTORE_METADATA_GRAPH_NAME = "hgstore-metadata"; @@ -12,14 +29,14 @@ public class GlobalMetaStore extends MetaStoreBase { private final MetadataOptions options; - private String dataPath; + private final String dataPath; public GlobalMetaStore(MetadataOptions options) { this.options = options; dataPath = Arrays.asList(options.getDataPath().split(",")).get(0); } - public MetadataOptions getOptions(){ + public MetadataOptions getOptions() { return options; } diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/MetaStoreBase.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/MetaStoreBase.java similarity index 79% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/MetaStoreBase.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/MetaStoreBase.java index d5cac283c..4cec12eb8 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/MetaStoreBase.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/MetaStoreBase.java @@ -1,18 +1,36 @@ -package com.baidu.hugegraph.store.meta.base; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import org.apache.hugegraph.rocksdb.access.RocksDBSession; -import org.apache.hugegraph.rocksdb.access.ScanIterator; -import org.apache.hugegraph.rocksdb.access.SessionOperator; -import com.baidu.hugegraph.store.util.HgStoreException; -import com.baidu.hugegraph.store.util.Asserts; -import com.google.protobuf.GeneratedMessageV3; -import com.google.protobuf.InvalidProtocolBufferException; +package org.apache.hugegraph.store.meta.base; import java.io.Closeable; import java.io.IOException; import java.util.LinkedList; import java.util.List; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.rocksdb.access.SessionOperator; +import org.apache.hugegraph.store.util.Asserts; +import org.apache.hugegraph.store.util.HgStoreException; + +import com.google.protobuf.GeneratedMessageV3; +import com.google.protobuf.InvalidProtocolBufferException; + /** * Store、Partition等元数据存储到hgstore-metadata图下 */ @@ -97,7 +115,8 @@ public abstract class MetaStoreBase implements Closeable { try (RocksDBSession dbSession = getRocksDBSession()) { SessionOperator op = dbSession.sessionOp(); ScanIterator iterator = op.scan(getCFName(), start, end, - ScanIterator.Trait.SCAN_GTE_BEGIN | ScanIterator.Trait.SCAN_LT_END); + ScanIterator.Trait.SCAN_GTE_BEGIN | + ScanIterator.Trait.SCAN_LT_END); while (iterator.hasNext()) { values.add(iterator.next()); } @@ -109,7 +128,8 @@ public abstract class MetaStoreBase implements Closeable { try (RocksDBSession dbSession = getRocksDBSession()) { SessionOperator op = dbSession.sessionOp(); ScanIterator iterator = op.scan(getCFName(), start, end, - ScanIterator.Trait.SCAN_GTE_BEGIN | ScanIterator.Trait.SCAN_LT_END); + ScanIterator.Trait.SCAN_GTE_BEGIN | + ScanIterator.Trait.SCAN_LT_END); List values = new LinkedList<>(); try { while (iterator.hasNext()) { diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/PartitionMetaStore.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/PartitionMetaStore.java similarity index 50% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/PartitionMetaStore.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/PartitionMetaStore.java index a05698436..8996bd1b0 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/PartitionMetaStore.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/base/PartitionMetaStore.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.meta.base; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta.base; import org.apache.hugegraph.rocksdb.access.RocksDBSession; diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/metric/HgMetricService.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/metric/HgMetricService.java similarity index 58% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/metric/HgMetricService.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/metric/HgMetricService.java index ff2628166..c4662bf3e 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/metric/HgMetricService.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/metric/HgMetricService.java @@ -1,11 +1,21 @@ -package com.baidu.hugegraph.store.metric; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.PartitionEngine; -import com.baidu.hugegraph.store.util.Lifecycle; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; +package org.apache.hugegraph.store.metric; import java.io.File; import java.nio.file.FileStore; @@ -17,10 +27,23 @@ import java.util.HashSet; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.PartitionEngine; +import org.apache.hugegraph.store.util.Lifecycle; + +import com.baidu.hugegraph.pd.grpc.Metapb; + +import lombok.extern.slf4j.Slf4j; + @Slf4j public class HgMetricService implements Lifecycle { private final static HgMetricService instance = new HgMetricService(); - public static HgMetricService getInstance() { return instance; } + + public static HgMetricService getInstance() { + return instance; + } + private final static AtomicLong bytesWritten = new AtomicLong(); private final static AtomicLong bytesRead = new AtomicLong(); private final static AtomicLong keysWritten = new AtomicLong(); @@ -30,9 +53,10 @@ public class HgMetricService implements Lifecycle { private static long lastQueryTime = 0; private Map systemMetrics = new HashMap<>(); - private SystemMetricService systemMetricService = new SystemMetricService(); + private final SystemMetricService systemMetricService = new SystemMetricService(); - private HgMetricService() { } + private HgMetricService() { + } @Override public boolean init(final Void v) { @@ -58,7 +82,7 @@ public class HgMetricService implements Lifecycle { getRaftMetrics(builder); getDiskMetrics(builder); getSystemMetrics(builder); - }catch (Exception e){ + } catch (Exception e) { log.error("HgMetricService getMetrics {}", e); } return builder; @@ -70,9 +94,9 @@ public class HgMetricService implements Lifecycle { long available = 0L; long used = 0L; HashSet fileStoreSet = new HashSet<>(); - for(String dbPath: this.storeEngine.getDataLocations()) { + for (String dbPath : this.storeEngine.getDataLocations()) { FileStore fs = Files.getFileStore(Paths.get(dbPath)); - if(fileStoreSet.contains(fs.name())) { + if (fileStoreSet.contains(fs.name())) { continue; } fileStoreSet.add(fs.name()); @@ -81,7 +105,7 @@ public class HgMetricService implements Lifecycle { used += FileUtils.sizeOfDirectory(new File(dbPath)); } builder.setCapacity(capacity); - builder.setAvailable(available); ; + builder.setAvailable(available); builder.setUsedSize(used); } catch (Exception e) { log.error("Failed to get disk metrics. {}", e.toString()); @@ -94,9 +118,9 @@ public class HgMetricService implements Lifecycle { builder.setPartitionCount(partitionEngines.size()); partitionEngines.forEach((partId, engine) -> { builder.addRaftStats(Metapb.RaftStats.newBuilder() - .setPartitionId(partId) - .setCommittedIndex(engine.getCommittedIndex()) - .build()); + .setPartitionId(partId) + .setCommittedIndex(engine.getCommittedIndex()) + .build()); }); return builder; } @@ -107,18 +131,25 @@ public class HgMetricService implements Lifecycle { builder.setStartTime((int) startTime); this.storeEngine.getPartitionManager().getPartitions().forEach((graphName, partitions) -> { partitions.forEach((partId, partition) -> { - HgStoreMetric.Graph graphMetric = this.storeEngine.getBusinessHandler().getGraphMetric(graphName, partId); - if ( (graphMetric != null) && - (storeEngine.getPartitionManager().getLocalRoleFromShard(partition) != null)) { + HgStoreMetric.Graph graphMetric = + this.storeEngine.getBusinessHandler().getGraphMetric(graphName, partId); + if ((graphMetric != null) && + (storeEngine.getPartitionManager().getLocalRoleFromShard(partition) != null)) { builder.addGraphStats(Metapb.GraphStats.newBuilder() - .setGraphName(graphName) - .setPartitionId(partId) - .setApproximateKeys(graphMetric.getApproxKeyCount()) - .setApproximateSize(graphMetric.getApproxDataSize()) - .setRole(storeEngine.getPartitionManager().getLocalRoleFromShard(partition).toShardRole()) - .setWorkState(partition.getWorkState()) + .setGraphName(graphName) + .setPartitionId(partId) + .setApproximateKeys( + graphMetric.getApproxKeyCount()) + .setApproximateSize( + graphMetric.getApproxDataSize()) + .setRole( + storeEngine.getPartitionManager() + .getLocalRoleFromShard( + partition) + .toShardRole()) + .setWorkState(partition.getWorkState()) - .build()); + .build()); } }); @@ -129,6 +160,7 @@ public class HgMetricService implements Lifecycle { /** * get system metrics each 1 minute + * * @param builder * @return */ @@ -142,9 +174,9 @@ public class HgMetricService implements Lifecycle { for (Map.Entry entry : systemMetrics.entrySet()) { if (entry.getValue() != null) { builder.addSystemMetrics(Metapb.RecordPair.newBuilder() - .setKey(entry.getKey()) - .setValue(entry.getValue()) - .build()); + .setKey(entry.getKey()) + .setValue(entry.getValue()) + .build()); } } diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/metric/HgStoreMetric.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/metric/HgStoreMetric.java new file mode 100644 index 000000000..9115400ac --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/metric/HgStoreMetric.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.metric; + +import java.util.List; + +import lombok.Data; + +public class HgStoreMetric { + + @Data + public static class Table { + private String tableName; + private long keyCount; + private String dataSize; + } + + @Data + public static class Partition { + private int partitionId; + private List

    tables; + } + + @Data + public static class Graph { + private long approxDataSize; + private long approxKeyCount; + } +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/metric/SystemMetricService.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/metric/SystemMetricService.java similarity index 72% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/metric/SystemMetricService.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/metric/SystemMetricService.java index 043c6781a..743402ceb 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/metric/SystemMetricService.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/metric/SystemMetricService.java @@ -1,13 +1,21 @@ -package com.baidu.hugegraph.store.metric; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import org.apache.hugegraph.rocksdb.access.RocksDBFactory; -import org.apache.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.sun.management.OperatingSystemMXBean; -import lombok.extern.slf4j.Slf4j; -import org.rocksdb.MemoryUsageType; -import org.rocksdb.Statistics; -import org.rocksdb.TickerType; +package org.apache.hugegraph.store.metric; import java.io.BufferedReader; import java.io.File; @@ -31,12 +39,23 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.HgStoreEngine; +import org.rocksdb.MemoryUsageType; +import org.rocksdb.Statistics; +import org.rocksdb.TickerType; + +import com.sun.management.OperatingSystemMXBean; + +import lombok.extern.slf4j.Slf4j; + @Slf4j public class SystemMetricService { private static final long MIB = 1024 * 1024; HgStoreEngine storeEngine; - private Deque>> deque = new LinkedList<>(); + private final Deque>> deque = new LinkedList<>(); public void setStoreEngine(HgStoreEngine hgStoreEngine) { this.storeEngine = hgStoreEngine; @@ -66,7 +85,7 @@ public class SystemMetricService { // rocksdb loadRocksDbInfo(systemMetrics); - }catch (Exception e){ + } catch (Exception e) { log.error("get system metric failed, {}", e.toString()); } @@ -74,13 +93,15 @@ public class SystemMetricService { } private void loadCpuInfo(Map map) { - OperatingSystemMXBean osBean = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); + OperatingSystemMXBean osBean = + (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); Double cpuLoad = osBean.getSystemLoadAverage(); map.put("cpu.load", cpuLoad.longValue()); } private void loadMemInfo(Map map) { - OperatingSystemMXBean osBean = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); + OperatingSystemMXBean osBean = + (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); map.put("mem.physical_total", osBean.getTotalPhysicalMemorySize()); map.put("mem.physical_free", osBean.getFreePhysicalMemorySize()); map.put("mem.swap_total", osBean.getTotalSwapSpaceSize()); @@ -122,15 +143,20 @@ public class SystemMetricService { private void loadNetFlowInfo(Map map) { for (Map.Entry> entry : getTraffic().entrySet()) { // exclude none-functional network interface - map.put("network." + entry.getKey() + ".sent_bytes", entry.getValue().get(0) / 1024 / 1024); - map.put("network." + entry.getKey() + ".recv_bytes", entry.getValue().get(1) / 1024 / 1024); - map.put("network." + entry.getKey() + ".sent_rates", entry.getValue().get(2) / 1024 / 1024); - map.put("network." + entry.getKey() + ".recv_rates", entry.getValue().get(3) / 1024 / 1024); + map.put("network." + entry.getKey() + ".sent_bytes", + entry.getValue().get(0) / 1024 / 1024); + map.put("network." + entry.getKey() + ".recv_bytes", + entry.getValue().get(1) / 1024 / 1024); + map.put("network." + entry.getKey() + ".sent_rates", + entry.getValue().get(2) / 1024 / 1024); + map.put("network." + entry.getKey() + ".recv_rates", + entry.getValue().get(3) / 1024 / 1024); } } private void loadRocksDbInfo(Map map) { - Map dbMem = storeEngine.getBusinessHandler().getApproximateMemoryUsageByType(null); + Map dbMem = + storeEngine.getBusinessHandler().getApproximateMemoryUsageByType(null); map.put("rocksdb.table.reader.total", dbMem.get(MemoryUsageType.kTableReadersTotal)); map.put("rocksdb.mem.table.total", dbMem.get(MemoryUsageType.kMemTableTotal)); map.put("rocksdb.cache.total", dbMem.get(MemoryUsageType.kCacheTotal)); @@ -144,20 +170,25 @@ public class SystemMetricService { RocksDBSession session = dbFactory.queryGraphDB(name); Statistics statistics = session.getRocksDbStats(); map.put( - "rocksdb.graph." + name + "." + TickerType.NUMBER_KEYS_WRITTEN.name().toLowerCase(), - statistics.getTickerCount(TickerType.NUMBER_KEYS_WRITTEN)); + "rocksdb.graph." + name + "." + + TickerType.NUMBER_KEYS_WRITTEN.name().toLowerCase(), + statistics.getTickerCount(TickerType.NUMBER_KEYS_WRITTEN)); map.put( - "rocksdb.graph." + name + "." + TickerType.NUMBER_KEYS_READ.name().toLowerCase(), - statistics.getTickerCount(TickerType.NUMBER_KEYS_READ)); + "rocksdb.graph." + name + "." + + TickerType.NUMBER_KEYS_READ.name().toLowerCase(), + statistics.getTickerCount(TickerType.NUMBER_KEYS_READ)); map.put( - "rocksdb.graph." + name + "." + TickerType.NUMBER_KEYS_UPDATED.name().toLowerCase(), - statistics.getTickerCount(TickerType.NUMBER_KEYS_UPDATED)); + "rocksdb.graph." + name + "." + + TickerType.NUMBER_KEYS_UPDATED.name().toLowerCase(), + statistics.getTickerCount(TickerType.NUMBER_KEYS_UPDATED)); map.put( - "rocksdb.graph." + name + "." + TickerType.BYTES_WRITTEN.name().toLowerCase(), - statistics.getTickerCount(TickerType.BYTES_WRITTEN)); + "rocksdb.graph." + name + "." + + TickerType.BYTES_WRITTEN.name().toLowerCase(), + statistics.getTickerCount(TickerType.BYTES_WRITTEN)); map.put( - "rocksdb.graph." + name + "." + TickerType.BYTES_READ.name().toLowerCase(), - statistics.getTickerCount(TickerType.BYTES_READ)); + "rocksdb.graph." + name + "." + + TickerType.BYTES_READ.name().toLowerCase(), + statistics.getTickerCount(TickerType.BYTES_READ)); } catch (Exception e) { } @@ -167,10 +198,10 @@ public class SystemMetricService { /** * get all network interface traffic(delta from last invoke). - * -sent bytes - * -receive bytes - * -in rates - * -out rates + * -sent bytes + * -receive bytes + * -in rates + * -out rates * * @return */ @@ -201,14 +232,14 @@ public class SystemMetricService { long diff = now.get(2) - prev.get(2); diff = diff > 0 ? diff : 1L; result.put( - entry.getKey(), - Arrays.asList( - now.get(0) - prev.get(0), - now.get(1) - prev.get(1), - // rate rate - (now.get(0) - prev.get(0)) / diff, - // recv rate - (now.get(1) - prev.get(1)) / diff)); + entry.getKey(), + Arrays.asList( + now.get(0) - prev.get(0), + now.get(1) - prev.get(1), + // rate rate + (now.get(0) - prev.get(0)) / diff, + // recv rate + (now.get(1) - prev.get(1)) / diff)); } } return result; @@ -243,7 +274,8 @@ public class SystemMetricService { continue; } - List arr = Arrays.stream(line.split(" ")).filter(x -> x.length() > 0).collect(Collectors.toList()); + List arr = Arrays.stream(line.split(" ")).filter(x -> x.length() > 0) + .collect(Collectors.toList()); long sentBytes = Long.parseLong(arr.get(arr.size() - 2)); long recvBytes = Long.parseLong(arr.get(arr.size() - 5)); @@ -259,8 +291,8 @@ public class SystemMetricService { /** * read the statistics file for network interface - * cat /sys/class/net/NETWORK_INTERFACE_NAME/statistics/tx_bytes - * cat /sys/class/net/NETWORK_INTERFACE_NAME/statistics/rx_bytes + * cat /sys/class/net/NETWORK_INTERFACE_NAME/statistics/tx_bytes + * cat /sys/class/net/NETWORK_INTERFACE_NAME/statistics/rx_bytes * * @return */ @@ -269,8 +301,10 @@ public class SystemMetricService { Map> flows = new HashMap<>(); try { for (String name : getAllNetworkInterfaces()) { - long sentBytes = getUnsignedLongFromFile(String.format("/sys/class/net/%s/statistics/tx_bytes", name)); - long recvBytes = getUnsignedLongFromFile(String.format("/sys/class/net/%s/statistics/rx_bytes", name)); + long sentBytes = getUnsignedLongFromFile( + String.format("/sys/class/net/%s/statistics/tx_bytes", name)); + long recvBytes = getUnsignedLongFromFile( + String.format("/sys/class/net/%s/statistics/rx_bytes", name)); flows.put(name, Arrays.asList(sentBytes, recvBytes, current)); } } catch (Exception e) { @@ -323,6 +357,7 @@ public class SystemMetricService { /** * get io data using iostat -d -x -k + * * @return */ private Map loadLinuxDiskIoData() { @@ -337,12 +372,15 @@ public class SystemMetricService { if (contentFlag) { List arr = - Arrays.stream(line.split(" ")).filter(x -> x.length() > 0).collect(Collectors.toList()); + Arrays.stream(line.split(" ")).filter(x -> x.length() > 0) + .collect(Collectors.toList()); try { // util% - result.put("disk.io." + arr.get(0) + ".util", Float.valueOf(arr.get(arr.size() - 1)) * 100); + result.put("disk.io." + arr.get(0) + ".util", + Float.valueOf(arr.get(arr.size() - 1)) * 100); // wait - result.put("disk.io." + arr.get(0) + ".wait", Float.valueOf(arr.get(arr.size() - 5)) * 100); + result.put("disk.io." + arr.get(0) + ".wait", + Float.valueOf(arr.get(arr.size() - 5)) * 100); } catch (Exception e) { log.debug("error get disk io data {}", line); } @@ -353,6 +391,7 @@ public class SystemMetricService { /** * get io data using iostat + * * @return */ private Map loadMacDiskIoData() { @@ -361,17 +400,19 @@ public class SystemMetricService { List lines = executeCmd("iostat -oK"); // disks List disks = - Arrays.stream(lines.get(0).split(" ")) - .filter(x -> x.length() > 0 && x.startsWith("disk")) - .collect(Collectors.toList()); + Arrays.stream(lines.get(0).split(" ")) + .filter(x -> x.length() > 0 && x.startsWith("disk")) + .collect(Collectors.toList()); // datas List data = - Arrays.stream(lines.get(2).split(" ")).filter(x -> x.length() > 0).collect(Collectors.toList()); + Arrays.stream(lines.get(2).split(" ")).filter(x -> x.length() > 0) + .collect(Collectors.toList()); // zip data for (int i = 0; i < disks.size(); i++) { try { // msps - result.put("disk.io." + disks.get(i) + ".wait", Float.valueOf(data.get(i * 3 + 2)) * 100); + result.put("disk.io." + disks.get(i) + ".wait", + Float.valueOf(data.get(i * 3 + 2)) * 100); // no such value result.put("disk.io." + disks.get(i) + ".util", 0.0F); } catch (Exception e) { diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/options/HgStoreEngineOptions.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/options/HgStoreEngineOptions.java new file mode 100644 index 000000000..14dc6254d --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/options/HgStoreEngineOptions.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.options; + +import java.util.Map; + +import org.apache.hugegraph.store.business.DataMover; +import org.apache.hugegraph.store.pd.PdProvider; +import org.apache.hugegraph.store.raft.RaftTaskHandler; + +import com.alipay.sofa.jraft.util.Utils; + +import lombok.Data; + +/** + * Storage engine configuration + */ +@Data +public class HgStoreEngineOptions { + public static String Raft_Path_Prefix = "raft"; + public static String DB_Path_Prefix = "db"; + public static String Snapshot_Path_Prefix = "snapshot"; + // pd 服务器地址 + private String pdAddress; + // 对外服务地址 + private String grpcAddress; + // Raft 对外服务地址 + private String raftAddress; + // 存储路径，支持多个位置，逗号分割 + private String dataPath; + + private String raftPath; + + private Map rocksdbConfig; + // store心跳间隔，单位秒 + private final int storeHBInterval = 30; + // 分区心跳间隔，单位秒 + private final int partitionHBInterval = 5; + // 等待leader超时时间，单位秒 + private final int waitLeaderTimeout = 30; + // 没有PD模式，用于开发调试使用 + private final boolean fakePD = false; + + private final int raftRpcThreadPoolSize = Utils.cpus() * 6; + // 自定义的标签，传给pd + private Map labels; + // fakePd配置项 + private final FakePdOptions fakePdOptions = new FakePdOptions(); + private final RaftOptions raftOptions = new RaftOptions(); + + // Raft任务处理器 + private RaftTaskHandler taskHandler; + + private PdProvider pdProvider; + + // 数据迁移服务 + private DataMover dataTransfer; + + @Data + public static class FakePdOptions { + private String storeList; + private String peersList; + private final int partitionCount = 0; + private final int shardCount = 0; + } + + @Data + public static class RaftOptions { + + /* + * Rpc connect timeout in milliseconds + * The time should be less than electionTimeoutMs, otherwise the election will timeout + */ + private final int rpcConnectTimeoutMs = 1000; + + /** + * RPC request default timeout in milliseconds + */ + private final int rpcDefaultTimeout = 5000; + + // A follower would become a candidate if it doesn't receive any message + // from the leader in |election_timeout_ms| milliseconds + + private final int electionTimeoutMs = 3000; + /** + * Install snapshot RPC request default timeout in milliseconds + */ + private final int rpcInstallSnapshotTimeout = 60 * 60 * 1000; + // A snapshot saving would be triggered every |snapshot_interval_s| seconds + // if this was reset as a positive number + // If |snapshot_interval_s| <= 0, the time based snapshot would be disabled. + // + // Default: 3600 (1 hour) + private final int snapshotIntervalSecs = 3600; + // A snapshot saving would be triggered every |snapshot_interval_s| seconds, + // and at this moment when state machine's lastAppliedIndex value + // minus lastSnapshotId value is greater than snapshotLogIndexMargin value, + // the snapshot action will be done really. + // If |snapshotLogIndexMargin| <= 0, the distance based snapshot would be disable. + // + // Default: 0 + private final int snapshotLogIndexMargin = 1024; + private final boolean metrics = true; + // 等待leader超时时间，单位秒 + private final int waitLeaderTimeout = 30; + /** + * Internal disruptor buffers size for Node/FSMCaller/LogManager etc. + */ + private final int disruptorBufferSize = 4096; + /** + * The maximum number of entries in AppendEntriesRequest + */ + private final int maxEntriesSize = 256; + /** + * The maximum replicator pipeline in-flight requests/responses, only valid when enable + * replicator pipeline. + */ + private final int maxReplicatorInflightMsgs = 256; + /** + * Raft集群发生数据积压后，限速等待时间 单位毫秒 + **/ + private final int overloadRateLimit = 100; + + /** + * The maximum byte size of log allowed by user. + */ + private final long maxLogSize = 100 * 1024 * 1024; + /** + * The ratio of exponential approximation for average size of log entry. + */ + private final double aveLogEntrySizeRatio = 0.95; + + private final boolean useRocksDBSegmentLogStorage = true; + private final int maxSegmentFileSize = 64 * 1024 * 1024; + private final int keepInMemorySegmentCount = 2; + private final int preAllocateSegmentCount = 1; + private final int splitPartitionLogIndexMargin = 10; + } +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/options/MetadataOptions.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/options/MetadataOptions.java new file mode 100644 index 000000000..1bfd2f844 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/options/MetadataOptions.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.options; + +import lombok.Data; + +@Data +public class MetadataOptions { + private String dataPath; + private String raftPath; +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/options/PartitionEngineOptions.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/options/PartitionEngineOptions.java new file mode 100644 index 000000000..dd30ed927 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/options/PartitionEngineOptions.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.options; + +import java.util.List; + +import org.apache.hugegraph.store.raft.RaftTaskHandler; + +import com.alipay.sofa.jraft.conf.Configuration; + +import lombok.Data; + +/** + * Partition engine configuration + */ +@Data +public class PartitionEngineOptions { + // raft存储路径 + private String raftDataPath; + private String raftSnapShotPath; + private Integer groupId; + private String raftAddress; + private List peerList; + private Configuration conf; + // 异步任务执行时间间隔, 单位秒 + private final int taskScheduleTime = 60; + // 分裂过程，等待数据对齐超时时间 + private final long splitPartitionTimeout = 30 * 60 * 1000; + // raft 任务处理器 + private RaftTaskHandler taskHandler; + + HgStoreEngineOptions.RaftOptions raftOptions; +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RaftRocksdbOptions.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RaftRocksdbOptions.java new file mode 100644 index 000000000..c7c533d6d --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RaftRocksdbOptions.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.options; + +import java.util.Map; + +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; +import org.apache.hugegraph.store.business.BusinessHandlerImpl; +import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.BloomFilter; +import org.rocksdb.Cache; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.CompressionType; +import org.rocksdb.DBOptions; +import org.rocksdb.Env; +import org.rocksdb.IndexType; +import org.rocksdb.LRUCache; +import org.rocksdb.RocksDB; +import org.rocksdb.WriteBufferManager; +import org.rocksdb.util.SizeUnit; + +import com.alipay.sofa.jraft.storage.impl.RocksDBLogStorage; +import com.alipay.sofa.jraft.util.StorageOptionsFactory; +import com.baidu.hugegraph.config.HugeConfig; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class RaftRocksdbOptions { + static class RocksdbConfig { + private final Env env; + private final LRUCache blockCache; + private final LRUCache writeCache; + private final WriteBufferManager bufferManager; + private final BlockBasedTableConfig tableConfig; + private final long blockCacheCapacity; + private final long writeCacheCapacity; + + public Env getEnv() { + return env; + } + + public LRUCache getBlockCache() { + return blockCache; + } + + public LRUCache getWriteCache() { + return writeCache; + } + + public WriteBufferManager getBufferManager() { + return bufferManager; + } + + public BlockBasedTableConfig getTableConfig() { + return tableConfig; + } + + public long getBlockCacheCapacity() { + return blockCacheCapacity; + } + + public long getWriteCacheCapacity() { + return writeCacheCapacity; + } + + public RocksdbConfig(HugeConfig options) { + RocksDB.loadLibrary(); + this.env = Env.getDefault(); + double writeBufferRatio = options.get(RocksDBOptions.WRITE_BUFFER_RATIO); + this.writeCacheCapacity = + (long) (options.get(RocksDBOptions.TOTAL_MEMORY_SIZE) * writeBufferRatio); + this.blockCacheCapacity = + options.get(RocksDBOptions.TOTAL_MEMORY_SIZE) - writeCacheCapacity; + this.writeCache = new LRUCache(writeCacheCapacity); + this.blockCache = new LRUCache(blockCacheCapacity); + this.bufferManager = new WriteBufferManager(writeCacheCapacity, writeCache, + options.get( + RocksDBOptions.WRITE_BUFFER_ALLOW_STALL)); + this.tableConfig = new BlockBasedTableConfig() // + .setIndexType( + IndexType.kTwoLevelIndexSearch) // + .setPartitionFilters(true) // + .setMetadataBlockSize(8 * SizeUnit.KB) // + .setCacheIndexAndFilterBlocks( + options.get( + RocksDBOptions.PUT_FILTER_AND_INDEX_IN_CACHE)) // + .setCacheIndexAndFilterBlocksWithHighPriority( + true) // + .setPinL0FilterAndIndexBlocksInCache( + options.get( + RocksDBOptions.PIN_L0_FILTER_AND_INDEX_IN_CACHE)) // + .setBlockSize(4 * SizeUnit.KB)// + .setBlockCache(blockCache); + + int bitsPerKey = options.get(RocksDBOptions.BLOOM_FILTER_BITS_PER_KEY); + if (bitsPerKey >= 0) { + tableConfig.setFilterPolicy(new BloomFilter(bitsPerKey, + options.get( + RocksDBOptions.BLOOM_FILTER_MODE))); + } + tableConfig.setWholeKeyFiltering( + options.get(RocksDBOptions.BLOOM_FILTER_WHOLE_KEY)); + log.info("RocksdbConfig {}", options.get(RocksDBOptions.BLOOM_FILTER_BITS_PER_KEY)); + } + } + + private static RocksdbConfig rocksdbConfig = null; + + private static RocksdbConfig getRocksdbConfig(HugeConfig options) { + if (rocksdbConfig == null) { + synchronized (RocksdbConfig.class) { + rocksdbConfig = new RocksdbConfig(options); + } + } + return rocksdbConfig; + } + + private static void registerRaftRocksdbConfig(HugeConfig options) { + Cache blockCache = new LRUCache(SizeUnit.GB); + BlockBasedTableConfig tableConfig = new BlockBasedTableConfig() // + .setIndexType( + IndexType.kTwoLevelIndexSearch) // + .setPartitionFilters( + true) // + .setMetadataBlockSize( + 8 * SizeUnit.KB) // + .setCacheIndexAndFilterBlocks( + options.get( + RocksDBOptions.PUT_FILTER_AND_INDEX_IN_CACHE)) // + .setCacheIndexAndFilterBlocksWithHighPriority( + true) // + .setPinL0FilterAndIndexBlocksInCache( + options.get( + RocksDBOptions.PIN_L0_FILTER_AND_INDEX_IN_CACHE)) // + .setBlockSize( + 4 * SizeUnit.KB)// + .setBlockCache(blockCache); + + StorageOptionsFactory.registerRocksDBTableFormatConfig(RocksDBLogStorage.class, + tableConfig); + + DBOptions dbOptions = StorageOptionsFactory.getDefaultRocksDBOptions(); + dbOptions.setEnv(rocksdbConfig.getEnv()); + + // raft rocksdb数量固定，通过max_write_buffer_number可以控制 + //dbOptions.setWriteBufferManager(rocksdbConfig.getBufferManager()); + dbOptions.setUnorderedWrite(true); + StorageOptionsFactory.registerRocksDBOptions(RocksDBLogStorage.class, + dbOptions); + + ColumnFamilyOptions cfOptions = + StorageOptionsFactory.getDefaultRocksDBColumnFamilyOptions(); + cfOptions.setTargetFileSizeBase(256 * SizeUnit.MB); + cfOptions.setWriteBufferSize(8 * SizeUnit.MB); + cfOptions.setNumLevels(3); + cfOptions.setMaxWriteBufferNumber(3); + cfOptions.setCompressionType(CompressionType.NO_COMPRESSION); + cfOptions.setMaxBytesForLevelBase(2048 * SizeUnit.GB); + + StorageOptionsFactory.registerRocksDBColumnFamilyOptions(RocksDBLogStorage.class, + cfOptions); + } + + + public static void initRocksdbGlobalConfig(Map config) { + HugeConfig hugeConfig = BusinessHandlerImpl.initRocksdb(config, null); + RocksdbConfig rocksdbConfig = getRocksdbConfig(hugeConfig); + registerRaftRocksdbConfig(hugeConfig); + config.put(RocksDBOptions.ENV, rocksdbConfig.getEnv()); + config.put(RocksDBOptions.WRITE_BUFFER_MANAGER, rocksdbConfig.getBufferManager()); + config.put(RocksDBOptions.BLOCK_TABLE_CONFIG, rocksdbConfig.getTableConfig()); + config.put(RocksDBOptions.BLOCK_CACHE, rocksdbConfig.getBlockCache()); + config.put(RocksDBOptions.WRITE_CACHE, rocksdbConfig.getWriteCache()); + } + + public static WriteBufferManager getWriteBufferManager() { + return rocksdbConfig.getBufferManager(); + } + + public static Env getEnv() { + return rocksdbConfig.getEnv(); + } + + public static Cache getWriteCache() { + return rocksdbConfig.getWriteCache(); + } + + public static Cache getBlockCache() { + return rocksdbConfig.getBlockCache(); + } + + public static long getWriteCacheCapacity() { + return rocksdbConfig.getWriteCacheCapacity(); + } + + public static long getBlockCacheCapacity() { + return rocksdbConfig.getBlockCacheCapacity(); + } +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/pd/DefaultPdProvider.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/pd/DefaultPdProvider.java similarity index 77% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/pd/DefaultPdProvider.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/pd/DefaultPdProvider.java index a666fabd7..e90e9d447 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/pd/DefaultPdProvider.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/pd/DefaultPdProvider.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.pd; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.pd; import java.util.ArrayList; @@ -8,7 +25,13 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; -import com.baidu.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.meta.Graph; +import org.apache.hugegraph.store.meta.GraphManager; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.meta.Store; +import org.apache.hugegraph.store.metric.HgMetricService; +import org.apache.hugegraph.store.util.Asserts; import org.slf4j.Logger; import com.baidu.hugegraph.pd.client.PDClient; @@ -25,12 +48,6 @@ import com.baidu.hugegraph.pd.grpc.watch.WatchResponse; import com.baidu.hugegraph.pd.pulse.PulseServerNotice; import com.baidu.hugegraph.pd.watch.NodeEvent; import com.baidu.hugegraph.pd.watch.PartitionEvent; -import com.baidu.hugegraph.store.meta.Graph; -import com.baidu.hugegraph.store.meta.GraphManager; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.meta.Store; -import com.baidu.hugegraph.store.metric.HgMetricService; -import com.baidu.hugegraph.store.util.Asserts; import com.baidu.hugegraph.util.Log; import lombok.extern.slf4j.Slf4j; @@ -38,13 +55,13 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class DefaultPdProvider implements PdProvider { private static final Logger LOG = Log.logger(DefaultPdProvider.class); - private PDClient pdClient; - private String pdServerAddress; + private final PDClient pdClient; + private final String pdServerAddress; private List partitionCommandListeners = Collections.synchronizedList(new ArrayList()); private PDPulse.Notifier pdPulse; - private static ConcurrentMap quotas = + private static final ConcurrentMap quotas = new ConcurrentHashMap<>(); private GraphManager graphManager = null; @@ -85,22 +102,29 @@ public class DefaultPdProvider implements PdProvider { @Override public long registerStore(Store store) throws PDException { Asserts.isTrue(this.pdClient != null, "pd client is null"); - LOG.info("registerStore pd={} storeId={}, store={}", this.pdServerAddress, store.getId(), store); + LOG.info("registerStore pd={} storeId={}, store={}", this.pdServerAddress, store.getId(), + store); long storeId = 0; Metapb.Store protoObj = store.getProtoObj(); try { storeId = pdClient.registerStore(protoObj); store.setId(storeId); - if (pdClient.getStore(storeId).getState() != Metapb.StoreState.Up) - LOG.warn("Store {} is not activated, state is {}", storeId, pdClient.getStore(storeId).getState()); + if (pdClient.getStore(storeId).getState() != Metapb.StoreState.Up) { + LOG.warn("Store {} is not activated, state is {}", storeId, + pdClient.getStore(storeId).getState()); + } } catch (PDException e) { - LOG.error("Exception in storage registration, StoreID= {} pd= {} exceptCode= {} except= {}.", + LOG.error( + "Exception in storage registration, StoreID= {} pd= {} exceptCode= {} except=" + + " {}.", protoObj.getId(), this.pdServerAddress, e.getErrorCode(), e.getMessage()); storeId = 0; throw e; } catch (Exception e) { - LOG.error("Exception in storage registration, StoreID= {} pd= {} except= {}, Please check your network settings.", + LOG.error( + "Exception in storage registration, StoreID= {} pd= {} except= {}, Please " + + "check your network settings.", protoObj.getId(), this.pdServerAddress, e.getMessage()); handleCommonException(e); storeId = 0; @@ -113,8 +137,9 @@ public class DefaultPdProvider implements PdProvider { try { KVPair pair = pdClient.getPartitionById( graph, partId); - if (null != pair) + if (null != pair) { return new Partition(pair.getKey()); + } } catch (PDException e) { log.error("Partition {}-{} getPartitionByID exception {}", graph, partId, e); } @@ -140,8 +165,9 @@ public class DefaultPdProvider implements PdProvider { try { KVPair pair = pdClient.getPartitionByCode( graph, code); - if (null != pair) + if (null != pair) { return pair.getKey(); + } } catch (PDException e) { log.error("Partition {} getPartitionByCode {} exception {}", graph, code, e); } @@ -153,8 +179,9 @@ public class DefaultPdProvider implements PdProvider { log.info("Partition {}-{} send delPartition to PD", graph, partId); try { Metapb.Partition partition = pdClient.delPartition(graph, partId); - if ( null != partition) + if (null != partition) { return new Partition(partition); + } } catch (PDException e) { log.error("Partition {}-{} remove exception {}", graph, partId, e); } @@ -162,7 +189,8 @@ public class DefaultPdProvider implements PdProvider { } @Override - public List updatePartition(List partitions) throws PDException { + public List updatePartition(List partitions) throws + PDException { try { List results = pdClient.updatePartition(partitions); @@ -176,7 +204,7 @@ public class DefaultPdProvider implements PdProvider { public List getPartitionsByStore(long storeId) throws PDException { List partitions = new ArrayList<>(); List parts = pdClient.getPartitionsByStore(storeId); - parts.forEach(e->{ + parts.forEach(e -> { partitions.add(new Partition(e)); }); return partitions; @@ -194,12 +222,13 @@ public class DefaultPdProvider implements PdProvider { } } } - if (!changeLeader){ + if (!changeLeader) { try { - leader = pdClient.getPartitionById(partition.getGraphName(), partition.getId()).getValue(); + leader = pdClient.getPartitionById(partition.getGraphName(), partition.getId()) + .getValue(); } catch (PDException e) { log.error("find leader error,leader changed to storeId:{}", leader.getStoreId()); - } catch (Exception e1){ + } catch (Exception e1) { log.error("exception ", e1); } } @@ -213,6 +242,7 @@ public class DefaultPdProvider implements PdProvider { /** * 启动partition心跳流式传输 + * * @return */ @Override @@ -231,7 +261,8 @@ public class DefaultPdProvider implements PdProvider { @Override public void accept(Integer integer) { LOG.debug("Partition heartbeat accept instruction: {}", instruction); - // LOG.info("accept notice id : {}, ts:{}", response.getNoticeId(), System.currentTimeMillis()); + // LOG.info("accept notice id : {}, ts:{}", response.getNoticeId(), + // System.currentTimeMillis()); // http2 并发问题，需要加锁 // synchronized (pdPulse) { response.ack(); @@ -241,30 +272,37 @@ public class DefaultPdProvider implements PdProvider { for (PartitionInstructionListener event : partitionCommandListeners) { if (instruction.hasChangeShard()) { - event.onChangeShard(instruction.getId(), partition, instruction.getChangeShard(), consumer); + event.onChangeShard(instruction.getId(), partition, + instruction.getChangeShard(), consumer); } if (instruction.hasSplitPartition()) { - event.onSplitPartition(instruction.getId(), partition, instruction.getSplitPartition(), consumer); + event.onSplitPartition(instruction.getId(), partition, + instruction.getSplitPartition(), consumer); } if (instruction.hasTransferLeader()) { - event.onTransferLeader(instruction.getId(), partition, instruction.getTransferLeader(), consumer); + event.onTransferLeader(instruction.getId(), partition, + instruction.getTransferLeader(), consumer); } if (instruction.hasDbCompaction()) { - event.onDbCompaction(instruction.getId(), partition, instruction.getDbCompaction(), consumer); + event.onDbCompaction(instruction.getId(), partition, + instruction.getDbCompaction(), consumer); } if (instruction.hasMovePartition()) { - event.onMovePartition(instruction.getId(), partition, instruction.getMovePartition(), consumer); + event.onMovePartition(instruction.getId(), partition, + instruction.getMovePartition(), consumer); } if (instruction.hasCleanPartition()) { - event.onCleanPartition(instruction.getId(), partition, instruction.getCleanPartition(), - consumer); + event.onCleanPartition(instruction.getId(), partition, + instruction.getCleanPartition(), + consumer); } - if (instruction.hasKeyRange()){ - event.onPartitionKeyRangeChanged(instruction.getId(), partition, instruction.getKeyRange(), - consumer); + if (instruction.hasKeyRange()) { + event.onPartitionKeyRangeChanged(instruction.getId(), partition, + instruction.getKeyRange(), + consumer); } } } @@ -286,11 +324,12 @@ public class DefaultPdProvider implements PdProvider { /** * 添加服务端消息监听 + * * @param listener * @return */ @Override - public boolean addPartitionInstructionListener(PartitionInstructionListener listener) { + public boolean addPartitionInstructionListener(PartitionInstructionListener listener) { partitionCommandListeners.add(listener); return true; } @@ -299,7 +338,7 @@ public class DefaultPdProvider implements PdProvider { public boolean partitionHeartbeat(List statsList) { for (Metapb.PartitionStats stats : statsList) { PartitionHeartbeatRequest.Builder request = PartitionHeartbeatRequest.newBuilder() - .setStates(stats); + .setStates(stats); pdPulse.notifyServer(request); } return false; @@ -316,8 +355,8 @@ public class DefaultPdProvider implements PdProvider { } @Override - public Metapb.Graph getGraph(String graphName) throws PDException{ - return pdClient.getGraph( graphName); + public Metapb.Graph getGraph(String graphName) throws PDException { + return pdClient.getGraph(graphName); } @Override @@ -353,7 +392,7 @@ public class DefaultPdProvider implements PdProvider { } catch (PDException e) { log.error("getClusterStats exception {}", e); return Metapb.ClusterStats.newBuilder() - .setState(Metapb.ClusterState.Cluster_Fault).build(); + .setState(Metapb.ClusterState.Cluster_Fault).build(); } } @@ -367,23 +406,24 @@ public class DefaultPdProvider implements PdProvider { stats.setCores(node.getCores()); return pdClient.storeHeartbeat(stats.build()); - } catch (PDException e){ + } catch (PDException e) { throw e; } catch (Exception e) { LOG.warn("Store {} report heartbeat exception: {}", node.getId(), e.toString()); } return Metapb.ClusterStats.newBuilder() - .setState(Metapb.ClusterState.Cluster_Fault).build(); + .setState(Metapb.ClusterState.Cluster_Fault).build(); } private void handleCommonException(Exception e) { } - public GraphManager getGraphManager(){ + public GraphManager getGraphManager() { return graphManager; } + public void setGraphManager(GraphManager graphManager) { this.graphManager = graphManager; } diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/pd/FakePdServiceProvider.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/pd/FakePdServiceProvider.java similarity index 65% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/pd/FakePdServiceProvider.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/pd/FakePdServiceProvider.java index f358041ba..a06299c74 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/pd/FakePdServiceProvider.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/pd/FakePdServiceProvider.java @@ -1,17 +1,22 @@ -package com.baidu.hugegraph.store.pd; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hugegraph.store.pd; -import com.baidu.hugegraph.pd.client.PDClient; -import com.baidu.hugegraph.pd.common.PDException; -import com.baidu.hugegraph.pd.common.PartitionUtils; -import com.baidu.hugegraph.pd.grpc.MetaTask; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.pd.grpc.Pdpb; -import com.baidu.hugegraph.store.meta.GraphManager; -import com.baidu.hugegraph.store.options.HgStoreEngineOptions; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.meta.Store; -import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -20,15 +25,30 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; +import org.apache.hugegraph.store.meta.GraphManager; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.meta.Store; +import org.apache.hugegraph.store.options.HgStoreEngineOptions; + +import com.baidu.hugegraph.pd.client.PDClient; +import com.baidu.hugegraph.pd.common.PDException; +import com.baidu.hugegraph.pd.common.PartitionUtils; +import com.baidu.hugegraph.pd.grpc.MetaTask; +import com.baidu.hugegraph.pd.grpc.Metapb; +import com.baidu.hugegraph.pd.grpc.Pdpb; + +import lombok.extern.slf4j.Slf4j; + /** * 内置PD服务，用于单机部署或开发调试 + * * @author liyan75 */ @Slf4j public class FakePdServiceProvider implements PdProvider { - private Map stores; + private final Map stores; private int partitionCount = 0; - private int shardCount = 0; + private final int shardCount = 0; private GraphManager graphManager = null; public FakePdServiceProvider(HgStoreEngineOptions.FakePdOptions options) { @@ -45,7 +65,7 @@ public class FakePdServiceProvider implements PdProvider { this.partitionCount = options.getPartitionCount(); } - private void addStore(String storeAddr, String raftAddr){ + private void addStore(String storeAddr, String raftAddr) { Store store = new Store() {{ setId(makeStoreId(storeAddr)); setRaftAddress(raftAddr); @@ -58,17 +78,20 @@ public class FakePdServiceProvider implements PdProvider { stores.put(store.getId(), store); } - public static long makeStoreId(String storeAddress){ + public static long makeStoreId(String storeAddress) { return storeAddress.hashCode(); } @Override public long registerStore(Store store) throws PDException { - log.info("registerStore storeId:{}, storeAddress:{}", store.getId(), store.getStoreAddress()); + log.info("registerStore storeId:{}, storeAddress:{}", store.getId(), + store.getStoreAddress()); // id 不匹配，禁止登录 - if ( store.getId() != 0 && store.getId() != makeStoreId(store.getStoreAddress())) - throw new PDException(Pdpb.ErrorType.STORE_ID_NOT_EXIST_VALUE, "Store id does not matched"); + if (store.getId() != 0 && store.getId() != makeStoreId(store.getStoreAddress())) { + throw new PDException(Pdpb.ErrorType.STORE_ID_NOT_EXIST_VALUE, + "Store id does not matched"); + } if (!stores.containsKey(makeStoreId(store.getStoreAddress()))) { store.setId(makeStoreId(store.getStoreAddress())); @@ -80,33 +103,37 @@ public class FakePdServiceProvider implements PdProvider { return store.getId(); } - private Map partitions = new ConcurrentHashMap<>(); + private final Map partitions = new ConcurrentHashMap<>(); + @Override public Partition getPartitionByID(String graph, int partId) { List storeList = new ArrayList(stores.values()); int shardCount = this.shardCount; - if (shardCount == 0 || shardCount >= stores.size()) + if (shardCount == 0 || shardCount >= stores.size()) { shardCount = stores.size(); + } int storeIdx = partId % storeList.size(); List shards = new ArrayList<>(); for (int i = 0; i < shardCount; i++) { - Metapb.Shard shard = Metapb.Shard.newBuilder().setStoreId(storeList.get(storeIdx).getId()) - .setRole(i == 0 ? Metapb.ShardRole.Leader : Metapb.ShardRole.Follower) // - .build(); + Metapb.Shard shard = + Metapb.Shard.newBuilder().setStoreId(storeList.get(storeIdx).getId()) + .setRole(i == 0 ? Metapb.ShardRole.Leader : + Metapb.ShardRole.Follower) // + .build(); shards.add(shard); storeIdx = (storeIdx + 1) >= storeList.size() ? 0 : ++storeIdx; // 顺序选择 } int partLength = getPartitionLength(); Metapb.Partition partition = Metapb.Partition.newBuilder() - .setGraphName(graph) - .setId(partId) - .setStartKey(partLength * partId) - .setEndKey(partLength * (partId + 1)) - //.addAllShards(shards) - .build(); + .setGraphName(graph) + .setId(partId) + .setStartKey(partLength * partId) + .setEndKey(partLength * (partId + 1)) + //.addAllShards(shards) + .build(); return new Partition(partition); } @@ -115,8 +142,9 @@ public class FakePdServiceProvider implements PdProvider { return null; } - private int getPartitionLength(){ - return PartitionUtils.MAX_VALUE / (partitionCount == 0 ? stores.size() : partitionCount) + 1; + private int getPartitionLength() { + return PartitionUtils.MAX_VALUE / (partitionCount == 0 ? stores.size() : partitionCount) + + 1; } @Override @@ -173,8 +201,8 @@ public class FakePdServiceProvider implements PdProvider { @Override public Metapb.Graph getGraph(String graphName) { return Metapb.Graph.newBuilder().setGraphName(graphName) - //.setId(PartitionUtils.calcHashcode(graphName.getBytes())) - .build(); + //.setId(PartitionUtils.calcHashcode(graphName.getBytes())) + .build(); } @Override @@ -196,7 +224,7 @@ public class FakePdServiceProvider implements PdProvider { @Override public Metapb.ClusterStats getClusterStats() { return Metapb.ClusterStats.newBuilder() - .setState(Metapb.ClusterState.Cluster_OK).build(); + .setState(Metapb.ClusterState.Cluster_OK).build(); } @Override @@ -213,9 +241,10 @@ public class FakePdServiceProvider implements PdProvider { /** * For unit test + * * @return */ - public static Store getDefaultStore(){ + public static Store getDefaultStore() { Store store = new Store(); store.setId(1); store.setStoreAddress("127.0.0.1:8501"); @@ -225,9 +254,10 @@ public class FakePdServiceProvider implements PdProvider { } - public GraphManager getGraphManager(){ + public GraphManager getGraphManager() { return graphManager; } + public void setGraphManager(GraphManager graphManager) { this.graphManager = graphManager; } diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/pd/PartitionInstructionListener.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/pd/PartitionInstructionListener.java new file mode 100644 index 000000000..a0252326e --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/pd/PartitionInstructionListener.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.pd; + + +import java.util.function.Consumer; + +import org.apache.hugegraph.store.meta.Partition; + +import com.baidu.hugegraph.pd.grpc.pulse.ChangeShard; +import com.baidu.hugegraph.pd.grpc.pulse.CleanPartition; +import com.baidu.hugegraph.pd.grpc.pulse.DbCompaction; +import com.baidu.hugegraph.pd.grpc.pulse.MovePartition; +import com.baidu.hugegraph.pd.grpc.pulse.PartitionKeyRange; +import com.baidu.hugegraph.pd.grpc.pulse.SplitPartition; +import com.baidu.hugegraph.pd.grpc.pulse.TransferLeader; + +public interface PartitionInstructionListener { + void onChangeShard(long taskId, Partition partition, ChangeShard changeShard, + Consumer consumer); + + void onTransferLeader(long taskId, Partition partition, TransferLeader transferLeader, + Consumer consumer); + + void onSplitPartition(long taskId, Partition partition, SplitPartition splitPartition, + Consumer consumer); + + void onDbCompaction(long taskId, Partition partition, DbCompaction rocksdbCompaction, + Consumer consumer); + + void onMovePartition(long taskId, Partition partition, MovePartition movePartition, + Consumer consumer); + + void onCleanPartition(long taskId, Partition partition, CleanPartition cleanPartition, + Consumer consumer); + + void onPartitionKeyRangeChanged(long taskId, Partition partition, + PartitionKeyRange partitionKeyRange, + Consumer consumer); +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/pd/PdProvider.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/pd/PdProvider.java similarity index 60% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/pd/PdProvider.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/pd/PdProvider.java index 0889eebf8..8cb75a044 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/pd/PdProvider.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/pd/PdProvider.java @@ -1,33 +1,58 @@ -package com.baidu.hugegraph.store.pd; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.pd; + +import java.util.List; +import java.util.function.Consumer; + +import org.apache.hugegraph.store.meta.GraphManager; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.meta.Store; +import org.apache.hugegraph.store.util.HgStoreException; import com.baidu.hugegraph.pd.client.PDClient; import com.baidu.hugegraph.pd.common.PDException; import com.baidu.hugegraph.pd.grpc.MetaTask; import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.meta.GraphManager; -import com.baidu.hugegraph.store.meta.ShardGroup; -import com.baidu.hugegraph.store.util.HgStoreException; -import com.baidu.hugegraph.store.meta.Store; -import com.baidu.hugegraph.store.meta.Partition; - -import java.util.List; -import java.util.function.Consumer; public interface PdProvider { long registerStore(Store store) throws PDException; + Store getStoreByID(Long storeId); + Metapb.ClusterStats getClusterStats(); Metapb.ClusterStats storeHeartbeat(Store node) throws HgStoreException, PDException; + Partition getPartitionByID(String graph, int partId); + Metapb.Shard getPartitionLeader(String graph, int partId); + Metapb.Partition getPartitionByCode(String graph, int code); Partition delPartition(String graph, int partId); - List updatePartition(List partitions) throws PDException; + + List updatePartition(List partitions) throws PDException; + List getPartitionsByStore(long storeId) throws PDException; void updatePartitionCache(Partition partition, Boolean changeLeader); + void invalidPartitionCache(String graph, int partId); boolean startHeartbeatStream(Consumer onError); @@ -38,25 +63,31 @@ public interface PdProvider { boolean isLocalPartition(long storeId, int partitionId); + Metapb.Graph getGraph(String graphName) throws PDException; void reportTask(MetaTask.Task task) throws PDException; PDClient getPDClient(); + boolean updatePartitionLeader(String graphName, int partId, long leaderStoreId); GraphManager getGraphManager(); + void setGraphManager(GraphManager graphManager); /** * 删除分区 shard group + * * @param groupId */ void deleteShardGroup(int groupId) throws PDException; - default Metapb.ShardGroup getShardGroup(int partitionId){ + + default Metapb.ShardGroup getShardGroup(int partitionId) { return null; } - default void updateShardGroup(Metapb.ShardGroup shardGroup) throws PDException { } + default void updateShardGroup(Metapb.ShardGroup shardGroup) throws PDException { + } } diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/HgStoreStateMachine.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/HgStoreStateMachine.java similarity index 75% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/HgStoreStateMachine.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/raft/HgStoreStateMachine.java index 17e789ab0..5e6d673ca 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/HgStoreStateMachine.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/HgStoreStateMachine.java @@ -1,9 +1,32 @@ -package com.baidu.hugegraph.store.raft; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.raft; import java.util.Base64; import java.util.List; -import java.util.concurrent.*; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hugegraph.store.snapshot.HgSnapshotHandler; +import org.apache.hugegraph.store.util.HgStoreException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.Iterator; import com.alipay.sofa.jraft.Status; @@ -16,14 +39,10 @@ import com.alipay.sofa.jraft.error.RaftException; import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; import com.alipay.sofa.jraft.util.Utils; -import com.baidu.hugegraph.store.util.HgStoreException; -import com.baidu.hugegraph.store.snapshot.HgSnapshotHandler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Raft 状态机 - * + * * @author Yanjinbing */ @@ -31,8 +50,8 @@ public class HgStoreStateMachine extends StateMachineAdapter { private static final Logger LOG = LoggerFactory.getLogger(HgStoreStateMachine.class); private final AtomicLong leaderTerm = new AtomicLong(-1); private final HgSnapshotHandler snapshotHandler; - private List taskHandlers; - private List stateListeners; + private final List taskHandlers; + private final List stateListeners; private final Integer groupId; private long committedIndex; @@ -43,10 +62,14 @@ public class HgStoreStateMachine extends StateMachineAdapter { this.taskHandlers = new CopyOnWriteArrayList<>(); } - public void addTaskHandler(RaftTaskHandler handler){ taskHandlers.add(handler); } + public void addTaskHandler(RaftTaskHandler handler) { + taskHandlers.add(handler); + } + public void addStateListener(RaftStateListener listener) { stateListeners.add(listener); } + public boolean isLeader() { return this.leaderTerm.get() > 0; } @@ -60,21 +83,24 @@ public class HgStoreStateMachine extends StateMachineAdapter { for (RaftTaskHandler taskHandler : taskHandlers) { if (done != null) { // Leader分支，本地调用 - if (taskHandler.invoke(groupId, done.op.getOp(), done.op.getReq(), done.closure)) { + if (taskHandler.invoke(groupId, done.op.getOp(), done.op.getReq(), + done.closure)) { done.run(Status.OK()); break; } } else { - if (taskHandler.invoke(groupId, inter.getData().array(), null)) + if (taskHandler.invoke(groupId, inter.getData().array(), null)) { break; + } } } } catch (Throwable t) { LOG.info("{}", Base64.getEncoder().encode(inter.getData().array())); LOG.error("StateMachine{} meet critical error: .", groupId, t); if (done != null) { - LOG.error("StateMachine meet critical error: op = {} {}.", done.op.getOp(), done.op.getReq()); - // done.run(new Status(RaftError.EINTERNAL, t.getMessage())); + LOG.error("StateMachine meet critical error: op = {} {}.", done.op.getOp(), + done.op.getReq()); + // done.run(new Status(RaftError.EINTERNAL, t.getMessage())); } } committedIndex = inter.getIndex(); @@ -83,26 +109,32 @@ public class HgStoreStateMachine extends StateMachineAdapter { listener.onDataCommitted(committedIndex); }); // 清理数据 - if ( done != null ) done.clear(); + if (done != null) done.clear(); // 遍历下一条 inter.next(); } } - public long getCommittedIndex(){ return committedIndex; } + public long getCommittedIndex() { + return committedIndex; + } - public long getLeaderTerm(){ return leaderTerm.get(); } + public long getLeaderTerm() { + return leaderTerm.get(); + } @Override public void onError(final RaftException e) { LOG.error("Raft {} StateMachine on error {}", groupId, e); Utils.runInThread(() -> { - stateListeners.forEach(listener -> { listener.onError(e); }); + stateListeners.forEach(listener -> { + listener.onError(e); + }); }); } @Override - public void onShutdown(){ + public void onShutdown() { super.onShutdown(); } @@ -111,7 +143,9 @@ public class HgStoreStateMachine extends StateMachineAdapter { this.leaderTerm.set(term); super.onLeaderStart(term); Utils.runInThread(() -> { - stateListeners.forEach(listener -> { listener.onLeaderStart(term); }); + stateListeners.forEach(listener -> { + listener.onLeaderStart(term); + }); }); LOG.info("Raft {} becomes leader ", groupId); } @@ -119,7 +153,9 @@ public class HgStoreStateMachine extends StateMachineAdapter { @Override public void onLeaderStop(final Status status) { Utils.runInThread(() -> { - stateListeners.forEach(listener -> { listener.onLeaderStop(this.leaderTerm.get());}); + stateListeners.forEach(listener -> { + listener.onLeaderStop(this.leaderTerm.get()); + }); }); this.leaderTerm.set(-1); super.onLeaderStop(status); @@ -149,14 +185,13 @@ public class HgStoreStateMachine extends StateMachineAdapter { } - @Override public void onConfigurationCommitted(final Configuration conf) { - stateListeners.forEach(listener->{ + stateListeners.forEach(listener -> { Utils.runInThread(() -> { try { listener.onConfigurationCommitted(conf); - }catch (Exception e){ + } catch (Exception e) { LOG.error("Raft {} onConfigurationCommitted {}", groupId, e); } }); @@ -210,7 +245,7 @@ public class HgStoreStateMachine extends StateMachineAdapter { public static class RaftClosureAdapter implements RaftClosure { private RaftOperation op; - private RaftClosure closure; + private final RaftClosure closure; public RaftClosureAdapter(RaftOperation op, RaftClosure closure) { this.op = op; @@ -226,7 +261,7 @@ public class HgStoreStateMachine extends StateMachineAdapter { return closure; } - public void clear(){ + public void clear() { op = null; } } diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftClosure.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftClosure.java new file mode 100644 index 000000000..50390bd4e --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftClosure.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.raft; + +import com.alipay.sofa.jraft.Closure; + +public interface RaftClosure extends Closure { + default void onLeaderChanged(Integer partId, Long storeId) { + } + +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/RaftOperation.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftOperation.java similarity index 59% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/RaftOperation.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftOperation.java index d2a71f54d..ec989ed59 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/RaftOperation.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftOperation.java @@ -1,10 +1,21 @@ -package com.baidu.hugegraph.store.raft; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.caucho.hessian.io.Hessian2Input; -import com.caucho.hessian.io.Hessian2Output; -import com.google.protobuf.CodedOutputStream; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.raft; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -13,28 +24,35 @@ import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.caucho.hessian.io.Hessian2Input; +import com.caucho.hessian.io.Hessian2Output; +import com.google.protobuf.CodedOutputStream; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + @Slf4j @Data public class RaftOperation { private static final Logger LOG = LoggerFactory.getLogger(RaftOperation.class); - public static final byte SYNC_PARTITION_TASK = 0x60; - public static final byte SYNC_PARTITION = 0x61; - public static final byte BLANK_TASK = 0x62; - public static final byte DO_SNAPSHOT = 0x63; + public static final byte SYNC_PARTITION_TASK = 0x60; + public static final byte SYNC_PARTITION = 0x61; + public static final byte BLANK_TASK = 0x62; + public static final byte DO_SNAPSHOT = 0x63; // 集群内部数据迁移操作 - public static final byte IN_WRITE_OP = 0x64; - public static final byte IN_CLEAN_OP = 0x65; + public static final byte IN_WRITE_OP = 0x64; + public static final byte IN_CLEAN_OP = 0x65; - public static final byte RAFT_UPDATE_PARTITION = 0x66; + public static final byte RAFT_UPDATE_PARTITION = 0x66; - public static final byte DB_COMPACTION = 0x67; + public static final byte DB_COMPACTION = 0x67; - final static byte[] EMPTY_Bytes = new byte[0]; + final static byte[] EMPTY_Bytes = new byte[0]; - private byte[] values; // req序列化的结果，用于传输给其他raft node - private Object req; // 原始对象，用于本机处理，减少一次反序列化操作 - private byte op; // 操作类型 + private byte[] values; // req序列化的结果，用于传输给其他raft node + private Object req; // 原始对象，用于本机处理，减少一次反序列化操作 + private byte op; // 操作类型 public static RaftOperation create(final byte op) { try { @@ -43,13 +61,13 @@ public class RaftOperation { operation.setReq(null); operation.setValues(toByteArray(op)); return operation; - }catch (Exception e){ + } catch (Exception e) { LOG.error("create error", e); return null; } } - public static RaftOperation create(final byte op, final byte[] values, final Object req){ + public static RaftOperation create(final byte op, final byte[] values, final Object req) { RaftOperation operation = new RaftOperation(); operation.setOp(op); operation.setReq(req); @@ -57,20 +75,22 @@ public class RaftOperation { return operation; } - public static RaftOperation create(final byte op, final Object req){ + public static RaftOperation create(final byte op, final Object req) { try { RaftOperation operation = new RaftOperation(); operation.setOp(op); operation.setReq(req); operation.setValues(toByteArray(op, req)); return operation; - }catch (Exception e){ + } catch (Exception e) { log.error("exception ", e); } return null; } + public static RaftOperation create(final byte op, - final com.google.protobuf.GeneratedMessageV3 req) throws IOException { + final com.google.protobuf.GeneratedMessageV3 req) throws + IOException { // 序列化， final byte[] buffer = new byte[req.getSerializedSize() + 1]; final CodedOutputStream output = CodedOutputStream.newInstance(buffer); @@ -99,8 +119,9 @@ public class RaftOperation { } } - public static Object toObject(final byte[] bytes, int offset) throws IOException{ - try (ByteArrayInputStream bis = new ByteArrayInputStream(bytes, offset + 1, bytes.length - offset)) { + public static Object toObject(final byte[] bytes, int offset) throws IOException { + try (ByteArrayInputStream bis = new ByteArrayInputStream(bytes, offset + 1, + bytes.length - offset)) { Hessian2Input input = new Hessian2Input(bis); Object obj = input.readObject(); input.close(); diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/RaftStateListener.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftStateListener.java similarity index 69% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/RaftStateListener.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftStateListener.java index eacabf6b8..b4502789e 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/RaftStateListener.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftStateListener.java @@ -1,13 +1,25 @@ -package com.baidu.hugegraph.store.raft; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.raft; -import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.conf.Configuration; import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.error.RaftException; -import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; -import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; - -import java.io.File; public interface RaftStateListener { /** @@ -22,7 +34,8 @@ public interface RaftStateListener { * * @param oldTerm the old term */ - default void onLeaderStop(final long oldTerm){}; + default void onLeaderStop(final long oldTerm) { + } /** * This method is called when a follower or candidate starts following a leader and its leaderId @@ -37,7 +50,8 @@ public interface RaftStateListener { * @param newLeaderId the new leader id whom the follower starts to follow * @param newTerm the new term */ - default void onStartFollowing(final PeerId newLeaderId, final long newTerm){}; + default void onStartFollowing(final PeerId newLeaderId, final long newTerm) { + } /** * This method is called when a follower stops following a leader and its leaderId becomes null, @@ -54,7 +68,8 @@ public interface RaftStateListener { * @param oldLeaderId the old leader id whom the follower followed before * @param oldTerm the old term */ - default void onStopFollowing(final PeerId oldLeaderId, final long oldTerm){}; + default void onStopFollowing(final PeerId oldLeaderId, final long oldTerm) { + } /** @@ -62,8 +77,11 @@ public interface RaftStateListener { * * @param conf committed configuration */ - default void onConfigurationCommitted(final Configuration conf){}; + default void onConfigurationCommitted(final Configuration conf) { + } + + default void onDataCommitted(long index) { + } - default void onDataCommitted(long index){}; void onError(final RaftException e); } diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftTaskHandler.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftTaskHandler.java new file mode 100644 index 000000000..9aca94931 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/RaftTaskHandler.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.raft; + +import org.apache.hugegraph.store.util.HgStoreException; + +/** + * 接收raft发送的数据 + */ +public interface RaftTaskHandler { + boolean invoke(final int groupId, final byte[] request, RaftClosure response) throws + HgStoreException; + + boolean invoke(final int groupId, final byte methodId, final Object req, + RaftClosure response) throws HgStoreException; +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/util/RaftUtils.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/util/RaftUtils.java similarity index 61% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/util/RaftUtils.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/raft/util/RaftUtils.java index cddde68f3..f644dc1bd 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/raft/util/RaftUtils.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/util/RaftUtils.java @@ -1,14 +1,32 @@ -package com.baidu.hugegraph.store.raft.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.alipay.sofa.jraft.Node; -import com.alipay.sofa.jraft.conf.Configuration; -import org.apache.commons.collections.ListUtils; +package org.apache.hugegraph.store.raft.util; import java.util.ArrayList; import java.util.List; +import org.apache.commons.collections.ListUtils; + +import com.alipay.sofa.jraft.Node; +import com.alipay.sofa.jraft.conf.Configuration; + public class RaftUtils { - public static List getAllEndpoints(Node node){ + public static List getAllEndpoints(Node node) { List endPoints = new ArrayList<>(); node.listPeers().forEach(peerId -> { endPoints.add(peerId.getEndpoint().toString()); @@ -18,7 +36,8 @@ public class RaftUtils { }); return endPoints; } - public static List getAllEndpoints(Configuration conf){ + + public static List getAllEndpoints(Configuration conf) { List endPoints = new ArrayList<>(); conf.listPeers().forEach(peerId -> { endPoints.add(peerId.getEndpoint().toString()); @@ -28,7 +47,8 @@ public class RaftUtils { }); return endPoints; } - public static List getPeerEndpoints(Node node){ + + public static List getPeerEndpoints(Node node) { List endPoints = new ArrayList<>(); node.listPeers().forEach(peerId -> { endPoints.add(peerId.getEndpoint().toString()); @@ -36,7 +56,7 @@ public class RaftUtils { return endPoints; } - public static List getPeerEndpoints(Configuration conf){ + public static List getPeerEndpoints(Configuration conf) { List endPoints = new ArrayList<>(); conf.listPeers().forEach(peerId -> { endPoints.add(peerId.getEndpoint().toString()); @@ -44,22 +64,24 @@ public class RaftUtils { return endPoints; } - public static List getLearnerEndpoints(Node node){ + public static List getLearnerEndpoints(Node node) { List endPoints = new ArrayList<>(); node.listLearners().forEach(peerId -> { endPoints.add(peerId.getEndpoint().toString()); }); return endPoints; } - public static List getLearnerEndpoints(Configuration conf){ + + public static List getLearnerEndpoints(Configuration conf) { List endPoints = new ArrayList<>(); conf.listLearners().forEach(peerId -> { endPoints.add(peerId.getEndpoint().toString()); }); return endPoints; } - public static boolean configurationEquals(Configuration oldConf, Configuration newConf){ + + public static boolean configurationEquals(Configuration oldConf, Configuration newConf) { return ListUtils.isEqualList(oldConf.listPeers(), newConf.listPeers()) && - ListUtils.isEqualList(oldConf.listLearners(), newConf.listLearners()); + ListUtils.isEqualList(oldConf.listLearners(), newConf.listLearners()); } } diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/snapshot/HgSnapshotHandler.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/snapshot/HgSnapshotHandler.java similarity index 76% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/snapshot/HgSnapshotHandler.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/snapshot/HgSnapshotHandler.java index 2b4ff97c4..3fe3d6a8e 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/snapshot/HgSnapshotHandler.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/snapshot/HgSnapshotHandler.java @@ -1,17 +1,21 @@ -package com.baidu.hugegraph.store.snapshot; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.alipay.sofa.jraft.entity.LocalFileMetaOutter; -import com.alipay.sofa.jraft.storage.snapshot.Snapshot; -import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; -import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; -import com.alipay.sofa.jraft.util.CRC64; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.PartitionEngine; -import com.baidu.hugegraph.store.business.BusinessHandler; -import com.baidu.hugegraph.store.util.HgStoreException; -import com.baidu.hugegraph.store.meta.Partition; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; +package org.apache.hugegraph.store.snapshot; import java.io.File; import java.io.IOException; @@ -22,6 +26,21 @@ import java.util.List; import java.util.Map; import java.util.zip.Checksum; +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.store.PartitionEngine; +import org.apache.hugegraph.store.business.BusinessHandler; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.util.HgStoreException; + +import com.alipay.sofa.jraft.entity.LocalFileMetaOutter; +import com.alipay.sofa.jraft.storage.snapshot.Snapshot; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; +import com.alipay.sofa.jraft.util.CRC64; +import com.baidu.hugegraph.pd.grpc.Metapb; + +import lombok.extern.slf4j.Slf4j; + @Slf4j public class HgSnapshotHandler { @@ -47,7 +66,7 @@ public class HgSnapshotHandler { public void onSnapshotSave(final SnapshotWriter writer) throws HgStoreException { final String snapshotDir = writer.getPath(); - if (partitionEngine != null){ + if (partitionEngine != null) { // rocks db snapshot final String graphSnapshotDir = snapshotDir + File.separator + SNAPSHOT_DATA_PATH; businessHandler.saveSnapshot(graphSnapshotDir, "", partitionEngine.getGroupId()); @@ -62,9 +81,10 @@ public class HgSnapshotHandler { for (String file : files) { String checksum = calculateChecksum(writer.getPath() + File.separator + file); if (checksum.length() != 0) { - LocalFileMetaOutter.LocalFileMeta meta = LocalFileMetaOutter.LocalFileMeta.newBuilder() - .setChecksum(checksum) - .build(); + LocalFileMetaOutter.LocalFileMeta meta = + LocalFileMetaOutter.LocalFileMeta.newBuilder() + .setChecksum(checksum) + .build(); writer.addFile(file, meta); } else { writer.addFile(file); @@ -116,7 +136,8 @@ public class HgSnapshotHandler { } } - public void onSnapshotLoad(final SnapshotReader reader, long committedIndex) throws HgStoreException { + public void onSnapshotLoad(final SnapshotReader reader, long committedIndex) throws + HgStoreException { final String snapshotDir = reader.getPath(); // 本地保存的快照没必要加载 @@ -128,23 +149,25 @@ public class HgSnapshotHandler { // 直接使用 snapshot final String graphSnapshotDir = snapshotDir + File.separator + SNAPSHOT_DATA_PATH; log.info("Raft {} begin loadSnapshot, {}", partitionEngine.getGroupId(), graphSnapshotDir); - businessHandler.loadSnapshot(graphSnapshotDir, "", partitionEngine.getGroupId(), committedIndex); + businessHandler.loadSnapshot(graphSnapshotDir, "", partitionEngine.getGroupId(), + committedIndex); log.info("Raft {} end loadSnapshot.", partitionEngine.getGroupId()); - for (Metapb.Partition snapPartition : partitionEngine.loadPartitionsFromLocalDb()){ + for (Metapb.Partition snapPartition : partitionEngine.loadPartitionsFromLocalDb()) { log.info("onSnapshotLoad loaded partition from local db. Partition: {}", snapPartition); partitionEngine.loadPartitionFromSnapshot(new Partition(snapPartition)); Partition partition = partitionEngine.getPartition(snapPartition.getGraphName()); if (partition == null) { log.warn("skip to load snapshot for {}-{}, it is not belong to this node", - snapPartition.getGraphName(), snapPartition.getId()); + snapPartition.getGraphName(), snapPartition.getId()); continue; } var taskManager = partitionEngine.getTaskManager(); // async tasks - for (var task : taskManager.scanAsyncTasks(partitionEngine.getGroupId(), snapPartition.getGraphName()) ){ + for (var task : taskManager.scanAsyncTasks(partitionEngine.getGroupId(), + snapPartition.getGraphName())) { task.handleTask(); } } @@ -171,7 +194,8 @@ public class HgSnapshotHandler { if (fs != null) { for (File f : fs) { if (f.isFile()) { - files.add(trimStartPath(dir.getPath(), rootDir.getPath()) + File.separator + f.getName()); + files.add(trimStartPath(dir.getPath(), rootDir.getPath()) + File.separator + + f.getName()); } else { findFileList(f, rootDir, files); } @@ -188,9 +212,11 @@ public class HgSnapshotHandler { String shouldNotLoadPath = getShouldNotLoadPath(snapshot); try { FileUtils.writeStringToFile(new File(shouldNotLoadPath), - saveSnapshot ? "saved snapshot" : "loaded snapshot", Charset.defaultCharset()); + saveSnapshot ? "saved snapshot" : "loaded snapshot", + Charset.defaultCharset()); } catch (IOException e) { - log.error("Failed to create snapshot should not load flag file {}. {}", shouldNotLoadPath, e); + log.error("Failed to create snapshot should not load flag file {}. {}", + shouldNotLoadPath, e); } } diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/Asserts.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/Asserts.java similarity index 51% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/util/Asserts.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/util/Asserts.java index 6dabfa3aa..c227e3131 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/Asserts.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/Asserts.java @@ -1,11 +1,28 @@ -package com.baidu.hugegraph.store.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.util; /** * @author lynn.bond@hotmail.com */ public final class Asserts { public static void isTrue(boolean expression, String message) { - if (message== null) throw new IllegalArgumentException("message is null"); + if (message == null) throw new IllegalArgumentException("message is null"); if (!expression) { throw new IllegalArgumentException(message); } @@ -41,5 +58,4 @@ public final class Asserts { } - } \ No newline at end of file diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/CopyOnWriteCache.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/CopyOnWriteCache.java similarity index 73% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/util/CopyOnWriteCache.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/util/CopyOnWriteCache.java index f466f3e67..91bb25307 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/CopyOnWriteCache.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/CopyOnWriteCache.java @@ -1,6 +1,27 @@ -package com.baidu.hugegraph.store.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import java.util.*; +package org.apache.hugegraph.store.util; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -11,6 +32,7 @@ public class CopyOnWriteCache implements ConcurrentMap { private volatile Map map; ScheduledExecutorService scheduledExecutor; + public CopyOnWriteCache(long effectiveTime) { this.map = Collections.emptyMap(); scheduledExecutor = Executors.newScheduledThreadPool(1); @@ -18,6 +40,7 @@ public class CopyOnWriteCache implements ConcurrentMap { this.clear(); }, effectiveTime, effectiveTime, TimeUnit.MILLISECONDS); } + @Override public boolean containsKey(Object k) { return map.containsKey(k); @@ -88,10 +111,11 @@ public class CopyOnWriteCache implements ConcurrentMap { @Override public synchronized V putIfAbsent(K k, V v) { - if (!containsKey(k)) + if (!containsKey(k)) { return put(k, v); - else + } else { return get(k); + } } @Override diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/FutureClosure.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/FutureClosure.java similarity index 62% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/util/FutureClosure.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/util/FutureClosure.java index d15b8d8aa..1632c1c6d 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/FutureClosure.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/FutureClosure.java @@ -1,19 +1,37 @@ -package com.baidu.hugegraph.store.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.alipay.sofa.jraft.Closure; -import com.alipay.sofa.jraft.Status; -import com.alipay.sofa.jraft.error.RaftError; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +package org.apache.hugegraph.store.util; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.error.RaftError; + public class FutureClosure implements Closure, Future { private static final Logger LOG = LoggerFactory.getLogger(FutureClosure.class); - private CountDownLatch latch; + private final CountDownLatch latch; private Status status; public FutureClosure() { @@ -54,7 +72,7 @@ public class FutureClosure implements Closure, Future { } @Override - public Status get(){ + public Status get() { try { latch.await(); } catch (InterruptedException e) { @@ -64,7 +82,7 @@ public class FutureClosure implements Closure, Future { } @Override - public Status get(long timeout, TimeUnit unit){ + public Status get(long timeout, TimeUnit unit) { try { latch.await(timeout, unit); } catch (InterruptedException e) { diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/HgRaftError.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/HgRaftError.java similarity index 59% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/util/HgRaftError.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/util/HgRaftError.java index 79e09cb3f..f6dc308d0 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/HgRaftError.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/HgRaftError.java @@ -1,11 +1,28 @@ -package com.baidu.hugegraph.store.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.alipay.sofa.jraft.Status; +package org.apache.hugegraph.store.util; import java.util.HashMap; import java.util.Map; -public enum HgRaftError{ +import com.alipay.sofa.jraft.Status; + +public enum HgRaftError { UNKNOWN(-1, "unknown"), OK(0, "OK"), NOT_LEADER(20000, "This partition is not leader"), diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/util/HgStoreConst.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/HgStoreConst.java new file mode 100644 index 000000000..3e1dcb824 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/HgStoreConst.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.util; + +import java.util.Collections; +import java.util.List; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/22 + */ +public final class HgStoreConst { + public final static byte[] EMPTY_BYTES = new byte[0]; + + public static final List EMPTY_LIST = Collections.EMPTY_LIST; + + public final static int SCAN_ALL_PARTITIONS_ID = -1; // means scan all partitions. + + private HgStoreConst() { + } + +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/HgStoreException.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/HgStoreException.java similarity index 70% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/util/HgStoreException.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/util/HgStoreException.java index 366c57b0b..7f58a9cb1 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/HgStoreException.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/HgStoreException.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.util; /** * @projectName: hugegraph-store @@ -36,7 +53,7 @@ public class HgStoreException extends RuntimeException { public final static int EC_METRIC_FAIL = 1401; - private int code; + private final int code; public HgStoreException() { super(); @@ -52,6 +69,7 @@ public class HgStoreException extends RuntimeException { super(message); this.code = exceptCode; } + public HgStoreException(int exceptCode, Throwable cause) { super(codeToMsg(exceptCode), cause); this.code = exceptCode; @@ -71,7 +89,7 @@ public class HgStoreException extends RuntimeException { return this.code; } - public static String codeToMsg(int code){ + public static String codeToMsg(int code) { return "errorCode = " + code; } } diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/IpUtil.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/IpUtil.java similarity index 66% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/util/IpUtil.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/util/IpUtil.java index 55cee5a54..4040fa4ec 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/IpUtil.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/IpUtil.java @@ -1,6 +1,21 @@ -package com.baidu.hugegraph.store.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.util; import java.net.Inet4Address; import java.net.InetAddress; @@ -13,10 +28,13 @@ import java.util.LinkedList; import java.util.List; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; + @Slf4j public class IpUtil { /** * 获取所有的ipv4 地址 + * * @return all ipv4 addr * @throws SocketException io error or no network interface */ @@ -42,9 +60,9 @@ public class IpUtil { * @param raftAddress raft addr * @return raft addr that have the nearest distance with given param */ - public static String getNearestAddress(String raftAddress){ - try{ - List ipv4s = getIpAddress() ; + public static String getNearestAddress(String raftAddress) { + try { + List ipv4s = getIpAddress(); String[] tmp = raftAddress.split(":"); if (ipv4s.size() == 0) { throw new Exception("no available ipv4"); @@ -55,16 +73,16 @@ public class IpUtil { } var raftSeg = Arrays.stream(tmp[0].split("\\.")) - .map(s -> Integer.parseInt(s)) - .collect(Collectors.toList()); + .map(s -> Integer.parseInt(s)) + .collect(Collectors.toList()); ipv4s.sort(Comparator.comparingLong(ip -> { String[] ipSegments = ip.split("\\."); long base = 256 * 256 * 256; int i = 0; long sum = 0; - for (String seg: ipSegments){ - sum += base * (Math.abs(raftSeg.get(i) - Integer.parseInt(seg))) ; + for (String seg : ipSegments) { + sum += base * (Math.abs(raftSeg.get(i) - Integer.parseInt(seg))); base = base / 256; i += 1; } @@ -72,9 +90,9 @@ public class IpUtil { })); return ipv4s.get(0) + ":" + tmp[1]; - }catch (SocketException e){ + } catch (SocketException e) { log.error("getIpAddress, get ip failed, {}", e.getMessage()); - } catch (Exception e){ + } catch (Exception e) { log.error("getRaftAddress, got exception, {}", e.getMessage()); } return raftAddress; diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/util/Lifecycle.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/Lifecycle.java new file mode 100644 index 000000000..ec6a9d078 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/Lifecycle.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.util; + +/** + * @author YanJinbing + */ +public interface Lifecycle { + + /** + * Initialize the service. + * + * @return true when successes. + */ + boolean init(final T opts); + + /** + * Dispose the resources for service. + */ + void shutdown(); +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/ManualResetEvent.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/ManualResetEvent.java similarity index 64% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/util/ManualResetEvent.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/util/ManualResetEvent.java index d40c99fd9..a477a2a8a 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/ManualResetEvent.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/ManualResetEvent.java @@ -1,4 +1,21 @@ -package com.baidu.hugegraph.store.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.util; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -15,8 +32,9 @@ public class ManualResetEvent { /** * Initializes a new instance of the System.Threading.ManualResetEvent class * with a Boolean value indicating whether to set the initial state to signaled. + * * @param signalled true to set the initial state to signaled; false to set the initial state - * to nonsignaled. + * to nonsignaled. */ public ManualResetEvent(boolean signalled) { if (signalled) { @@ -46,6 +64,7 @@ public class ManualResetEvent { /** * Blocks the current thread until the current wait handle receives a signal. + * * @throws InterruptedException */ public void waitOne() throws InterruptedException { @@ -54,12 +73,13 @@ public class ManualResetEvent { /** * Blocks the current thread until the current wait handle receives a signal. + * * @param timeout the maximum time to wait - * @param unit the time unit of the {@code timeout} argument + * @param unit the time unit of the {@code timeout} argument * @return {@code true} if the count reached zero and {@code false} - * if the waiting time elapsed before the count reached zero + * if the waiting time elapsed before the count reached zero * @throws InterruptedException if the current thread is interrupted - * while waiting + * while waiting */ public boolean waitOne(int timeout, TimeUnit unit) throws InterruptedException { return event.await(timeout, unit); @@ -67,6 +87,7 @@ public class ManualResetEvent { /** * Check if the handle was signalled + * * @return Boolean state */ public boolean isSignalled() { diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/util/PartitionMetaStoreWrapper.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/PartitionMetaStoreWrapper.java new file mode 100644 index 000000000..1d3e3debf --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/PartitionMetaStoreWrapper.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.util; + +import java.util.List; + +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.meta.base.MetaStoreBase; + +public class PartitionMetaStoreWrapper { + + private static final InnerMetaStore store = new InnerMetaStore(); + + + public void put(int partitionId, byte[] key, byte[] value) { + store.setPartitionId(partitionId); + store.put(key, value); + } + + public T get(int partitionId, byte[] key, com.google.protobuf.Parser parser) { + store.setPartitionId(partitionId); + return store.get(parser, key); + } + + public byte[] get(int partitionId, byte[] key) { + store.setPartitionId(partitionId); + return store.get(key); + } + + public void delete(int partitionId, byte[] key) { + store.setPartitionId(partitionId); + store.delete(key); + } + + public List scan(int partitionId, com.google.protobuf.Parser parser, byte[] prefix) { + store.setPartitionId(partitionId); + return store.scan(parser, prefix); + } + + private static class InnerMetaStore extends MetaStoreBase { + + private int partitionId; + + private void setPartitionId(int partitionId) { + this.partitionId = partitionId; + } + + @Override + protected RocksDBSession getRocksDBSession() { + return HgStoreEngine.getInstance().getBusinessHandler().getSession(this.partitionId); + } + + @Override + protected String getCFName() { + return "default"; + } + } +} diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/UnsafeUtf8Util.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/UnsafeUtf8Util.java similarity index 71% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/util/UnsafeUtf8Util.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/util/UnsafeUtf8Util.java index cae3fb513..48995feb7 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/UnsafeUtf8Util.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/UnsafeUtf8Util.java @@ -1,10 +1,32 @@ -package com.baidu.hugegraph.store.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.util; + +import static java.lang.Character.MAX_SURROGATE; +import static java.lang.Character.MIN_HIGH_SURROGATE; +import static java.lang.Character.MIN_LOW_SURROGATE; +import static java.lang.Character.MIN_SUPPLEMENTARY_CODE_POINT; +import static java.lang.Character.MIN_SURROGATE; +import static java.lang.Character.isSurrogatePair; +import static java.lang.Character.toCodePoint; import java.util.Arrays; -import static java.lang.Character.*; -import static java.lang.Character.isSurrogatePair; - /** * @projectName: hugegraph-store * @package: com.baidu.hugegraph.store.util @@ -40,7 +62,8 @@ public class UnsafeUtf8Util { if (utf8Length < utf16Length) { // Necessary and sufficient condition for overflow because of maximum 3x expansion - throw new IllegalArgumentException("UTF-8 length does not fit in int: " + (utf8Length + (1L << 32))); + throw new IllegalArgumentException( + "UTF-8 length does not fit in int: " + (utf8Length + (1L << 32))); } return utf8Length; } @@ -59,7 +82,8 @@ public class UnsafeUtf8Util { // Check that we have a well-formed surrogate pair. int cp = Character.codePointAt(sequence, i); if (cp < MIN_SUPPLEMENTARY_CODE_POINT) { - throw new IllegalArgumentException("Unpaired surrogate at index " + i + " of " + utf16Length); + throw new IllegalArgumentException( + "Unpaired surrogate at index " + i + " of " + utf16Length); } i++; } @@ -74,7 +98,8 @@ public class UnsafeUtf8Util { final int inLimit = in.length(); if (inLimit > length || out.length - length < offset) { // Not even enough room for an ASCII-encoded string. - throw new ArrayIndexOutOfBoundsException("Failed writing " + in.charAt(inLimit - 1) + " at index " + throw new ArrayIndexOutOfBoundsException( + "Failed writing " + in.charAt(inLimit - 1) + " at index " + (offset + length)); } @@ -102,11 +127,13 @@ public class UnsafeUtf8Util { UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & (c >>> 6)))); UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & c))); } else if (outIx <= outLimit - 4L) { - // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8 + // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four + // UTF-8 // bytes final char low; if (inIx + 1 == inLimit || !isSurrogatePair(c, (low = in.charAt(++inIx)))) { - throw new IllegalArgumentException("Unpaired surrogate at index " + (inIx - 1) + " of " + inLimit); + throw new IllegalArgumentException( + "Unpaired surrogate at index " + (inIx - 1) + " of " + inLimit); } int codePoint = toCodePoint(c, low); UnsafeUtil.putByte(out, outIx++, (byte) ((0xF << 4) | (codePoint >>> 18))); @@ -115,12 +142,14 @@ public class UnsafeUtf8Util { UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & codePoint))); } else { if ((MIN_SURROGATE <= c && c <= MAX_SURROGATE) - && (inIx + 1 == inLimit || !isSurrogatePair(c, in.charAt(inIx + 1)))) { + && (inIx + 1 == inLimit || !isSurrogatePair(c, in.charAt(inIx + 1)))) { // We are surrogates and we're not a surrogate pair. - throw new IllegalArgumentException("Unpaired surrogate at index " + inIx + " of " + inLimit); + throw new IllegalArgumentException( + "Unpaired surrogate at index " + inIx + " of " + inLimit); } // Not enough space in the output buffer. - throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + outIx); + throw new ArrayIndexOutOfBoundsException( + "Failed writing " + c + " at index " + outIx); } } @@ -129,17 +158,18 @@ public class UnsafeUtf8Util { } - public static String decodeUtf8(byte[] bytes, int index, int size) { if ((index | size | bytes.length - index - size) < 0) { - throw new ArrayIndexOutOfBoundsException("buffer length=" + bytes.length + ", index=" + index + ", size=" + throw new ArrayIndexOutOfBoundsException( + "buffer length=" + bytes.length + ", index=" + index + ", size=" + size); } int offset = index; final int limit = offset + size; - // The longest possible resulting String is the same as the number of input bytes, when it is + // The longest possible resulting String is the same as the number of input bytes, when + // it is // all ASCII. For other cases, this over-allocates and we will truncate in the end. char[] resultArr = new char[size]; int resultPos = 0; @@ -159,7 +189,8 @@ public class UnsafeUtf8Util { byte byte1 = UnsafeUtil.getByte(bytes, offset++); if (DecodeUtil.isOneByte(byte1)) { DecodeUtil.handleOneByte(byte1, resultArr, resultPos++); - // It's common for there to be multiple ASCII characters in a run mixed in, so add an + // It's common for there to be multiple ASCII characters in a run mixed in, so + // add an // extra optimized loop to take care of these runs. while (offset < limit) { byte b = UnsafeUtil.getByte(bytes, offset); @@ -173,8 +204,9 @@ public class UnsafeUtf8Util { if (offset >= limit) { throw invalidUtf8(); } - DecodeUtil.handleTwoBytes(byte1, /* byte2 */UnsafeUtil.getByte(bytes, offset++), resultArr, - resultPos++); + DecodeUtil.handleTwoBytes(byte1, /* byte2 */UnsafeUtil.getByte(bytes, offset++), + resultArr, + resultPos++); } else if (DecodeUtil.isThreeBytes(byte1)) { if (offset >= limit - 1) { throw invalidUtf8(); @@ -228,8 +260,10 @@ public class UnsafeUtf8Util { resultArr[resultPos] = (char) byte1; } - private static void handleTwoBytes(byte byte1, byte byte2, char[] resultArr, int resultPos) { - // Simultaneously checks for illegal trailing-byte in leading position (<= '11000000') and + private static void handleTwoBytes(byte byte1, byte byte2, char[] resultArr, + int resultPos) { + // Simultaneously checks for illegal trailing-byte in leading position (<= + // '11000000') and // overlong 2-byte, '11000001'. if (byte1 < (byte) 0xC2 || isNotTrailingByte(byte2)) { throw invalidUtf8(); @@ -237,33 +271,38 @@ public class UnsafeUtf8Util { resultArr[resultPos] = (char) (((byte1 & 0x1F) << 6) | trailingByteValue(byte2)); } - private static void handleThreeBytes(byte byte1, byte byte2, byte byte3, char[] resultArr, int resultPos) { + private static void handleThreeBytes(byte byte1, byte byte2, byte byte3, char[] resultArr, + int resultPos) { if (isNotTrailingByte(byte2) - // overlong? 5 most significant bits must not all be zero - || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) - // check for illegal surrogate codepoints - || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) || isNotTrailingByte(byte3)) { + // overlong? 5 most significant bits must not all be zero + || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) + // check for illegal surrogate codepoints + || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) || isNotTrailingByte(byte3)) { throw invalidUtf8(); } - resultArr[resultPos] = (char) (((byte1 & 0x0F) << 12) | (trailingByteValue(byte2) << 6) | trailingByteValue(byte3)); + resultArr[resultPos] = + (char) (((byte1 & 0x0F) << 12) | (trailingByteValue(byte2) << 6) | + trailingByteValue(byte3)); } - private static void handleFourBytes(byte byte1, byte byte2, byte byte3, byte byte4, char[] resultArr, + private static void handleFourBytes(byte byte1, byte byte2, byte byte3, byte byte4, + char[] resultArr, int resultPos) { if (isNotTrailingByte(byte2) - // Check that 1 <= plane <= 16. Tricky optimized form of: - // valid 4-byte leading byte? - // if (byte1 > (byte) 0xF4 || - // overlong? 4 most significant bits must not all be zero - // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || - // codepoint larger than the highest code point (U+10FFFF)? - // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) - || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 || isNotTrailingByte(byte3) - || isNotTrailingByte(byte4)) { + // Check that 1 <= plane <= 16. Tricky optimized form of: + // valid 4-byte leading byte? + // if (byte1 > (byte) 0xF4 || + // overlong? 4 most significant bits must not all be zero + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // codepoint larger than the highest code point (U+10FFFF)? + // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 || isNotTrailingByte(byte3) + || isNotTrailingByte(byte4)) { throw invalidUtf8(); } - int codePoint = ((byte1 & 0x07) << 18) | (trailingByteValue(byte2) << 12) | (trailingByteValue(byte3) << 6) - | trailingByteValue(byte4); + int codePoint = ((byte1 & 0x07) << 18) | (trailingByteValue(byte2) << 12) | + (trailingByteValue(byte3) << 6) + | trailingByteValue(byte4); resultArr[resultPos] = DecodeUtil.highSurrogate(codePoint); resultArr[resultPos + 1] = DecodeUtil.lowSurrogate(codePoint); } @@ -283,7 +322,8 @@ public class UnsafeUtf8Util { } private static char highSurrogate(int codePoint) { - return (char) ((MIN_HIGH_SURROGATE - (MIN_SUPPLEMENTARY_CODE_POINT >>> 10)) + (codePoint >>> 10)); + return (char) ((MIN_HIGH_SURROGATE - (MIN_SUPPLEMENTARY_CODE_POINT >>> 10)) + + (codePoint >>> 10)); } private static char lowSurrogate(int codePoint) { diff --git a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/UnsafeUtil.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/UnsafeUtil.java similarity index 88% rename from hg-store-core/src/main/java/com/baidu/hugegraph/store/util/UnsafeUtil.java rename to hg-store-core/src/main/java/org/apache/hugegraph/store/util/UnsafeUtil.java index e68bc744d..b4e25a6ea 100644 --- a/hg-store-core/src/main/java/com/baidu/hugegraph/store/util/UnsafeUtil.java +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/UnsafeUtil.java @@ -1,9 +1,26 @@ -package com.baidu.hugegraph.store.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import lombok.extern.slf4j.Slf4j; +package org.apache.hugegraph.store.util; import java.lang.reflect.Field; +import lombok.extern.slf4j.Slf4j; + /** * @projectName: hugegraph-store * @package: com.baidu.hugegraph.store.util @@ -15,11 +32,11 @@ import java.lang.reflect.Field; */ @Slf4j public class UnsafeUtil { - private static final Object UNSAFE = getUnsafe0(); - private static final UnsafeAccessor UNSAFE_ACCESSOR = getUnsafeAccessor0(); + private static final Object UNSAFE = getUnsafe0(); + private static final UnsafeAccessor UNSAFE_ACCESSOR = getUnsafeAccessor0(); - private static final long BYTE_ARRAY_BASE_OFFSET = arrayBaseOffset(byte[].class); - private static final long STRING_VALUE_OFFSET = objectFieldOffset(stringValueField()); + private static final long BYTE_ARRAY_BASE_OFFSET = arrayBaseOffset(byte[].class); + private static final long STRING_VALUE_OFFSET = objectFieldOffset(stringValueField()); public static boolean hasUnsafe() { return UNSAFE != null; @@ -42,7 +59,6 @@ public class UnsafeUtil { } - public static void putByte(final byte[] target, final long index, final byte value) { UNSAFE_ACCESSOR.putByte(target, BYTE_ARRAY_BASE_OFFSET + index, value); } @@ -57,7 +73,8 @@ public class UnsafeUtil { public static String moveToString(final char[] chars) { if (STRING_VALUE_OFFSET == -1) { - // In the off-chance that this JDK does not implement String as we'd expect, just do a copy. + // In the off-chance that this JDK does not implement String as we'd expect, just do + // a copy. return new String(chars); } final String str; @@ -79,7 +96,8 @@ public class UnsafeUtil { return field(String.class, "value", char[].class); } - private static Field field(final Class clazz, final String fieldName, final Class expectedType) { + private static Field field(final Class clazz, final String fieldName, + final Class expectedType) { Field field; try { field = clazz.getDeclaredField(fieldName); @@ -209,7 +227,8 @@ public class UnsafeUtil { this.unsafe.putLong(address, value); } - public void copyMemory(final Object srcBase, final long srcOffset, final Object dstBase, final long dstOffset, + public void copyMemory(final Object srcBase, final long srcOffset, final Object dstBase, + final long dstOffset, final long bytes) { this.unsafe.copyMemory(srcBase, srcOffset, dstBase, dstOffset, bytes); } @@ -254,7 +273,8 @@ public class UnsafeUtil { return this.unsafe.getBooleanVolatile(target, offset); } - public void putBooleanVolatile(final Object target, final long offset, final boolean value) { + public void putBooleanVolatile(final Object target, final long offset, + final boolean value) { this.unsafe.putBooleanVolatile(target, offset, value); } diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/util/Version.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/Version.java new file mode 100644 index 000000000..2ad2d36ad --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/Version.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.util; + +import java.io.InputStream; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class Version { + + private static String version = ""; + + /** + * 软件版本号 + * + * @return + */ + public static String getVersion() { + if (version.isEmpty()) { + try (InputStream is = Version.class.getResourceAsStream("/version.txt")) { + byte[] buf = new byte[64]; + int len = is.read(buf); + version = new String(buf, 0, len); + } catch (Exception e) { + log.error("Version.getVersion exception {}", e); + } + } + return version; + } + + /** + * 存储格式版本号 + * + * @return + */ + public static int getDataFmtVersion() { + return 1; + } +} diff --git a/hg-store-core/src/main/java/org/apache/hugegraph/store/util/ZipUtils.java b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/ZipUtils.java new file mode 100644 index 000000000..798e00e61 --- /dev/null +++ b/hg-store-core/src/main/java/org/apache/hugegraph/store/util/ZipUtils.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.util; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.zip.CheckedInputStream; +import java.util.zip.CheckedOutputStream; +import java.util.zip.Checksum; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.output.NullOutputStream; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public final class ZipUtils { + + public static void compress(final String rootDir, final String sourceDir, + final String outputFile, final Checksum checksum) throws + IOException { + try (final FileOutputStream fos = new FileOutputStream(outputFile); + final CheckedOutputStream cos = new CheckedOutputStream(fos, checksum); + final ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(cos))) { + ZipUtils.compressDirectoryToZipFile(rootDir, sourceDir, zos); + zos.flush(); + fos.getFD().sync(); + } + } + + private static void compressDirectoryToZipFile(final String rootDir, final String sourceDir, + final ZipOutputStream zos) throws IOException { + final String dir = Paths.get(rootDir, sourceDir).toString(); + final File[] files = new File(dir).listFiles(); + for (final File file : files) { + final String child = Paths.get(sourceDir, file.getName()).toString(); + if (file.isDirectory()) { + compressDirectoryToZipFile(rootDir, child, zos); + } else { + zos.putNextEntry(new ZipEntry(child)); + try (final FileInputStream fis = new FileInputStream(file); + final BufferedInputStream bis = new BufferedInputStream(fis)) { + IOUtils.copy(bis, zos); + } + } + } + } + + public static void decompress(final String sourceFile, final String outputDir, + final Checksum checksum) throws IOException { + try (final FileInputStream fis = new FileInputStream(sourceFile); + final CheckedInputStream cis = new CheckedInputStream(fis, checksum); + final ZipInputStream zis = new ZipInputStream(new BufferedInputStream(cis))) { + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + final String fileName = entry.getName(); + final File entryFile = new File(Paths.get(outputDir, fileName).toString()); + FileUtils.forceMkdir(entryFile.getParentFile()); + try (final FileOutputStream fos = new FileOutputStream(entryFile); + final BufferedOutputStream bos = new BufferedOutputStream(fos)) { + IOUtils.copy(zis, bos); + bos.flush(); + fos.getFD().sync(); + } + } + IOUtils.copy(cis, NullOutputStream.NULL_OUTPUT_STREAM); + } + } +} diff --git a/hg-store-core/src/test/java/com/baidu/hugegraph/store/PartitionEngineTest.java b/hg-store-core/src/test/java/com/baidu/hugegraph/store/PartitionEngineTest.java deleted file mode 100644 index fe05f18a2..000000000 --- a/hg-store-core/src/test/java/com/baidu/hugegraph/store/PartitionEngineTest.java +++ /dev/null @@ -1,185 +0,0 @@ -package com.baidu.hugegraph.store; - -import com.alipay.sofa.jraft.option.RpcOptions; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.cmd.HgCmdClient; -import com.baidu.hugegraph.store.meta.Partition; - -import org.junit.Assert; -import org.junit.Test; -// import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - -public class PartitionEngineTest { - - // @Test - public void test(){ - String[] peers = new String[]{"1", "2", "6"}; - String[] learners = new String[]{"3", "7"}; - List oldPeers1 = Arrays.asList(new String[]{"1", "2", "3"}); - List oldLearner1 = Arrays.asList(new String[]{"4", "5"}); - - - List oldPeers = oldPeers1.stream().map(peer -> peer.toString()).collect(Collectors.toList()); - oldPeers.addAll(oldLearner1.stream().map(peer -> peer.toString()).collect(Collectors.toList())); - - List addedNodes = Arrays.stream(peers).filter(peer -> - !oldPeers.stream().map(e -> e.toString()).collect(Collectors.toSet()).contains(peer)) - .collect(Collectors.toList()); - addedNodes.addAll(Arrays.stream(learners).filter(peer -> - !oldPeers.stream().map(e -> e.toString()).collect(Collectors.toSet()).contains(peer)) - .collect(Collectors.toList())); - - List removedNodes = oldPeers.stream().filter(peer -> - !Arrays.stream(peers).map(e -> e).collect(Collectors.toSet()).contains(peer)) - .collect(Collectors.toList()); - removedNodes = removedNodes.stream().filter(peer -> - !Arrays.stream(learners).map(e -> e).collect(Collectors.toSet()).contains(peer)) - .collect(Collectors.toList()); - List mixedPeer = oldPeers1.stream().filter(peer -> - Arrays.stream(learners).map(e -> e).collect(Collectors.toSet()).contains(peer)) - .collect(Collectors.toList()); - - // 新增 6、7 - Assert.assertEquals(2, addedNodes.size()); - addedNodes.clear(); - addedNodes.addAll(Arrays.asList(peers)); - addedNodes.addAll(Arrays.asList(learners)); - addedNodes.removeAll(oldPeers); - Assert.assertEquals(2, addedNodes.size()); - - addedNodes.forEach(s->System.out.print(s + " "));System.out.println(); - // 删除 4，5 - Assert.assertEquals(2, removedNodes.size()); - - removedNodes.clear(); - removedNodes.addAll(oldPeers); - removedNodes.removeAll(Arrays.asList(peers)); - removedNodes.removeAll(Arrays.asList(learners)); - Assert.assertEquals(2, removedNodes.size()); - removedNodes.forEach(s->System.out.print(s + " "));System.out.println(); - // 交集 5 - Assert.assertEquals(1, mixedPeer.size()); - oldPeers1.removeAll(Arrays.asList(learners)); - Assert.assertEquals(1, oldPeers1.size()); - mixedPeer.forEach(s->System.out.print(s + " ")); - - } - - - - // @Test - public void testPartition(){ - Partition p = new Partition(); - p.setId(1); - List shards = new ArrayList<>(); - shards.add(Metapb.Shard.newBuilder().build()); - // p.setShardsList(shards); - - Partition p2 = p.clone(); - p.setId(2); - Assert.assertNotEquals(p2.getId(), p.getId()); - - } - - // @Test - public void testUpdateShardsList() { - List curShards = new ArrayList<>(); - curShards.add(Metapb.Shard.newBuilder().setStoreId(1001).setRole(Metapb.ShardRole.Follower).build()); - curShards.add(Metapb.Shard.newBuilder().setStoreId(1002).setRole(Metapb.ShardRole.Leader).build()); - curShards.add(Metapb.Shard.newBuilder().setStoreId(1003).setRole(Metapb.ShardRole.Follower).build()); - - List reqShards = new ArrayList<>(); - reqShards.add(Metapb.Shard.newBuilder().setStoreId(1001).setRole(Metapb.ShardRole.Leader).build()); - reqShards.add(Metapb.Shard.newBuilder().setStoreId(1002).setRole(Metapb.ShardRole.Leader).build()); - reqShards.add(Metapb.Shard.newBuilder().setStoreId(1004).setRole(Metapb.ShardRole.Leader).build()); - - - long leaderStoreId = 0; - for (Metapb.Shard shard : curShards) { - if (shard.getRole() == Metapb.ShardRole.Leader) { - leaderStoreId = shard.getStoreId(); - break; - } - } - - // remove - List shards = curShards.stream().filter(shard -> - reqShards.stream().map(Metapb.Shard::getStoreId).collect(Collectors.toSet()).contains(shard.getStoreId())) - .collect(Collectors.toList()); - - // add - List addShards = reqShards.stream().filter(shard -> - !curShards.stream().map(Metapb.Shard::getStoreId).collect(Collectors.toSet()).contains(shard.getStoreId())) - .collect(Collectors.toList()); - shards.addAll(addShards); - - // change leader - for (Metapb.Shard shard : shards) { - if (shard.getStoreId() == leaderStoreId) { - shard.toBuilder().setRole(Metapb.ShardRole.Leader).build(); - } else { - shard.toBuilder().setRole(Metapb.ShardRole.Follower).build(); - } - } - - } - - // @Test - public void testPriority(){ - List oldPeers = new ArrayList<>(); - oldPeers.add("127.0.0.1:8001::100"); - oldPeers.add("127.0.0.1:8002::75"); - oldPeers.add("127.0.0.1:8003::50"); - - List peers = new ArrayList<>(); - peers.add("127.0.0.1:8001"); - peers.add("127.0.0.1:8003::60"); - peers.add("127.0.0.1:8004"); - - List priPeers = new ArrayList<>(); - for(String peer :peers) { - if(peer.contains("::")) { - priPeers.add(peer); - System.out.println(peer); - } else { - boolean find = false; - for(String oldPeer :oldPeers) { - if(oldPeer.contains(peer + "::")) { - find = true; - priPeers.add(oldPeer); - System.out.println(oldPeer); - break; - } - } - if(!find) { - priPeers.add(peer); - System.out.println(peer); - } - } - } - - - } - -// @Test - public void testRpcCall() throws ExecutionException, InterruptedException { - HgCmdClient client = new HgCmdClient(); - client.init(new RpcOptions(), null); - ArrayList ps = new ArrayList<>(); - Metapb.Partition p = Metapb.Partition.newBuilder() - .setGraphName("OK") - .setId(1) - // .addShards(Metapb.Shard.newBuilder().setStoreId(1).build()) - .build(); - ps.add(new Partition(p)); - client.createRaftNode("127.0.0.1:8511", ps, status -> { - System.out.println(status); - }).get(); - } -} diff --git a/hg-store-core/src/test/java/com/baidu/hugegraph/store/meta/GraphManagerTest.java b/hg-store-core/src/test/java/com/baidu/hugegraph/store/meta/GraphManagerTest.java deleted file mode 100644 index f50d8e3ba..000000000 --- a/hg-store-core/src/test/java/com/baidu/hugegraph/store/meta/GraphManagerTest.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.baidu.hugegraph.store.meta; - - -import org.junit.Assert; -// import org.junit.Test; - -public class GraphManagerTest { - // @Test - public void testCloneGraph(){ - Graph graph = new Graph(); - graph.setGraphName("test1"); - - - - Graph graph1 = graph.clone(); - - Assert.assertTrue(graph!=graph1); - - Assert.assertEquals(graph.getGraphName(), graph1.getGraphName()); - graph1.setGraphName("test4"); - - - Assert.assertNotEquals(graph.getGraphName(), graph1.getGraphName()); - - Assert.assertEquals(graph.getGraphName(), "test1"); - - } - -} diff --git a/hg-store-core/src/test/java/com/baidu/hugegraph/store/util/FutureClosureTest.java b/hg-store-core/src/test/java/com/baidu/hugegraph/store/util/FutureClosureTest.java deleted file mode 100644 index 9ee1de424..000000000 --- a/hg-store-core/src/test/java/com/baidu/hugegraph/store/util/FutureClosureTest.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.baidu.hugegraph.store.util; - -import com.alipay.sofa.jraft.Status; -import org.apdplat.word.vector.F; -import org.junit.Assert; -import org.junit.Test; - -public class FutureClosureTest { - @Test - public void test(){ - FutureClosure closure = new FutureClosure(); - new Thread(()->{ - try { - Thread.sleep(1000); - closure.run(Status.OK()); - } catch (InterruptedException e) { - closure.run(new Status(-1, e.getMessage())); - } - - }).start(); - - Assert.assertEquals(closure.get().getCode(), Status.OK().getCode()); - Assert.assertEquals(closure.get().getCode(), Status.OK().getCode()); - Assert.assertEquals(closure.get().getCode(), Status.OK().getCode()); - } -} diff --git a/hg-store-core/src/test/java/com/baidu/hugegraph/store/HgBusinessImplTest.java b/hg-store-core/src/test/java/org/apache/hugegraph/store/HgBusinessImplTest.java similarity index 69% rename from hg-store-core/src/test/java/com/baidu/hugegraph/store/HgBusinessImplTest.java rename to hg-store-core/src/test/java/org/apache/hugegraph/store/HgBusinessImplTest.java index 9437959df..a66ae5d19 100644 --- a/hg-store-core/src/test/java/com/baidu/hugegraph/store/HgBusinessImplTest.java +++ b/hg-store-core/src/test/java/org/apache/hugegraph/store/HgBusinessImplTest.java @@ -1,19 +1,21 @@ -package com.baidu.hugegraph.store; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import org.apache.hugegraph.rocksdb.access.RocksDBFactory; -import org.apache.hugegraph.rocksdb.access.RocksDBSession; -import org.apache.hugegraph.rocksdb.access.ScanIterator; -import com.baidu.hugegraph.store.business.BusinessHandler; -import com.baidu.hugegraph.store.business.BusinessHandlerImpl; -import com.baidu.hugegraph.store.options.HgStoreEngineOptions; -import com.baidu.hugegraph.store.options.RaftRocksdbOptions; -import com.baidu.hugegraph.store.meta.PartitionManager; -import com.baidu.hugegraph.store.pd.FakePdServiceProvider; -import com.baidu.hugegraph.store.pd.PdProvider; -import com.baidu.hugegraph.util.Bytes; -import org.junit.Assert; -// import org.junit.BeforeClass; -// import org.junit.Test; +package org.apache.hugegraph.store; import java.io.File; import java.util.ArrayList; @@ -22,13 +24,27 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.business.BusinessHandler; +import org.apache.hugegraph.store.business.BusinessHandlerImpl; +import org.apache.hugegraph.store.meta.PartitionManager; +import org.apache.hugegraph.store.options.HgStoreEngineOptions; +import org.apache.hugegraph.store.options.RaftRocksdbOptions; +import org.apache.hugegraph.store.pd.FakePdServiceProvider; +import org.apache.hugegraph.store.pd.PdProvider; +import org.junit.Assert; + +import com.baidu.hugegraph.util.Bytes; + public class HgBusinessImplTest { static String GRAPH_NAME = "graph_test"; static String[] GRAPH_TABLE = {"table1", "table2"}; static String dbPath = "tmp/junit"; -// @BeforeClass + // @BeforeClass public static void init() { UnitTestBase.deleteDir(new File(dbPath)); Map configMap = new HashMap<>(); @@ -67,7 +83,7 @@ public class HgBusinessImplTest { return handler; } -// @Test + // @Test public void testPut() { String graph1 = "test-graph11"; String graph2 = "test-graph12"; @@ -108,7 +124,8 @@ public class HgBusinessImplTest { Assert.assertEquals(4, count); System.out.println("--------------------dump scan range key1 key4 -------"); - iterator = handler.scan(graph1, 0, table, "key1".getBytes(), "key4".getBytes(), ScanIterator.Trait.SCAN_LT_END); + iterator = handler.scan(graph1, 0, table, "key1".getBytes(), "key4".getBytes(), + ScanIterator.Trait.SCAN_LT_END); count = 0; while (iterator.hasNext()) { RocksDBSession.BackendColumn entry = iterator.next(); @@ -119,7 +136,7 @@ public class HgBusinessImplTest { Assert.assertEquals(3, count); } -// @Test + // @Test public void testLoadSnapshot() throws InterruptedException { BusinessHandler handler = getBusinessHandler(); @@ -149,27 +166,30 @@ public class HgBusinessImplTest { } CountDownLatch latch = new CountDownLatch(1); - RocksDBFactory.getInstance().addRocksdbChangedListener(new RocksDBFactory.RocksdbChangedListener() { - @Override - public void onCompacted(String dbName) { - RocksDBFactory.RocksdbChangedListener.super.onCompacted(dbName); - } + RocksDBFactory.getInstance() + .addRocksdbChangedListener(new RocksDBFactory.RocksdbChangedListener() { + @Override + public void onCompacted(String dbName) { + RocksDBFactory.RocksdbChangedListener.super.onCompacted(dbName); + } - @Override - public void onDBDeleteBegin(String dbName, String filePath) { - RocksDBFactory.RocksdbChangedListener.super.onDBDeleteBegin(dbName, filePath); - } + @Override + public void onDBDeleteBegin(String dbName, String filePath) { + RocksDBFactory.RocksdbChangedListener.super.onDBDeleteBegin(dbName, + filePath); + } - @Override - public void onDBDeleted(String dbName, String filePath) { - latch.countDown(); - } + @Override + public void onDBDeleted(String dbName, String filePath) { + latch.countDown(); + } - @Override - public void onDBSessionReleased(RocksDBSession dbSession) { - RocksDBFactory.RocksdbChangedListener.super.onDBSessionReleased(dbSession); - } - }); + @Override + public void onDBSessionReleased(RocksDBSession dbSession) { + RocksDBFactory.RocksdbChangedListener.super.onDBSessionReleased( + dbSession); + } + }); latch.await(); } diff --git a/hg-store-core/src/test/java/org/apache/hugegraph/store/PartitionEngineTest.java b/hg-store-core/src/test/java/org/apache/hugegraph/store/PartitionEngineTest.java new file mode 100644 index 000000000..688f24d41 --- /dev/null +++ b/hg-store-core/src/test/java/org/apache/hugegraph/store/PartitionEngineTest.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import org.apache.hugegraph.store.cmd.HgCmdClient; +import org.apache.hugegraph.store.meta.Partition; +import org.junit.Assert; + +import com.alipay.sofa.jraft.option.RpcOptions; +import com.baidu.hugegraph.pd.grpc.Metapb; + +public class PartitionEngineTest { + + // @Test + public void test() { + String[] peers = new String[]{"1", "2", "6"}; + String[] learners = new String[]{"3", "7"}; + List oldPeers1 = Arrays.asList("1", "2", "3"); + List oldLearner1 = Arrays.asList("4", "5"); + + + List oldPeers = + oldPeers1.stream().map(peer -> peer).collect(Collectors.toList()); + oldPeers.addAll( + oldLearner1.stream().map(peer -> peer).collect(Collectors.toList())); + + List addedNodes = Arrays.stream(peers).filter(peer -> + !oldPeers.stream() + .map(e -> e) + .collect( + Collectors.toSet()) + .contains(peer)) + .collect(Collectors.toList()); + addedNodes.addAll(Arrays.stream(learners).filter(peer -> + !oldPeers.stream() + .map(e -> e) + .collect( + Collectors.toSet()) + .contains(peer)) + .collect(Collectors.toList())); + + List removedNodes = oldPeers.stream().filter(peer -> + !Arrays.stream(peers) + .map(e -> e).collect( + Collectors.toSet()) + .contains(peer)) + .collect(Collectors.toList()); + removedNodes = removedNodes.stream().filter(peer -> + !Arrays.stream(learners).map(e -> e) + .collect(Collectors.toSet()) + .contains(peer)) + .collect(Collectors.toList()); + List mixedPeer = oldPeers1.stream().filter(peer -> + Arrays.stream(learners) + .map(e -> e).collect( + Collectors.toSet()) + .contains(peer)) + .collect(Collectors.toList()); + + // 新增 6、7 + Assert.assertEquals(2, addedNodes.size()); + addedNodes.clear(); + addedNodes.addAll(Arrays.asList(peers)); + addedNodes.addAll(Arrays.asList(learners)); + addedNodes.removeAll(oldPeers); + Assert.assertEquals(2, addedNodes.size()); + + addedNodes.forEach(s -> System.out.print(s + " ")); + System.out.println(); + // 删除 4，5 + Assert.assertEquals(2, removedNodes.size()); + + removedNodes.clear(); + removedNodes.addAll(oldPeers); + removedNodes.removeAll(Arrays.asList(peers)); + removedNodes.removeAll(Arrays.asList(learners)); + Assert.assertEquals(2, removedNodes.size()); + removedNodes.forEach(s -> System.out.print(s + " ")); + System.out.println(); + // 交集 5 + Assert.assertEquals(1, mixedPeer.size()); + oldPeers1.removeAll(Arrays.asList(learners)); + Assert.assertEquals(1, oldPeers1.size()); + mixedPeer.forEach(s -> System.out.print(s + " ")); + + } + + + // @Test + public void testPartition() { + Partition p = new Partition(); + p.setId(1); + List shards = new ArrayList<>(); + shards.add(Metapb.Shard.newBuilder().build()); + // p.setShardsList(shards); + + Partition p2 = p.clone(); + p.setId(2); + Assert.assertNotEquals(p2.getId(), p.getId()); + + } + + // @Test + public void testUpdateShardsList() { + List curShards = new ArrayList<>(); + curShards.add(Metapb.Shard.newBuilder().setStoreId(1001).setRole(Metapb.ShardRole.Follower) + .build()); + curShards.add(Metapb.Shard.newBuilder().setStoreId(1002).setRole(Metapb.ShardRole.Leader) + .build()); + curShards.add(Metapb.Shard.newBuilder().setStoreId(1003).setRole(Metapb.ShardRole.Follower) + .build()); + + List reqShards = new ArrayList<>(); + reqShards.add(Metapb.Shard.newBuilder().setStoreId(1001).setRole(Metapb.ShardRole.Leader) + .build()); + reqShards.add(Metapb.Shard.newBuilder().setStoreId(1002).setRole(Metapb.ShardRole.Leader) + .build()); + reqShards.add(Metapb.Shard.newBuilder().setStoreId(1004).setRole(Metapb.ShardRole.Leader) + .build()); + + + long leaderStoreId = 0; + for (Metapb.Shard shard : curShards) { + if (shard.getRole() == Metapb.ShardRole.Leader) { + leaderStoreId = shard.getStoreId(); + break; + } + } + + // remove + List shards = curShards.stream().filter(shard -> + reqShards.stream() + .map(Metapb.Shard::getStoreId) + .collect( + Collectors.toSet()) + .contains( + shard.getStoreId())) + .collect(Collectors.toList()); + + // add + List addShards = reqShards.stream().filter(shard -> + !curShards.stream() + .map(Metapb.Shard::getStoreId) + .collect( + Collectors.toSet()) + .contains( + shard.getStoreId())) + .collect(Collectors.toList()); + shards.addAll(addShards); + + // change leader + for (Metapb.Shard shard : shards) { + if (shard.getStoreId() == leaderStoreId) { + shard.toBuilder().setRole(Metapb.ShardRole.Leader).build(); + } else { + shard.toBuilder().setRole(Metapb.ShardRole.Follower).build(); + } + } + + } + + // @Test + public void testPriority() { + List oldPeers = new ArrayList<>(); + oldPeers.add("127.0.0.1:8001::100"); + oldPeers.add("127.0.0.1:8002::75"); + oldPeers.add("127.0.0.1:8003::50"); + + List peers = new ArrayList<>(); + peers.add("127.0.0.1:8001"); + peers.add("127.0.0.1:8003::60"); + peers.add("127.0.0.1:8004"); + + List priPeers = new ArrayList<>(); + for (String peer : peers) { + if (peer.contains("::")) { + priPeers.add(peer); + System.out.println(peer); + } else { + boolean find = false; + for (String oldPeer : oldPeers) { + if (oldPeer.contains(peer + "::")) { + find = true; + priPeers.add(oldPeer); + System.out.println(oldPeer); + break; + } + } + if (!find) { + priPeers.add(peer); + System.out.println(peer); + } + } + } + + + } + + // @Test + public void testRpcCall() throws ExecutionException, InterruptedException { + HgCmdClient client = new HgCmdClient(); + client.init(new RpcOptions(), null); + ArrayList ps = new ArrayList<>(); + Metapb.Partition p = Metapb.Partition.newBuilder() + .setGraphName("OK") + .setId(1) + // .addShards(Metapb.Shard.newBuilder().setStoreId + // (1).build()) + .build(); + ps.add(new Partition(p)); + client.createRaftNode("127.0.0.1:8511", ps, status -> { + System.out.println(status); + }).get(); + } +} diff --git a/hg-store-core/src/test/java/com/baidu/hugegraph/store/UnitTestBase.java b/hg-store-core/src/test/java/org/apache/hugegraph/store/UnitTestBase.java similarity index 65% rename from hg-store-core/src/test/java/com/baidu/hugegraph/store/UnitTestBase.java rename to hg-store-core/src/test/java/org/apache/hugegraph/store/UnitTestBase.java index 4f0c99c3d..92c19c849 100644 --- a/hg-store-core/src/test/java/com/baidu/hugegraph/store/UnitTestBase.java +++ b/hg-store-core/src/test/java/org/apache/hugegraph/store/UnitTestBase.java @@ -1,19 +1,36 @@ -package com.baidu.hugegraph.store; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import org.apache.hugegraph.rocksdb.access.RocksDBFactory; -import org.apache.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.store.business.BusinessHandler; -import com.baidu.hugegraph.store.business.BusinessHandlerImpl; -import com.baidu.hugegraph.store.meta.PartitionManager; -import com.baidu.hugegraph.store.options.HgStoreEngineOptions; -import com.baidu.hugegraph.store.options.RaftRocksdbOptions; -import com.baidu.hugegraph.store.pd.FakePdServiceProvider; -import com.baidu.hugegraph.store.pd.PdProvider; +package org.apache.hugegraph.store; import java.io.File; import java.util.HashMap; import java.util.Map; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.business.BusinessHandler; +import org.apache.hugegraph.store.business.BusinessHandlerImpl; +import org.apache.hugegraph.store.meta.PartitionManager; +import org.apache.hugegraph.store.options.HgStoreEngineOptions; +import org.apache.hugegraph.store.options.RaftRocksdbOptions; +import org.apache.hugegraph.store.pd.FakePdServiceProvider; +import org.apache.hugegraph.store.pd.PdProvider; + public class UnitTestBase { public static boolean deleteDir(File dir) { diff --git a/hg-store-core/src/test/java/com/baidu/hugegraph/store/meta/GraphIDManagerTest.java b/hg-store-core/src/test/java/org/apache/hugegraph/store/meta/GraphIDManagerTest.java similarity index 66% rename from hg-store-core/src/test/java/com/baidu/hugegraph/store/meta/GraphIDManagerTest.java rename to hg-store-core/src/test/java/org/apache/hugegraph/store/meta/GraphIDManagerTest.java index 82adf21e2..fd3fb3b17 100644 --- a/hg-store-core/src/test/java/com/baidu/hugegraph/store/meta/GraphIDManagerTest.java +++ b/hg-store-core/src/test/java/org/apache/hugegraph/store/meta/GraphIDManagerTest.java @@ -1,18 +1,36 @@ -package com.baidu.hugegraph.store.meta; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta; + +import java.io.File; -import com.baidu.hugegraph.pd.common.PDException; import org.apache.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.store.UnitTestBase; -import com.baidu.hugegraph.store.meta.base.DBSessionBuilder; +import org.apache.hugegraph.store.UnitTestBase; +import org.apache.hugegraph.store.meta.base.DBSessionBuilder; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.io.File; +import com.baidu.hugegraph.pd.common.PDException; public class GraphIDManagerTest extends UnitTestBase { @Before - public void init(){ + public void init() { String dbPath = "/tmp/junit"; deleteDir(new File(dbPath)); super.initDB(dbPath); @@ -22,14 +40,14 @@ public class GraphIDManagerTest extends UnitTestBase { public void test() throws PDException { GraphIdManager.maxGraphID = 64; int max = GraphIdManager.maxGraphID; - try(RocksDBSession session = getDBSession("test")) { + try (RocksDBSession session = getDBSession("test")) { GraphIdManager gid = new GraphIdManager(new DBSessionBuilder() { @Override public RocksDBSession getSession(int partId) { return session.clone(); } }, 0); - for(int i = 0; i < max; i++ ){ + for (int i = 0; i < max; i++) { Assert.assertEquals(i, gid.getCId("Test", max)); } @@ -59,7 +77,7 @@ public class GraphIDManagerTest extends UnitTestBase { start = System.currentTimeMillis(); for (int i = 0; i < GraphIdManager.maxGraphID; i++) { long id = gid.getCId(GraphIdManager.GRAPH_ID_PREFIX, GraphIdManager.maxGraphID); - // long id = gid.getGraphId("g" + i); + // long id = gid.getGraphId("g" + i); Assert.assertTrue(id >= 0); } System.out.println("time is " + (System.currentTimeMillis() - start)); diff --git a/hg-store-core/src/test/java/org/apache/hugegraph/store/meta/GraphManagerTest.java b/hg-store-core/src/test/java/org/apache/hugegraph/store/meta/GraphManagerTest.java new file mode 100644 index 000000000..04888a21d --- /dev/null +++ b/hg-store-core/src/test/java/org/apache/hugegraph/store/meta/GraphManagerTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.meta; + + +import org.junit.Assert; +// import org.junit.Test; + +public class GraphManagerTest { + // @Test + public void testCloneGraph() { + Graph graph = new Graph(); + graph.setGraphName("test1"); + + + Graph graph1 = graph.clone(); + + Assert.assertNotSame(graph, graph1); + + Assert.assertEquals(graph.getGraphName(), graph1.getGraphName()); + graph1.setGraphName("test4"); + + + Assert.assertNotEquals(graph.getGraphName(), graph1.getGraphName()); + + Assert.assertEquals(graph.getGraphName(), "test1"); + + } + +} diff --git a/hg-store-core/src/test/java/com/baidu/hugegraph/store/util/CopyOnWriteCacheTest.java b/hg-store-core/src/test/java/org/apache/hugegraph/store/util/CopyOnWriteCacheTest.java similarity index 61% rename from hg-store-core/src/test/java/com/baidu/hugegraph/store/util/CopyOnWriteCacheTest.java rename to hg-store-core/src/test/java/org/apache/hugegraph/store/util/CopyOnWriteCacheTest.java index 9bb409d49..29483cbd6 100644 --- a/hg-store-core/src/test/java/com/baidu/hugegraph/store/util/CopyOnWriteCacheTest.java +++ b/hg-store-core/src/test/java/org/apache/hugegraph/store/util/CopyOnWriteCacheTest.java @@ -1,14 +1,30 @@ -package com.baidu.hugegraph.store.util; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ -import com.caucho.hessian.io.Hessian2Input; -import com.caucho.hessian.io.Hessian2Output; -// import org.junit.Test; +package org.apache.hugegraph.store.util; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Map; +import com.caucho.hessian.io.Hessian2Input; +import com.caucho.hessian.io.Hessian2Output; + public class CopyOnWriteCacheTest { // @Test diff --git a/hg-store-core/src/test/java/org/apache/hugegraph/store/util/FutureClosureTest.java b/hg-store-core/src/test/java/org/apache/hugegraph/store/util/FutureClosureTest.java new file mode 100644 index 000000000..d29683967 --- /dev/null +++ b/hg-store-core/src/test/java/org/apache/hugegraph/store/util/FutureClosureTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.util; + +import org.junit.Assert; +import org.junit.Test; + +import com.alipay.sofa.jraft.Status; + +public class FutureClosureTest { + @Test + public void test() { + FutureClosure closure = new FutureClosure(); + new Thread(() -> { + try { + Thread.sleep(1000); + closure.run(Status.OK()); + } catch (InterruptedException e) { + closure.run(new Status(-1, e.getMessage())); + } + + }).start(); + + Assert.assertEquals(closure.get().getCode(), Status.OK().getCode()); + Assert.assertEquals(closure.get().getCode(), Status.OK().getCode()); + Assert.assertEquals(closure.get().getCode(), Status.OK().getCode()); + } +} diff --git a/hg-store-core/src/test/resources/log4j2.xml b/hg-store-core/src/test/resources/log4j2.xml index 505040c22..8c7912b45 100644 --- a/hg-store-core/src/test/resources/log4j2.xml +++ b/hg-store-core/src/test/resources/log4j2.xml @@ -1,5 +1,22 @@ + + @@ -28,7 +45,7 @@ - + @@ -36,8 +53,8 @@ - - + + @@ -56,7 +73,7 @@ - + @@ -64,8 +81,8 @@ - - + + diff --git a/hg-store-dist/pom.xml b/hg-store-dist/pom.xml index 647a253f4..f74e11273 100644 --- a/hg-store-dist/pom.xml +++ b/hg-store-dist/pom.xml @@ -21,9 +21,9 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + org.apache.hugegraph hugegraph-store-root - com.baidu.hugegraph - 3.6.3-SNAPSHOT + ${revision} 4.0.0 @@ -39,9 +39,9 @@ - com.baidu.hugegraph + org.apache.hugegraph hugegraph-store - ${project.version} + ${revision} diff --git a/hg-store-grpc/pom.xml b/hg-store-grpc/pom.xml index 44a954d99..5a35857ff 100644 --- a/hg-store-grpc/pom.xml +++ b/hg-store-grpc/pom.xml @@ -21,9 +21,9 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - com.baidu.hugegraph + org.apache.hugegraph hg-store-grpc - 3.6.3-SNAPSHOT + ${revision} 11 @@ -83,10 +83,12 @@ ${protobuf.plugin.version} true - com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier} + + com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier} grpc-java - io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} + + io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} ${project.basedir}/src/main/proto diff --git a/hg-store-node/pom.xml b/hg-store-node/pom.xml index f9b486529..87a4e3f93 100644 --- a/hg-store-node/pom.xml +++ b/hg-store-node/pom.xml @@ -22,9 +22,9 @@ 4.0.0 - com.baidu.hugegraph + org.apache.hugegraph hugegraph-store-root - 3.6.3-SNAPSHOT + ${revision} hugegraph-store @@ -90,7 +90,7 @@ - com.baidu.hugegraph + org.apache.hugegraph hg-store-common @@ -99,13 +99,13 @@ 3.17.2 - com.baidu.hugegraph + org.apache.hugegraph hg-store-grpc - com.baidu.hugegraph + org.apache.hugegraph hg-pd-client - ${project.version} + ${revision} io.grpc @@ -114,11 +114,11 @@ - com.baidu.hugegraph + org.apache.hugegraph hg-store-core - com.baidu.hugegraph + org.apache.hugegraph hugegraph-core ${hugegraph.server.version} diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgTestController.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgTestController.java index af4c55f62..23203ab2f 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgTestController.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgTestController.java @@ -20,6 +20,9 @@ package org.apache.hugegraph.store.node.controller; import java.util.ArrayList; import java.util.List; +import org.apache.hugegraph.store.PartitionEngine; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.meta.Store; import org.apache.hugegraph.store.node.grpc.HgStoreNodeService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; @@ -28,10 +31,6 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import com.baidu.hugegraph.store.PartitionEngine; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.meta.Store; - import lombok.extern.slf4j.Slf4j; /** diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/IndexAPI.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/IndexAPI.java index 1d0ff0bd8..fd6f103cc 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/IndexAPI.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/IndexAPI.java @@ -22,6 +22,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.metric.HgStoreMetric; import org.apache.hugegraph.store.node.grpc.HgStoreNodeService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; @@ -31,8 +33,6 @@ import org.springframework.web.bind.annotation.RestController; import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.util.Endpoint; import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.metric.HgStoreMetric; import lombok.Data; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/PartitionAPI.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/PartitionAPI.java index 5588e8609..5989e44b0 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/PartitionAPI.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/PartitionAPI.java @@ -25,6 +25,12 @@ import java.util.Map; import org.apache.hugegraph.rocksdb.access.RocksDBSession; import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.PartitionEngine; +import org.apache.hugegraph.store.business.BusinessHandler; +import org.apache.hugegraph.store.business.InnerKeyCreator; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.metric.HgStoreMetric; import org.apache.hugegraph.store.node.AppConfig; import org.apache.hugegraph.store.node.grpc.HgStoreNodeService; import org.springframework.beans.factory.annotation.Autowired; @@ -39,12 +45,6 @@ import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.util.Endpoint; import com.baidu.hugegraph.pd.common.PDException; import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.PartitionEngine; -import com.baidu.hugegraph.store.business.BusinessHandler; -import com.baidu.hugegraph.store.business.InnerKeyCreator; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.metric.HgStoreMetric; import com.baidu.hugegraph.util.Bytes; import com.taobao.arthas.agent.attach.ArthasAgent; @@ -227,7 +227,7 @@ public class PartitionAPI { private String leader; - public PartitionInfo(com.baidu.hugegraph.store.meta.Partition pt) { + public PartitionInfo(Partition pt) { id = pt.getId(); graphName = pt.getGraphName(); startKey = pt.getStartKey(); diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchGrpcClosure.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchGrpcClosure.java index 978baa3db..b830a92d9 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchGrpcClosure.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchGrpcClosure.java @@ -33,6 +33,8 @@ import java.util.function.Consumer; import java.util.function.Function; import org.apache.commons.collections.CollectionUtils; +import org.apache.hugegraph.store.raft.RaftClosure; +import org.apache.hugegraph.store.util.HgRaftError; import com.alipay.sofa.jraft.Status; import com.baidu.hugegraph.store.grpc.common.ResCode; @@ -41,8 +43,6 @@ import com.baidu.hugegraph.store.grpc.session.FeedbackRes; import com.baidu.hugegraph.store.grpc.session.PartitionFaultResponse; import com.baidu.hugegraph.store.grpc.session.PartitionFaultType; import com.baidu.hugegraph.store.grpc.session.PartitionLeader; -import com.baidu.hugegraph.store.raft.RaftClosure; -import com.baidu.hugegraph.store.util.HgRaftError; import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcClosure.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcClosure.java index c3ba1bf3e..8f722cdfc 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcClosure.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcClosure.java @@ -21,8 +21,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.hugegraph.store.raft.RaftClosure; + import com.baidu.hugegraph.store.grpc.session.FeedbackRes; -import com.baidu.hugegraph.store.raft.RaftClosure; import io.grpc.stub.StreamObserver; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java index 9b5544be9..1b28332fa 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java @@ -26,22 +26,22 @@ import java.util.Map; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.business.DefaultDataMover; import org.apache.hugegraph.store.node.AppConfig; +import org.apache.hugegraph.store.options.HgStoreEngineOptions; +import org.apache.hugegraph.store.options.RaftRocksdbOptions; +import org.apache.hugegraph.store.raft.RaftClosure; +import org.apache.hugegraph.store.raft.RaftOperation; +import org.apache.hugegraph.store.raft.RaftTaskHandler; +import org.apache.hugegraph.store.util.HgRaftError; +import org.apache.hugegraph.store.util.HgStoreException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.core.NodeMetrics; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.business.DefaultDataMover; import com.baidu.hugegraph.store.grpc.session.*; -import com.baidu.hugegraph.store.options.HgStoreEngineOptions; -import com.baidu.hugegraph.store.options.RaftRocksdbOptions; -import com.baidu.hugegraph.store.raft.RaftClosure; -import com.baidu.hugegraph.store.raft.RaftOperation; -import com.baidu.hugegraph.store.raft.RaftTaskHandler; -import com.baidu.hugegraph.store.util.HgRaftError; -import com.baidu.hugegraph.store.util.HgStoreException; import com.google.protobuf.CodedInputStream; import com.google.protobuf.CodedOutputStream; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreSessionImpl.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreSessionImpl.java index 9212ad8ba..56455f273 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreSessionImpl.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreSessionImpl.java @@ -23,9 +23,14 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hugegraph.store.meta.Graph; +import org.apache.hugegraph.store.meta.GraphManager; import org.apache.hugegraph.store.node.AppConfig; import org.apache.hugegraph.store.node.util.HgGrpc; import org.apache.hugegraph.store.node.util.HgStoreNodeUtil; +import org.apache.hugegraph.store.pd.PdProvider; +import org.apache.hugegraph.store.raft.RaftClosure; +import org.apache.hugegraph.store.util.HgStoreConst; import org.lognet.springboot.grpc.GRpcService; import org.springframework.beans.factory.annotation.Autowired; @@ -48,11 +53,6 @@ import com.baidu.hugegraph.store.grpc.session.HgStoreSessionGrpc; import com.baidu.hugegraph.store.grpc.session.KeyValueResponse; import com.baidu.hugegraph.store.grpc.session.TableReq; import com.baidu.hugegraph.store.grpc.session.ValueResponse; -import com.baidu.hugegraph.store.meta.Graph; -import com.baidu.hugegraph.store.meta.GraphManager; -import com.baidu.hugegraph.store.pd.PdProvider; -import com.baidu.hugegraph.store.raft.RaftClosure; -import com.baidu.hugegraph.store.util.HgStoreConst; import com.google.protobuf.ByteString; import io.grpc.stub.StreamObserver; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreWrapperEx.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreWrapperEx.java index 5e774efac..988f46f3c 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreWrapperEx.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreWrapperEx.java @@ -22,13 +22,13 @@ import java.util.function.Consumer; import java.util.function.Supplier; import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.business.BusinessHandler; +import org.apache.hugegraph.store.business.FilterIterator; +import org.apache.hugegraph.store.term.HgPair; -import com.baidu.hugegraph.store.business.BusinessHandler; -import com.baidu.hugegraph.store.business.FilterIterator; import com.baidu.hugegraph.store.grpc.common.GraphMethod; import com.baidu.hugegraph.store.grpc.common.TableMethod; import com.baidu.hugegraph.store.grpc.session.BatchEntry; -import com.baidu.hugegraph.store.term.HgPair; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java index f9121216c..9aaef08dd 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java @@ -30,15 +30,15 @@ import java.util.function.Supplier; import org.apache.hugegraph.rocksdb.access.RocksDBSession; import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.buffer.KVByteBuffer; import org.apache.hugegraph.store.node.util.HgAssert; import org.apache.hugegraph.store.node.util.PropertyUtil; +import org.apache.hugegraph.store.term.Bits; import com.alipay.sofa.jraft.util.Utils; import com.baidu.hugegraph.pd.common.KVPair; -import com.baidu.hugegraph.store.buffer.KVByteBuffer; import com.baidu.hugegraph.store.grpc.common.ScanOrderType; import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest; -import com.baidu.hugegraph.store.term.Bits; import lombok.extern.slf4j.Slf4j; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java index 428288ad2..20b847156 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java @@ -23,12 +23,12 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.locks.ReentrantLock; import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.buffer.ByteBufferAllocator; +import org.apache.hugegraph.store.buffer.KVByteBuffer; import org.apache.hugegraph.store.grpc.stream.KvStream; import org.apache.hugegraph.store.node.util.HgGrpc; import org.apache.hugegraph.store.node.util.PropertyUtil; -import com.baidu.hugegraph.store.buffer.ByteBufferAllocator; -import com.baidu.hugegraph.store.buffer.KVByteBuffer; import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest; import com.baidu.hugegraph.store.grpc.stream.ScanStreamBatchReq; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanUtil.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanUtil.java index 10c9e93af..270f68d05 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanUtil.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanUtil.java @@ -32,9 +32,9 @@ import java.util.stream.Collectors; import javax.annotation.concurrent.NotThreadSafe; import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.business.SelectIterator; import com.baidu.hugegraph.pd.common.KVPair; -import com.baidu.hugegraph.store.business.SelectIterator; import com.baidu.hugegraph.store.grpc.common.ScanMethod; import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest; import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq; @@ -210,7 +210,7 @@ class ScanUtil { request.getScanType(); sq.query = request.getQuery() - .toByteArray(); + .toByteArray(); sq.position = request.getPosition() .toByteArray(); @@ -219,12 +219,12 @@ class ScanUtil { condition.getCode(); sq.start = condition.getStart() - .toByteArray(); + .toByteArray(); sq.end = condition.getEnd() .toByteArray(); sq.prefix = condition.getPrefix() - .toByteArray(); + .toByteArray(); sq.serialNo = condition.getSerialNo(); return sq; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/GraphStoreImpl.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/GraphStoreImpl.java index 240799007..32d6724d2 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/GraphStoreImpl.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/GraphStoreImpl.java @@ -19,12 +19,12 @@ package org.apache.hugegraph.store.node.grpc.scan; import java.util.concurrent.ThreadPoolExecutor; +import org.apache.hugegraph.store.business.BusinessHandler; import org.apache.hugegraph.store.node.grpc.HgStoreNodeService; import org.apache.hugegraph.store.node.grpc.HgStoreStreamImpl; import org.lognet.springboot.grpc.GRpcService; import org.springframework.beans.factory.annotation.Autowired; -import com.baidu.hugegraph.store.business.BusinessHandler; import com.baidu.hugegraph.store.grpc.GraphStoreGrpc.GraphStoreImplBase; import com.baidu.hugegraph.store.grpc.Graphpb; import com.baidu.hugegraph.store.grpc.Graphpb.ResponseHeader; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/ScanResponseObserver.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/ScanResponseObserver.java index 3c2c79d72..f50275b16 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/ScanResponseObserver.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/ScanResponseObserver.java @@ -26,8 +26,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; -import com.baidu.hugegraph.store.business.BusinessHandler; -import com.baidu.hugegraph.store.business.GraphStoreIterator; +import org.apache.hugegraph.store.business.BusinessHandler; +import org.apache.hugegraph.store.business.GraphStoreIterator; + import com.baidu.hugegraph.store.grpc.Graphpb.ErrorType; import com.baidu.hugegraph.store.grpc.Graphpb.ResponseHeader; import com.baidu.hugegraph.store.grpc.Graphpb.ScanPartitionRequest; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/JRaftMetrics.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/JRaftMetrics.java index 0c14663eb..11adf85a9 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/JRaftMetrics.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/JRaftMetrics.java @@ -26,10 +26,10 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hugegraph.store.HgStoreEngine; import org.apache.hugegraph.store.node.util.HgRegexUtil; import com.alipay.sofa.jraft.core.NodeMetrics; -import com.baidu.hugegraph.store.HgStoreEngine; import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; import com.codahale.metrics.Snapshot; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetrics.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetrics.java index 47e78e2af..586f73b03 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetrics.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetrics.java @@ -38,6 +38,7 @@ import java.util.stream.Collectors; import org.apache.hugegraph.rocksdb.access.RocksDBFactory; import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.HgStoreEngine; import org.apache.hugegraph.store.node.util.HgAssert; import org.rocksdb.HistogramData; import org.rocksdb.HistogramType; @@ -45,8 +46,6 @@ import org.rocksdb.MemoryUsageType; import org.rocksdb.Statistics; import org.rocksdb.TickerType; -import com.baidu.hugegraph.store.HgStoreEngine; - import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.Meter; import io.micrometer.core.instrument.MeterRegistry; @@ -305,7 +304,8 @@ public class RocksDBMetrics { private final String graphName; long lastTime = 0; private final Map tickerCounteMap = new ConcurrentHashMap<>(); - private final Map histogramDataMap = new ConcurrentHashMap<>(); + private final Map histogramDataMap = + new ConcurrentHashMap<>(); StatisticsWrapper(String graph) { diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/StoreMetrics.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/StoreMetrics.java index a12c46dad..fdb9a0b78 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/StoreMetrics.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/StoreMetrics.java @@ -23,8 +23,8 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.meta.Partition; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.MeterRegistry; diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgGrpc.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgGrpc.java index c136a3b52..1accb8bca 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgGrpc.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgGrpc.java @@ -17,13 +17,14 @@ package org.apache.hugegraph.store.node.util; +import org.apache.hugegraph.store.term.HgPair; + import com.baidu.hugegraph.store.grpc.common.Key; import com.baidu.hugegraph.store.grpc.common.Kv; import com.baidu.hugegraph.store.grpc.common.ResCode; import com.baidu.hugegraph.store.grpc.common.ResStatus; import com.baidu.hugegraph.store.grpc.common.Tk; import com.baidu.hugegraph.store.grpc.common.Tse; -import com.baidu.hugegraph.store.term.HgPair; import com.google.common.base.Throwables; import com.google.protobuf.ByteString; diff --git a/hg-store-rocksdb/pom.xml b/hg-store-rocksdb/pom.xml index e281a1b0a..772ada3fd 100644 --- a/hg-store-rocksdb/pom.xml +++ b/hg-store-rocksdb/pom.xml @@ -22,16 +22,16 @@ 4.0.0 - com.baidu.hugegraph + org.apache.hugegraph hugegraph-store-root - 3.6.3-SNAPSHOT + ${revision} hg-store-rocksdb - com.baidu.hugegraph + org.apache.hugegraph hugegraph-common 1.8.12 @@ -50,8 +50,9 @@ - com.baidu.hugegraph + org.apache.hugegraph hg-store-common + ${revision} org.rocksdb diff --git a/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java index 5e2e11383..d3a93c63d 100644 --- a/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java @@ -37,6 +37,7 @@ import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; import org.apache.hugegraph.rocksdb.access.util.Asserts; +import org.apache.hugegraph.store.term.HgPair; import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.Checkpoint; import org.rocksdb.ColumnFamilyDescriptor; @@ -63,7 +64,6 @@ import org.rocksdb.WriteBufferManager; import org.rocksdb.WriteOptions; import com.baidu.hugegraph.config.HugeConfig; -import com.baidu.hugegraph.store.term.HgPair; import com.baidu.hugegraph.util.Bytes; import com.baidu.hugegraph.util.E; @@ -87,7 +87,8 @@ public class RocksDBSession implements AutoCloseable, Cloneable { private volatile boolean closed = false; final String tempSuffix = "_temp_"; - public RocksDBSession(HugeConfig hugeConfig, String dbDataPath, String graphName, long version) { + public RocksDBSession(HugeConfig hugeConfig, String dbDataPath, String graphName, + long version) { this.hugeConfig = hugeConfig; this.graphName = graphName; this.cfHandleLock = new ReentrantReadWriteLock(); @@ -191,7 +192,8 @@ public class RocksDBSession implements AutoCloseable, Cloneable { String latestDBPath = ""; if (!dbs.isEmpty()) { - dbs.sort((o1, o2) -> o1.getKey().equals(o2.getKey()) ? o1.getValue().compareTo(o2.getValue()) : + dbs.sort((o1, o2) -> o1.getKey().equals(o2.getKey()) ? + o1.getValue().compareTo(o2.getValue()) : o1.getKey().compareTo(o2.getKey())); final int dbCount = dbs.size(); for (int i = 0; i < dbCount; i++) { @@ -202,7 +204,8 @@ public class RocksDBSession implements AutoCloseable, Cloneable { } else if (pair.getValue() == -1L) { curDBName = String.format("%s_%d", defaultName, pair.getKey()); } else { - curDBName = String.format("%s_%d_%d", defaultName, pair.getKey(), pair.getValue()); + curDBName = + String.format("%s_%d_%d", defaultName, pair.getKey(), pair.getValue()); } String curDBPath = Paths.get(parentFile.getPath(), curDBName).toString(); if (i == dbCount - 1) { @@ -224,7 +227,9 @@ public class RocksDBSession implements AutoCloseable, Cloneable { } if (factory.findPathInRemovedList(latestDBPath)) { // 已经被删除，创建新的目录 - latestDBPath = Paths.get(parentFile.getPath(), String.format("%s_%d", defaultName, version)).toString(); + latestDBPath = + Paths.get(parentFile.getPath(), String.format("%s_%d", defaultName, version)) + .toString(); } // log.info("{} latest db path {}", this.graphName, latestDBPath); @@ -249,7 +254,7 @@ public class RocksDBSession implements AutoCloseable, Cloneable { } public boolean tableIsExist(String table) { - return this.tables.keySet().contains(table); + return this.tables.containsKey(table); } private void openRocksDB(String dbDataPath, long version) { @@ -262,7 +267,8 @@ public class RocksDBSession implements AutoCloseable, Cloneable { this.dbPath = findLatestDBPath(dbPath, version); - Asserts.isTrue((dbPath != null), () -> new DBStoreException("the data-path of RocksDB is null")); + Asserts.isTrue((dbPath != null), + () -> new DBStoreException("the data-path of RocksDB is null")); //makedir for rocksdb createDirectory(dbPath); @@ -273,7 +279,8 @@ public class RocksDBSession implements AutoCloseable, Cloneable { dbOptions.setStatistics(rocksDbStats); try { - List columnFamilyDescriptorList = new ArrayList(); + List columnFamilyDescriptorList = + new ArrayList(); List columnFamilyBytes = RocksDB.listColumnFamilies(new Options(), dbPath); ColumnFamilyOptions cfOptions = new ColumnFamilyOptions(); @@ -281,13 +288,16 @@ public class RocksDBSession implements AutoCloseable, Cloneable { if (columnFamilyBytes.size() > 0) { for (byte[] columnFamilyByte : columnFamilyBytes) { - columnFamilyDescriptorList.add(new ColumnFamilyDescriptor(columnFamilyByte, cfOptions)); + columnFamilyDescriptorList.add( + new ColumnFamilyDescriptor(columnFamilyByte, cfOptions)); } } else { - columnFamilyDescriptorList.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions)); + columnFamilyDescriptorList.add( + new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions)); } List columnFamilyHandleList = new ArrayList<>(); - this.rocksDB = RocksDB.open(dbOptions, dbPath, columnFamilyDescriptorList, columnFamilyHandleList); + this.rocksDB = RocksDB.open(dbOptions, dbPath, columnFamilyDescriptorList, + columnFamilyHandleList); Asserts.isTrue(columnFamilyHandleList.size() > 0, "must have column family"); for (ColumnFamilyHandle handle : columnFamilyHandleList) { @@ -305,7 +315,8 @@ public class RocksDBSession implements AutoCloseable, Cloneable { if (handle == null) { ColumnFamilyOptions cfOptions = new ColumnFamilyOptions(); RocksDBSession.initOptions(this.hugeConfig, null, null, cfOptions, cfOptions); - ColumnFamilyDescriptor cfDescriptor = new ColumnFamilyDescriptor(table.getBytes(), cfOptions); + ColumnFamilyDescriptor cfDescriptor = + new ColumnFamilyDescriptor(table.getBytes(), cfOptions); handle = this.rocksDB.createColumnFamily(cfDescriptor); tables.put(table, handle); } @@ -347,8 +358,8 @@ public class RocksDBSession implements AutoCloseable, Cloneable { public static class CFHandleLock implements Closeable { - private ColumnFamilyHandle handle; - private ReentrantReadWriteLock lock; + private final ColumnFamilyHandle handle; + private final ReentrantReadWriteLock lock; public CFHandleLock(ColumnFamilyHandle handle, ReentrantReadWriteLock lock) { this.handle = handle; @@ -497,7 +508,8 @@ public class RocksDBSession implements AutoCloseable, Cloneable { db.setAllowMmapWrites(conf.get(RocksDBOptions.ALLOW_MMAP_WRITES)); db.setAllowMmapReads(conf.get(RocksDBOptions.ALLOW_MMAP_READS)); db.setUseDirectReads(conf.get(RocksDBOptions.USE_DIRECT_READS)); - db.setUseDirectIoForFlushAndCompaction(conf.get(RocksDBOptions.USE_DIRECT_READS_WRITES_FC)); + db.setUseDirectIoForFlushAndCompaction( + conf.get(RocksDBOptions.USE_DIRECT_READS_WRITES_FC)); db.setMaxManifestFileSize(conf.get(RocksDBOptions.MAX_MANIFEST_FILE_SIZE)); db.setSkipStatsUpdateOnDbOpen(conf.get(RocksDBOptions.SKIP_STATS_UPDATE_ON_DB_OPEN)); db.setMaxFileOpeningThreads(conf.get(RocksDBOptions.MAX_FILE_OPENING_THREADS)); @@ -524,7 +536,8 @@ public class RocksDBSession implements AutoCloseable, Cloneable { mdb.setDelayedWriteRate(conf.get(RocksDBOptions.DELAYED_WRITE_RATE)); mdb.setMaxOpenFiles(conf.get(RocksDBOptions.MAX_OPEN_FILES)); mdb.setMaxTotalWalSize(conf.get(RocksDBOptions.MAX_TOTAL_WAL_SIZE)); - mdb.setDeleteObsoleteFilesPeriodMicros(1000000 * conf.get(RocksDBOptions.DELETE_OBSOLETE_FILE_PERIOD)); + mdb.setDeleteObsoleteFilesPeriodMicros( + 1000000 * conf.get(RocksDBOptions.DELETE_OBSOLETE_FILE_PERIOD)); } if (cf != null) { @@ -677,14 +690,9 @@ public class RocksDBSession implements AutoCloseable, Cloneable { @Override public String toString() { - try { - return String.format("%s=%s", - new String(name, "utf-8"), - new String(value, "utf-8")); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - return ""; + return String.format("%s=%s", + new String(name, StandardCharsets.UTF_8), + new String(value, StandardCharsets.UTF_8)); } @Override @@ -735,17 +743,20 @@ public class RocksDBSession implements AutoCloseable, Cloneable { FileUtils.moveDirectory(tempFile, snapshotFile); } catch (IOException e) { log.error("Fail to rename {} to {}", tempPath, snapshotPath, e); - throw new DBStoreException(String.format("Fail to rename %s to %s", tempPath, snapshotPath)); + throw new DBStoreException( + String.format("Fail to rename %s to %s", tempPath, snapshotPath)); } } catch (final DBStoreException e) { throw e; } catch (final Exception e) { log.error("Fail to write snapshot at path: {}", snapshotPath, e); - throw new DBStoreException(String.format("Fail to write snapshot at path %s", snapshotPath)); + throw new DBStoreException( + String.format("Fail to write snapshot at path %s", snapshotPath)); } finally { cfHandleLock.readLock().unlock(); } - log.info("saved snapshot into {}, time cost {} ms", snapshotPath, System.currentTimeMillis() - startTime); + log.info("saved snapshot into {}, time cost {} ms", snapshotPath, + System.currentTimeMillis() - startTime); } @@ -758,7 +769,8 @@ public class RocksDBSession implements AutoCloseable, Cloneable { if (columnFamilyBytes.size() > 0) { for (byte[] columnFamilyByte : columnFamilyBytes) { - columnFamilyDescriptorList.add(new ColumnFamilyDescriptor(columnFamilyByte, cfOptions)); + columnFamilyDescriptorList.add( + new ColumnFamilyDescriptor(columnFamilyByte, cfOptions)); } } else { columnFamilyDescriptorList.add( @@ -768,10 +780,12 @@ public class RocksDBSession implements AutoCloseable, Cloneable { try (final DBOptions dbOptions = new DBOptions(); final RocksDB db = RocksDB.openReadOnly(dbOptions, snapshotPath, - columnFamilyDescriptorList, columnFamilyHandleList)) { + columnFamilyDescriptorList, + columnFamilyHandleList)) { for (ColumnFamilyHandle handle : columnFamilyHandleList) { if (handle == null) { - log.error("verifySnapshot some ColumnFamilyHandle is null in {}", snapshotPath); + log.error("verifySnapshot some ColumnFamilyHandle is null in {}", + snapshotPath); return false; } } @@ -793,7 +807,8 @@ public class RocksDBSession implements AutoCloseable, Cloneable { final File snapshotFile = new File(snapshotPath); if (!snapshotFile.exists()) { log.error("Snapshot file {} not exists.", snapshotPath); - throw new DBStoreException(String.format("Snapshot file %s not exists", snapshotPath)); + throw new DBStoreException( + String.format("Snapshot file %s not exists", snapshotPath)); } // replace rocksdb data with snapshot data @@ -816,11 +831,11 @@ public class RocksDBSession implements AutoCloseable, Cloneable { // create hard link try { Files.createLink(target.toPath(), f.toPath()); - }catch (IOException e) { + } catch (IOException e) { log.error("link failed, {} -> {}, error:{}", - f.getAbsolutePath(), - target.getAbsolutePath(), - e.getMessage()); + f.getAbsolutePath(), + target.getAbsolutePath(), + e.getMessage()); // diff disk Files.copy(f.toPath(), target.toPath()); } @@ -833,11 +848,13 @@ public class RocksDBSession implements AutoCloseable, Cloneable { } catch (IOException e2) { log.error("Fail to delete directory {}. {}", tempDBPath, e2.toString()); } - throw new DBStoreException(String.format("Fail to copy %s to %s", snapshotPath, tempDBPath)); + throw new DBStoreException( + String.format("Fail to copy %s to %s", snapshotPath, tempDBPath)); } // verify temp path if (!verifySnapshot(tempDBPath)) { - throw new DBStoreException(String.format("failed to verify snapshot %s", tempDBPath)); + throw new DBStoreException( + String.format("failed to verify snapshot %s", tempDBPath)); } // move temp to newDBPath @@ -846,17 +863,20 @@ public class RocksDBSession implements AutoCloseable, Cloneable { FileUtils.moveDirectory(new File(tempDBPath), new File(newDBPath)); } catch (IOException e) { log.error("Fail to copy {} to {}. {}", snapshotPath, tempDBPath, e.toString()); - throw new DBStoreException(String.format("Fail to move %s to %s", tempDBPath, newDBPath)); + throw new DBStoreException( + String.format("Fail to move %s to %s", tempDBPath, newDBPath)); } } catch (final DBStoreException e) { throw e; } catch (final Exception e) { log.error("failed to load snapshot from {}", snapshotPath); - throw new DBStoreException(String.format("Fail to write snapshot at path %s", snapshotPath)); + throw new DBStoreException( + String.format("Fail to write snapshot at path %s", snapshotPath)); } finally { cfHandleLock.writeLock().unlock(); } - log.info("loaded snapshot from {}, time cost {} ms", snapshotPath, System.currentTimeMillis() - startTime); + log.info("loaded snapshot from {}, time cost {} ms", snapshotPath, + System.currentTimeMillis() - startTime); } /** @@ -899,7 +919,7 @@ public class RocksDBSession implements AutoCloseable, Cloneable { for (ColumnFamilyHandle h : this.tables.values()) { long[] sizes = this.rocksDB.getApproximateSizes( h, - Arrays.asList(r1), + List.of(r1), SizeApproximationFlag.INCLUDE_FILES, SizeApproximationFlag.INCLUDE_MEMTABLES); @@ -924,7 +944,7 @@ public class RocksDBSession implements AutoCloseable, Cloneable { for (ColumnFamilyHandle h : this.tables.values()) { long[] sizes = this.rocksDB.getApproximateSizes( h, - Arrays.asList(r1), + List.of(r1), SizeApproximationFlag.INCLUDE_FILES, SizeApproximationFlag.INCLUDE_MEMTABLES); String name = new String(h.getDescriptor().getName()); @@ -939,7 +959,8 @@ public class RocksDBSession implements AutoCloseable, Cloneable { return map; } - public Map getKeyCountPerCF(byte[] start, byte[] end, boolean accurate) throws DBStoreException { + public Map getKeyCountPerCF(byte[] start, byte[] end, boolean accurate) throws + DBStoreException { ConcurrentHashMap map = new ConcurrentHashMap<>(this.tables.size()); cfHandleLock.readLock().lock(); try { @@ -973,10 +994,13 @@ public class RocksDBSession implements AutoCloseable, Cloneable { for (Map.Entry> entry : sstFiles.entrySet()) { String cfName = new String(entry.getKey()); try (CFHandleLock cfHandle = this.getCFHandleLock(cfName)) { - try (final IngestExternalFileOptions ingestOptions = new IngestExternalFileOptions() + try (final IngestExternalFileOptions ingestOptions = + new IngestExternalFileOptions() .setMoveFiles(true)) { - this.rocksDB.ingestExternalFile(cfHandle.get(), entry.getValue(), ingestOptions); - log.info("Rocksdb {} ingestSstFile cf:{}, sst: {}", this.graphName, cfName, entry.getValue()); + this.rocksDB.ingestExternalFile(cfHandle.get(), entry.getValue(), + ingestOptions); + log.info("Rocksdb {} ingestSstFile cf:{}, sst: {}", this.graphName, cfName, + entry.getValue()); } } } diff --git a/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperator.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperator.java index b17b1d471..19a750464 100644 --- a/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperator.java +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperator.java @@ -17,19 +17,24 @@ package org.apache.hugegraph.rocksdb.access; -import com.baidu.hugegraph.store.term.HgPair; +import org.apache.hugegraph.store.term.HgPair; public interface SessionOperator { HgPair keyRange(String table); void compactRange(String table) throws DBStoreException; + void compactRange() throws DBStoreException; + void put(String table, byte[] key, byte[] value) throws DBStoreException; ScanIterator scan(String tableName); + ScanIterator scan(String tableName, byte[] prefix); + ScanIterator scan(String tableName, byte[] prefix, int scanType); + ScanIterator scan(String tableName, byte[] keyFrom, byte[] keyTo, int scanType); /** @@ -55,14 +60,19 @@ public interface SessionOperator { void deletePrefix(String table, byte[] key) throws DBStoreException; void deleteRange(String table, byte[] keyFrom, byte[] keyTo) throws DBStoreException; + /** * 删除所有cf指定范围的数据 */ void deleteRange(byte[] keyFrom, byte[] keyTo) throws DBStoreException; + byte[] get(String table, byte[] key) throws DBStoreException; void prepare(); + Integer commit() throws DBStoreException; + void rollback(); + RocksDBSession getDBSession(); } diff --git a/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java index bc886681c..a21d1155b 100644 --- a/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java +++ b/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java @@ -21,7 +21,9 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Iterator; +import org.apache.hugegraph.rocksdb.access.RocksDBSession.CFHandleLock; import org.apache.hugegraph.rocksdb.access.util.Asserts; +import org.apache.hugegraph.store.term.HgPair; import org.rocksdb.ReadOptions; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; @@ -30,9 +32,6 @@ import org.rocksdb.Slice; import org.rocksdb.Snapshot; import org.rocksdb.WriteBatch; -import org.apache.hugegraph.rocksdb.access.RocksDBSession.CFHandleLock; - -import com.baidu.hugegraph.store.term.HgPair; import com.baidu.hugegraph.util.Bytes; import lombok.extern.slf4j.Slf4j; @@ -40,9 +39,9 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class SessionOperatorImpl implements SessionOperator { - private RocksDB db; + private final RocksDB db; private WriteBatch batch; - private RocksDBSession session; + private final RocksDBSession session; public SessionOperatorImpl(RocksDBSession session) { this.session = session; @@ -105,6 +104,7 @@ public class SessionOperatorImpl implements SessionOperator { throw new DBStoreException(e); } } + /* * only support 'long data' operator */ @@ -148,7 +148,7 @@ public class SessionOperatorImpl implements SessionOperator { public void deletePrefix(String table, byte[] key) throws DBStoreException { byte[] keyFrom = key; byte[] keyTo = Arrays.copyOf(key, key.length); - keyTo = this.increaseOne(keyTo); + keyTo = increaseOne(keyTo); try { this.getBatch().deleteRange(session.getCF(table), keyFrom, keyTo); } catch (RocksDBException e) { @@ -323,7 +323,7 @@ public class SessionOperatorImpl implements SessionOperator { if (iterator == null) return null; RocksIterator finalIterator = iterator; return (T) new ScanIterator() { - private ReadOptions holdReadOptions = readOptions; + private final ReadOptions holdReadOptions = readOptions; @Override public boolean hasNext() { @@ -377,7 +377,8 @@ public class SessionOperatorImpl implements SessionOperator { @Override public long estimatedKeyCount(String tableName) throws DBStoreException { try { - return this.rocksdb().getLongProperty(session.getCF(tableName), "rocksdb.estimate-num-keys"); + return this.rocksdb() + .getLongProperty(session.getCF(tableName), "rocksdb.estimate-num-keys"); } catch (RocksDBException e) { throw new DBStoreException(e); } diff --git a/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/SnapshotManagerTest.java b/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/SnapshotManagerTest.java index d66a1158b..d777cd0f0 100644 --- a/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/SnapshotManagerTest.java +++ b/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/SnapshotManagerTest.java @@ -17,17 +17,14 @@ package org.apache.hugegraph.rocksdb.access; -import static org.junit.Assert.assertEquals; - import java.io.File; import java.nio.file.Paths; -import java.util.*; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hugegraph.store.term.HgPair; -import com.baidu.hugegraph.config.HugeConfig; -import com.baidu.hugegraph.config.OptionSpace; -import com.baidu.hugegraph.store.term.HgPair; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; // import org.junit.BeforeClass; // import org.junit.Test; @@ -38,7 +35,9 @@ public class SnapshotManagerTest { // @Test public void testRetrieveIndex() throws DBStoreException { - String snapshotPath = "D:\\Code\\baidu\\starhugegraph\\hugegraph-store\\tmp\\8501\\raft\\1/snapshot\\snapshot_40\\graph_1_p1"; + String snapshotPath = + "D:\\Code\\baidu\\starhugegraph\\hugegraph-store\\tmp\\8501\\raft\\1/snapshot" + + "\\snapshot_40\\graph_1_p1"; long lastIndex = 0L; File file = new File(snapshotPath); File parentFile = new File(file.getParent()); @@ -53,10 +52,13 @@ public class SnapshotManagerTest { } // @Test - public void testFoundMaxDir () { -// String path = "D:\\Code\\baidu\\starhugegraph\\hugegraph-store\\tmp\\8503\\db\\default\\hugegraph\\g\\0_123"; -// String path = "D:\\Code\\baidu\\starhugegraph\\hugegraph-store\\tmp\\8503\\db\\default\\hugegraph\\g\\0"; -// String path = "D:\\Code\\baidu\\starhugegraph\\hugegraph-store\\tmp\\8501\\db\\default\\hugegraph\\g\\0"; + public void testFoundMaxDir() { +// String path = "D:\\Code\\baidu\\starhugegraph\\hugegraph-store\\tmp\\8503\\db\\default +// \\hugegraph\\g\\0_123"; +// String path = "D:\\Code\\baidu\\starhugegraph\\hugegraph-store\\tmp\\8503\\db\\default +// \\hugegraph\\g\\0"; +// String path = "D:\\Code\\baidu\\starhugegraph\\hugegraph-store\\tmp\\8501\\db\\default +// \\hugegraph\\g\\0"; // String path = "D:\\tmp\\db\\0"; // String path = "D:\\tmp\\db\\0_111"; String path = "D:\\tmp\\db\\0_111_22222"; @@ -64,10 +66,11 @@ public class SnapshotManagerTest { int strIndex = file.getName().indexOf("_"); String defaultName; - if(strIndex < 0) + if (strIndex < 0) { defaultName = file.getName(); - else + } else { defaultName = file.getName().substring(0, strIndex); + } String prefix = defaultName + "_"; File parentFile = new File(file.getParent()); final List> dbs = new ArrayList<>(); @@ -79,16 +82,16 @@ public class SnapshotManagerTest { if (!name.startsWith(prefix) && !name.equals(defaultName)) { continue; } - if(name.endsWith(tempSuffix)) { + if (name.endsWith(tempSuffix)) { continue; } long v1 = -1L; long v2 = -1L; - if(name.length() > defaultName.length()) { + if (name.length() > defaultName.length()) { String[] versions = name.substring(prefix.length()).split("_"); - if(versions.length == 1) { + if (versions.length == 1) { v1 = Long.parseLong(versions[0]); - } else if(versions.length == 2) { + } else if (versions.length == 2) { v1 = Long.parseLong(versions[0]); v2 = Long.parseLong(versions[1]); } else { @@ -102,22 +105,25 @@ public class SnapshotManagerTest { // get last index db path String latestDBPath = ""; if (!dbs.isEmpty()) { - dbs.sort((o1, o2) -> o1.getKey().equals(o2.getKey()) ? o1.getValue().compareTo(o2.getValue()) : o1.getKey().compareTo(o2.getKey())); + dbs.sort((o1, o2) -> o1.getKey().equals(o2.getKey()) ? + o1.getValue().compareTo(o2.getValue()) : + o1.getKey().compareTo(o2.getKey())); final int dbCount = dbs.size(); // delete old db for (int i = 0; i < dbCount; i++) { final HgPair pair = dbs.get(i); String curDBName; - if(pair.getKey() == -1L) { + if (pair.getKey() == -1L) { curDBName = defaultName; - } else if (pair.getValue() == -1L){ + } else if (pair.getValue() == -1L) { curDBName = String.format("%s_%d", defaultName, pair.getKey()); } else { - curDBName = String.format("%s_%d_%d", defaultName, pair.getKey(), pair.getValue()); + curDBName = + String.format("%s_%d_%d", defaultName, pair.getKey(), pair.getValue()); } String curDBPath = Paths.get(parentFile.getPath(), curDBName).toString(); - if(i == dbCount-1) { + if (i == dbCount - 1) { latestDBPath = curDBPath; } else { log.info("delete old dbpath {}", curDBPath); @@ -134,8 +140,11 @@ public class SnapshotManagerTest { // @Test public void testDefaultPath() { -// String latestDBPath = "D:\\Code\\baidu\\starhugegraph\\hugegraph-store\\tmp\\8501\\db\\default\\hugegraph\\g\\0_123"; - String latestDBPath = "D:\\Code\\baidu\\starhugegraph\\hugegraph-store\\tmp\\8501\\db\\default\\hugegraph\\g\\0"; +// String latestDBPath = "D:\\Code\\baidu\\starhugegraph\\hugegraph-store\\tmp\\8501\\db +// \\default\\hugegraph\\g\\0_123"; + String latestDBPath = + "D:\\Code\\baidu\\starhugegraph\\hugegraph-store\\tmp\\8501\\db\\default" + + "\\hugegraph\\g\\0"; File file = new File(latestDBPath); String parent = file.getParent(); String defaultName = file.getName().split("_")[0]; @@ -351,7 +360,8 @@ public class SnapshotManagerTest { // String endKey = String.format("%d_%08d", partition_1, 100); // // session.importSnapshot(importDir, partition_1); -// long keyCount = session.sessionOp().keyCount(table, startKey.getBytes(), endKey.getBytes()); +// long keyCount = session.sessionOp().keyCount(table, startKey.getBytes(), endKey +// .getBytes()); // System.out.printf("key count of %s is %d\n", graphName, keyCount); // // @@ -376,7 +386,8 @@ public class SnapshotManagerTest { // String startKey = String.format("%d_%08d", partition_1, index - 200); // String endKey = String.format("%d_%08d", partition_1, index - 100); // -// ScanIterator cfIterator = session.sessionOp().scanRaw(startKey.getBytes(), endKey.getBytes(), seqNum); +// ScanIterator cfIterator = session.sessionOp().scanRaw(startKey.getBytes(), endKey +// .getBytes(), seqNum); // while (cfIterator.hasNext()) { // ScanIterator iterator = cfIterator.next(); // System.out.println("cf name is " + new String(cfIterator.position())); diff --git a/hg-store-test/pom.xml b/hg-store-test/pom.xml index de96e461f..14714ac5d 100644 --- a/hg-store-test/pom.xml +++ b/hg-store-test/pom.xml @@ -21,8 +21,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> hugegraph-store-root - com.baidu.hugegraph - 3.6.3-SNAPSHOT + org.apache.hugegraph + ${revision} 4.0.0 @@ -55,7 +55,8 @@ report-aggregate - ${project.basedir}/../target/site/jacoco + ${project.basedir}/../target/site/jacoco + @@ -67,14 +68,14 @@ - com.baidu.hugegraph + org.apache.hugegraph hg-store-grpc - ${project.version} + ${revision} - com.baidu.hugegraph + org.apache.hugegraph hg-store-common - ${project.version} + ${revision} org.projectlombok @@ -87,9 +88,9 @@ ${log4j2.version} - com.baidu.hugegraph + org.apache.hugegraph hg-pd-client - ${project.version} + ${revision} io.grpc @@ -98,13 +99,13 @@ - com.baidu.hugegraph + org.apache.hugegraph hg-store-core - com.baidu.hugegraph + org.apache.hugegraph hg-pd-grpc - ${project.version} + ${revision} commons-io @@ -185,7 +186,7 @@ compile - com.baidu.hugegraph + org.apache.hugegraph hg-store-client @@ -200,9 +201,9 @@ - com.baidu.hugegraph + org.apache.hugegraph hugegraph-store - 3.6.3-SNAPSHOT + ${revision} compile diff --git a/hg-store-test/src/main/java/client/BaseClientTest.java b/hg-store-test/src/main/java/client/BaseClientTest.java index dd6285abf..4f8a31add 100644 --- a/hg-store-test/src/main/java/client/BaseClientTest.java +++ b/hg-store-test/src/main/java/client/BaseClientTest.java @@ -17,14 +17,15 @@ package client; -import com.baidu.hugegraph.pd.client.PDClient; -import com.baidu.hugegraph.pd.client.PDConfig; -import com.baidu.hugegraph.store.HgStoreClient; +import org.apache.hugegraph.store.HgStoreClient; import org.junit.After; import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.mockito.runners.MockitoJUnitRunner; +import com.baidu.hugegraph.pd.client.PDClient; +import com.baidu.hugegraph.pd.client.PDConfig; + @RunWith(MockitoJUnitRunner.class) public class BaseClientTest { public static HgStoreClient storeClient; @@ -37,7 +38,7 @@ public class BaseClientTest { @BeforeClass public static void beforeClass() { storeClient = HgStoreClient.create(PDConfig.of("127.0.0.1:8686") - .setEnableCache(true)); + .setEnableCache(true)); pdClient = storeClient.getPdClient(); } diff --git a/hg-store-test/src/main/java/client/ChangeShardNumTest.java b/hg-store-test/src/main/java/client/ChangeShardNumTest.java index 508bb4e01..3d8399bcc 100644 --- a/hg-store-test/src/main/java/client/ChangeShardNumTest.java +++ b/hg-store-test/src/main/java/client/ChangeShardNumTest.java @@ -17,26 +17,28 @@ package client; -import com.baidu.hugegraph.pd.common.PDException; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgStoreSession; import org.junit.Assert; import org.junit.Test; + +import com.baidu.hugegraph.pd.common.PDException; +import com.baidu.hugegraph.pd.grpc.Metapb; + import util.HgStoreTestUtil; /** * 测试修改副本数 */ -public class ChangeShardNumTest extends BaseClientTest{ +public class ChangeShardNumTest extends BaseClientTest { @Test public void test3To1() throws PDException { int number = 10000; HgStoreSession session = storeClient.openSession(graphName); HgStoreTestUtil.batchPut(session, tableName, "testKey", number); - try(HgKvIterator iterators = session.scanIterator(tableName)){ + try (HgKvIterator iterators = session.scanIterator(tableName)) { // Assert.assertEquals(number, HgStoreTestUtil.amountOf(iterators)); } @@ -47,13 +49,13 @@ public class ChangeShardNumTest extends BaseClientTest{ pdClient.balancePartition(); } -// @Test + // @Test public void test1To3() throws PDException { int number = 10000; HgStoreSession session = storeClient.openSession(graphName); HgStoreTestUtil.batchPut(session, tableName, "testKey", number); - try(HgKvIterator iterators = session.scanIterator(tableName)){ + try (HgKvIterator iterators = session.scanIterator(tableName)) { Assert.assertEquals(number, HgStoreTestUtil.amountOf(iterators)); } diff --git a/hg-store-test/src/main/java/client/HgAssertTest.java b/hg-store-test/src/main/java/client/HgAssertTest.java index 86a2f4fa4..1ffcb89d9 100644 --- a/hg-store-test/src/main/java/client/HgAssertTest.java +++ b/hg-store-test/src/main/java/client/HgAssertTest.java @@ -23,17 +23,16 @@ import static org.junit.Assert.assertTrue; import java.util.List; import java.util.Map; +import org.apache.hugegraph.store.client.util.HgAssert; import org.junit.Test; -import com.baidu.hugegraph.store.client.util.HgAssert; - public class HgAssertTest { @Test public void testIsTrue1() { // Setup // Run the test - try{ + try { HgAssert.isTrue(false, "message"); } catch (Exception e) { assertTrue(e instanceof IllegalArgumentException); @@ -42,7 +41,7 @@ public class HgAssertTest { @Test public void testIsTrue2() { - try{ + try { HgAssert.isTrue(false, () -> "message"); } catch (Exception e) { assertTrue(e instanceof IllegalArgumentException); @@ -51,7 +50,7 @@ public class HgAssertTest { @Test public void testIsFalse1() { - try{ + try { HgAssert.isFalse(true, "message"); } catch (Exception e) { assertTrue(e instanceof IllegalArgumentException); @@ -60,7 +59,7 @@ public class HgAssertTest { @Test public void testIsFalse2() { - try{ + try { HgAssert.isFalse(true, () -> "message"); } catch (Exception e) { assertTrue(e instanceof IllegalArgumentException); diff --git a/hg-store-test/src/main/java/client/HgKvStoreTest.java b/hg-store-test/src/main/java/client/HgKvStoreTest.java index 02f9bffce..aa0555aa9 100644 --- a/hg-store-test/src/main/java/client/HgKvStoreTest.java +++ b/hg-store-test/src/main/java/client/HgKvStoreTest.java @@ -17,18 +17,18 @@ package client; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgStoreSession; +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toBytes; +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toOwnerKey; +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toStr; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreSession; import org.junit.Assert; import org.junit.Test; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toBytes; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toOwnerKey; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toStr; - -public class HgKvStoreTest extends BaseClientTest{ +public class HgKvStoreTest extends BaseClientTest { public static final String TABLE_NAME = "unit-table"; @Test @@ -40,7 +40,7 @@ public class HgKvStoreTest extends BaseClientTest{ graph0.truncate(); graph1.truncate(); - for (int i=0; i < 3; i++) { + for (int i = 0; i < 3; i++) { HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i); byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i); graph0.put(TABLE_NAME, key, value0); @@ -51,14 +51,14 @@ public class HgKvStoreTest extends BaseClientTest{ HgKvIterator iterator = graph0.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); // log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); Assert.assertEquals(0, toStr(entry.value()).indexOf("g0")); } iterator = graph1.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); // log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); @@ -70,7 +70,7 @@ public class HgKvStoreTest extends BaseClientTest{ iterator = graph1.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); // System.out.println("key:" + toStr(entry.key()) + " value:" + toStr(entry.value())); Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); @@ -85,7 +85,7 @@ public class HgKvStoreTest extends BaseClientTest{ graph0.truncate(); graph1.truncate(); - for (int i=0; i < 2; i++) { + for (int i = 0; i < 2; i++) { HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i); byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i); graph0.put(TABLE_NAME, key, value0); @@ -96,13 +96,13 @@ public class HgKvStoreTest extends BaseClientTest{ HgKvIterator iterator = graph0.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); Assert.assertEquals(0, toStr(entry.value()).indexOf("g0")); } iterator = graph1.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); } @@ -111,13 +111,13 @@ public class HgKvStoreTest extends BaseClientTest{ Assert.assertTrue(graph0.existsTable(TABLE_NAME)); iterator = graph0.scanIterator(TABLE_NAME); Assert.assertFalse(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); } iterator = graph1.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); } @@ -131,7 +131,7 @@ public class HgKvStoreTest extends BaseClientTest{ graph0.truncate(); graph1.truncate(); - for (int i=0; i < 2; i++) { + for (int i = 0; i < 2; i++) { HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i); byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i); graph0.put(TABLE_NAME, key, value0); @@ -142,13 +142,13 @@ public class HgKvStoreTest extends BaseClientTest{ HgKvIterator iterator = graph0.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); Assert.assertEquals(0, toStr(entry.value()).indexOf("g0")); } iterator = graph1.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); } @@ -157,13 +157,13 @@ public class HgKvStoreTest extends BaseClientTest{ Assert.assertTrue(graph0.existsTable(TABLE_NAME)); iterator = graph0.scanIterator(TABLE_NAME); Assert.assertFalse(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); } iterator = graph1.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); } @@ -175,7 +175,7 @@ public class HgKvStoreTest extends BaseClientTest{ HgStoreSession graph0 = storeClient.openSession("hugegraph0"); graph0.truncate(); - for (int i=0; i < 2; i++) { + for (int i = 0; i < 2; i++) { HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i); byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i); graph0.put(TABLE_NAME, key, value0); @@ -183,7 +183,7 @@ public class HgKvStoreTest extends BaseClientTest{ HgKvIterator iterator = graph0.scanIterator(TABLE_NAME); Assert.assertTrue(iterator.hasNext()); - while (iterator.hasNext()){ + while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); Assert.assertEquals(0, toStr(entry.value()).indexOf("g0")); } diff --git a/hg-store-test/src/main/java/client/HgPairTest.java b/hg-store-test/src/main/java/client/HgPairTest.java index 540747945..5a5c63999 100644 --- a/hg-store-test/src/main/java/client/HgPairTest.java +++ b/hg-store-test/src/main/java/client/HgPairTest.java @@ -17,16 +17,17 @@ package client; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; +import org.apache.hugegraph.store.term.Bits; +import org.apache.hugegraph.store.term.HgPair; +import org.apache.hugegraph.store.term.HgTriple; import org.junit.Before; import org.junit.Test; -import com.baidu.hugegraph.store.term.Bits; -import com.baidu.hugegraph.store.term.HgPair; -import com.baidu.hugegraph.store.term.HgTriple; - public class HgPairTest { private HgPair pair; @@ -46,19 +47,19 @@ public class HgPairTest { pair.setValue("value1"); pair.getKey(); pair.getValue(); - assertTrue(new HgPair<>("key1", "value1").equals(pair)); + assertEquals(new HgPair<>("key1", "value1"), pair); var pair2 = new HgPair<>(); pair2.setKey("key1"); pair2.hashCode(); - assertFalse(pair2.equals(pair)); + assertNotEquals(pair2, pair); triple.getZ(); triple.getX(); triple.getY(); triple.toString(); triple.hashCode(); triple.hashCode(); - assertTrue(triple.equals(new HgTriple<>("x", "y", "z"))); - assertFalse(pair2.equals(triple)); + assertEquals(triple, new HgTriple<>("x", "y", "z")); + assertNotEquals(pair2, triple); } @Test @@ -66,14 +67,14 @@ public class HgPairTest { byte[] buf = new byte[4]; Bits.putInt(buf, 0, 3); int i = Bits.getInt(buf, 0); - assertTrue(i == 3); + assertEquals(3, i); buf = new byte[2]; Bits.putShort(buf, 0, 2); int s = Bits.getShort(buf, 0); - assertTrue(s == 2); + assertEquals(2, s); buf = new byte[4]; Bits.put(buf, 0, new byte[]{0, 0, 0, 66}); int toInt = Bits.toInt(buf); - assertTrue(toInt == 66); + assertEquals(66, toInt); } } diff --git a/hg-store-test/src/main/java/client/HgSessionManagerRaftPDTest.java b/hg-store-test/src/main/java/client/HgSessionManagerRaftPDTest.java index 32f73f08b..c163bc190 100644 --- a/hg-store-test/src/main/java/client/HgSessionManagerRaftPDTest.java +++ b/hg-store-test/src/main/java/client/HgSessionManagerRaftPDTest.java @@ -17,25 +17,19 @@ package client; -import com.baidu.hugegraph.pd.client.PDClient; -import com.baidu.hugegraph.pd.client.PDConfig; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgKvStore; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgScanQuery; -import com.baidu.hugegraph.store.HgStoreClient; -import com.baidu.hugegraph.store.HgStoreSession; -import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator; -import com.baidu.hugegraph.store.client.util.HgStoreClientConfig; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import com.baidu.hugegraph.store.client.util.MetricX; -import com.baidu.hugegraph.store.grpc.common.ScanOrderType; -import org.apache.commons.io.FileUtils; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import util.HgStoreTestUtil; +import static client.HgKvStoreTest.TABLE_NAME; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; +import static util.HgStoreTestUtil.GRAPH_NAME; +import static util.HgStoreTestUtil.amountOf; +import static util.HgStoreTestUtil.batchPut; +import static util.HgStoreTestUtil.padLeftZeros; +import static util.HgStoreTestUtil.println; +import static util.HgStoreTestUtil.toAllPartitionKey; +import static util.HgStoreTestUtil.toBytes; +import static util.HgStoreTestUtil.toLong; +import static util.HgStoreTestUtil.toOwnerKey; +import static util.HgStoreTestUtil.toStr; +import static util.HgStoreTestUtil.toSuffix; import java.io.File; import java.io.FileInputStream; @@ -54,19 +48,27 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.stream.Collectors; -import static client.HgKvStoreTest.TABLE_NAME; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; -import static util.HgStoreTestUtil.GRAPH_NAME; -import static util.HgStoreTestUtil.amountOf; -import static util.HgStoreTestUtil.batchPut; -import static util.HgStoreTestUtil.padLeftZeros; -import static util.HgStoreTestUtil.println; -import static util.HgStoreTestUtil.toAllPartitionKey; -import static util.HgStoreTestUtil.toBytes; -import static util.HgStoreTestUtil.toLong; -import static util.HgStoreTestUtil.toOwnerKey; -import static util.HgStoreTestUtil.toStr; -import static util.HgStoreTestUtil.toSuffix; +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvStore; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.HgStoreClient; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.client.grpc.KvCloseableIterator; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.MetricX; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.baidu.hugegraph.pd.client.PDClient; +import com.baidu.hugegraph.pd.client.PDConfig; +import com.baidu.hugegraph.store.grpc.common.ScanOrderType; + +import util.HgStoreTestUtil; public class HgSessionManagerRaftPDTest { private static PDClient pdClient; @@ -125,7 +127,8 @@ public class HgSessionManagerRaftPDTest { HgStoreSession session = getStoreSession(); - Iterator iterator = session.scanIterator(TABLE_NAME, 0, 65535, HgKvStore.SCAN_HASHCODE, EMPTY_BYTES); + Iterator iterator = + session.scanIterator(TABLE_NAME, 0, 65535, HgKvStore.SCAN_HASHCODE, EMPTY_BYTES); System.out.println(amountOf(iterator)); } @@ -201,11 +204,13 @@ public class HgSessionManagerRaftPDTest { System.out.printf("%d entries get from %s\n", map.size(), outputFile.getPath()); HgStoreSession session = getStoreSession(); - List keyList = map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); + List keyList = + map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); List resList = session.batchGetOwner(TABLE_NAME, keyList); Assert.assertTrue( - (resList.stream().map(e -> map.containsKey(toOwnerKey(e.key()))).allMatch(Boolean::booleanValue))); + (resList.stream().map(e -> map.containsKey(toOwnerKey(e.key()))) + .allMatch(Boolean::booleanValue))); } @Test @@ -297,7 +302,7 @@ public class HgSessionManagerRaftPDTest { session.delete(TABLE_NAME, key); value = session.get(TABLE_NAME, key); println("- get " + delKey + ": " + toStr(value)); - Assert.assertTrue(EMPTY_BYTES.equals(value)); + Assert.assertEquals(EMPTY_BYTES, value); } @Test @@ -329,24 +334,26 @@ public class HgSessionManagerRaftPDTest { String rangePrefix = "DEL-RANGE-KEY"; String owner = "batch-delete-owner"; Map map = - batchPut( - session, - TABLE_NAME, - rangePrefix, - 10, - key -> { - return toOwnerKey(owner, key); - }); + batchPut( + session, + TABLE_NAME, + rangePrefix, + 10, + key -> { + return toOwnerKey(owner, key); + }); HgOwnerKey startKey = toOwnerKey(owner, rangePrefix + "-00"); HgOwnerKey endKey = toOwnerKey(owner, rangePrefix + "-05"); Assert.assertTrue(session.deleteRange(TABLE_NAME, startKey, endKey)); - println("- after delete range from [" + toStr(startKey.getKey()) + "] to [" + toStr(endKey.getKey()) + "]"); + println("- after delete range from [" + toStr(startKey.getKey()) + "] to [" + + toStr(endKey.getKey()) + "]"); for (int i = 0; i < 10; i++) { - HgOwnerKey key = toOwnerKey(owner, rangePrefix + "-" + padLeftZeros(String.valueOf(i), 2)); + HgOwnerKey key = + toOwnerKey(owner, rangePrefix + "-" + padLeftZeros(String.valueOf(i), 2)); String value = toStr(session.get(TABLE_NAME, key)); // TODO: [start,end)? @@ -367,14 +374,14 @@ public class HgSessionManagerRaftPDTest { String prefixStr = "DEL-PREFIX-KEY"; String owner = "batch-delete-owner"; Map map = - batchPut( - session, - TABLE_NAME, - prefixStr, - 10, - key -> { - return toOwnerKey(owner, key); - }); + batchPut( + session, + TABLE_NAME, + prefixStr, + 10, + key -> { + return toOwnerKey(owner, key); + }); // printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList())); @@ -406,21 +413,23 @@ public class HgSessionManagerRaftPDTest { HgKvIterator iterator = null; println("-- test 0 element --"); - iterator = session.scanIterator(tableName, toAllPartitionKey("__SCAN-001"), toAllPartitionKey("__SCAN-100"), 0); + iterator = session.scanIterator(tableName, toAllPartitionKey("__SCAN-001"), + toAllPartitionKey("__SCAN-100"), 0); Assert.assertFalse(iterator.hasNext()); try { iterator.next(); - Assert.assertTrue(false); + Assert.fail(); } catch (Throwable t) { println("-- test NoSuchElementException --"); - Assert.assertTrue(NoSuchElementException.class.isInstance(t)); + Assert.assertTrue(t instanceof NoSuchElementException); } for (int i = 1; i <= 10; i++) { limit = i; iterator = - session.scanIterator( - tableName, toAllPartitionKey(keyName + "-0"), toAllPartitionKey(keyName + "-1"), limit); + session.scanIterator( + tableName, toAllPartitionKey(keyName + "-0"), + toAllPartitionKey(keyName + "-1"), limit); count = 0; while (iterator.hasNext()) { count++; @@ -432,8 +441,9 @@ public class HgSessionManagerRaftPDTest { for (int i = 1; i <= 10; i++) { limit = i; iterator = - session.scanIterator( - tableName, toAllPartitionKey(keyName + "-00001"), toAllPartitionKey(keyName + "-00005"), limit); + session.scanIterator( + tableName, toAllPartitionKey(keyName + "-00001"), + toAllPartitionKey(keyName + "-00005"), limit); count = 0; while (iterator.hasNext()) { count++; @@ -448,8 +458,9 @@ public class HgSessionManagerRaftPDTest { limit = 0; iterator = - session.scanIterator( - tableName, toAllPartitionKey(keyName + "-0"), toAllPartitionKey(keyName + "-1"), limit); + session.scanIterator( + tableName, toAllPartitionKey(keyName + "-0"), + toAllPartitionKey(keyName + "-1"), limit); count = 0; while (iterator.hasNext()) { @@ -501,13 +512,13 @@ public class HgSessionManagerRaftPDTest { count = 0; iterator = - session.scanIterator( - tableName, - toAllPartitionKey(keyName + "-000"), - HgStoreClientConst.EMPTY_OWNER_KEY, - 0, - HgKvStore.SCAN_ANY, - EMPTY_BYTES); + session.scanIterator( + tableName, + toAllPartitionKey(keyName + "-000"), + HgStoreClientConst.EMPTY_OWNER_KEY, + 0, + HgKvStore.SCAN_ANY, + EMPTY_BYTES); List positionList = new LinkedList<>(); while (iterator.hasNext()) { HgKvEntry entry = iterator.next(); @@ -618,11 +629,12 @@ public class HgSessionManagerRaftPDTest { List keys = new ArrayList<>(); data.forEach((k, v) -> keys.add(k)); KvCloseableIterator> iterators = - session.scanBatch2( - HgScanQuery.prefixIteratorOf(tableName, keys.iterator(), ScanOrderType.ORDER_NONE) - .builder() - .setScanType(0x40) - .build()); + session.scanBatch2( + HgScanQuery.prefixIteratorOf(tableName, keys.iterator(), + ScanOrderType.ORDER_NONE) + .builder() + .setScanType(0x40) + .build()); while (iterators.hasNext()) { HgKvIterator iterator = iterators.next(); count += amountOf(iterator); @@ -640,14 +652,14 @@ public class HgSessionManagerRaftPDTest { int keyAmt = 300; byte[] owner = "Owner".getBytes(); Map data = - batchPut( - session, - tableName, - keyName, - keyAmt, - key -> { - return toOwnerKey(owner, key); - }); + batchPut( + session, + tableName, + keyName, + keyAmt, + key -> { + return toOwnerKey(owner, key); + }); println("-- Starting scan --"); MetricX metrics = MetricX.ofStart(); long t = System.currentTimeMillis(); @@ -657,12 +669,13 @@ public class HgSessionManagerRaftPDTest { keys.add(toOwnerKey(owner, keyName)); KvCloseableIterator> iterators = - session.scanBatch2( - HgScanQuery.prefixIteratorOf(tableName, keys.iterator(), ScanOrderType.ORDER_NONE) - .builder() - .setScanType(0x40) - .setSkipDegree(1) - .build()); + session.scanBatch2( + HgScanQuery.prefixIteratorOf(tableName, keys.iterator(), + ScanOrderType.ORDER_NONE) + .builder() + .setScanType(0x40) + .setSkipDegree(1) + .build()); while (iterators.hasNext()) { HgKvIterator iterator = iterators.next(); count += amountOf(iterator); diff --git a/hg-store-test/src/main/java/client/HgStoreClientTest.java b/hg-store-test/src/main/java/client/HgStoreClientTest.java index 575533d04..f2a8739f2 100644 --- a/hg-store-test/src/main/java/client/HgStoreClientTest.java +++ b/hg-store-test/src/main/java/client/HgStoreClientTest.java @@ -17,24 +17,26 @@ package client; -import com.baidu.hugegraph.pd.common.PDException; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgKvStore; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgStoreSession; -import util.HgStoreTestUtil; -import org.junit.Assert; -import org.junit.Test; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; import java.util.List; -import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvStore; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreSession; +import org.junit.Assert; +import org.junit.Test; -public class HgStoreClientTest extends BaseClientTest{ +import com.baidu.hugegraph.pd.common.PDException; +import com.baidu.hugegraph.pd.grpc.Metapb; + +import util.HgStoreTestUtil; + +public class HgStoreClientTest extends BaseClientTest { public static final byte[] EMPTY_BYTES = new byte[0]; - private static String graphName = "testGraphName"; + private static final String graphName = "testGraphName"; private static String tableName = "testTableName"; @Test @@ -50,6 +52,7 @@ public class HgStoreClientTest extends BaseClientTest{ Assert.assertEquals(loop, HgStoreTestUtil.amountOf(iterator)); } } + @Test public void testPutData2() { String graphName = "testGraphName2"; @@ -64,6 +67,7 @@ public class HgStoreClientTest extends BaseClientTest{ Assert.assertEquals(loop, HgStoreTestUtil.amountOf(iterator)); } } + @Test public void testScan() throws PDException { @@ -99,8 +103,12 @@ public class HgStoreClientTest extends BaseClientTest{ byte[] start = new byte[]{0x0}; byte[] end = new byte[]{-1}; try (HgKvIterator iterator = session.scanIterator(tableName, - HgOwnerKey.of(ALL_PARTITION_OWNER, start), - HgOwnerKey.of(ALL_PARTITION_OWNER, end))) { + HgOwnerKey.of( + ALL_PARTITION_OWNER, + start), + HgOwnerKey.of( + ALL_PARTITION_OWNER, + end))) { iterator.seek(position); while (iterator.hasNext()) { iterator.next(); @@ -127,8 +135,8 @@ public class HgStoreClientTest extends BaseClientTest{ HgStoreSession session = storeClient.openSession(graphName); int ownerCode = 1; HgStoreTestUtil.batchPut(session, tableName, "T", 10, (key) -> { - return HgStoreTestUtil.toOwnerKey(ownerCode, key); - } + return HgStoreTestUtil.toOwnerKey(ownerCode, key); + } ); try (HgKvIterator iterators = session.scanIterator(tableName)) { // while (iterators.hasNext()){ @@ -191,10 +199,13 @@ public class HgStoreClientTest extends BaseClientTest{ HgStoreSession session = storeClient.openSession("DEFAULT/hugegraph/g"); for (Metapb.Partition partition : partitions) { try (HgKvIterator iterators = session.scanIterator("g+v", - (int) (partition.getStartKey()), (int) (partition.getEndKey()), - HgKvStore.SCAN_HASHCODE, EMPTY_BYTES)) { + (int) (partition.getStartKey()), + (int) (partition.getEndKey()), + HgKvStore.SCAN_HASHCODE, + EMPTY_BYTES)) { - System.out.println(" " + partition.getId() + " " + HgStoreTestUtil.amountOf(iterators)); + System.out.println( + " " + partition.getId() + " " + HgStoreTestUtil.amountOf(iterators)); } } } diff --git a/hg-store-test/src/main/java/client/HgStoreNodeStateTest.java b/hg-store-test/src/main/java/client/HgStoreNodeStateTest.java index e68aeeaa9..ae59bb81b 100644 --- a/hg-store-test/src/main/java/client/HgStoreNodeStateTest.java +++ b/hg-store-test/src/main/java/client/HgStoreNodeStateTest.java @@ -17,14 +17,15 @@ package client; -import com.baidu.hugegraph.store.client.HgStoreNodeManager; -import org.junit.Test; - -import java.util.concurrent.atomic.AtomicInteger; import static util.HgStoreTestUtil.GRAPH_NAME; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hugegraph.store.client.HgStoreNodeManager; +import org.junit.Test; + public class HgStoreNodeStateTest { - private static final HgStoreNodeManager NODE_MANAGER = HgStoreNodeManager.getInstance(); + private static final HgStoreNodeManager NODE_MANAGER = HgStoreNodeManager.getInstance(); static int nodeNumber = 0; static { @@ -34,7 +35,9 @@ public class HgStoreNodeStateTest { } private static void registerNode(String graphName, Long nodeId, String address) { - NODE_MANAGER.addNode(graphName, NODE_MANAGER.getNodeBuilder().setNodeId(nodeId).setAddress(address).build()); + NODE_MANAGER.addNode(graphName, + NODE_MANAGER.getNodeBuilder().setNodeId(nodeId).setAddress(address) + .build()); } @@ -44,13 +47,13 @@ public class HgStoreNodeStateTest { for (int i = 0; i < 100; i++) { NODE_MANAGER.getStoreNodes(GRAPH_NAME) - .stream().map( - node -> { - System.out.println(node.getNodeId() + " " + node.getAddress() - + "is healthy: " + node.isHealthy()); - return node.isHealthy(); - } - ).count(); + .stream().map( + node -> { + System.out.println(node.getNodeId() + " " + node.getAddress() + + "is healthy: " + node.isHealthy()); + return node.isHealthy(); + } + ).count(); Thread.yield(); } diff --git a/hg-store-test/src/main/java/common/ByteBufferAllocatorTest.java b/hg-store-test/src/main/java/common/ByteBufferAllocatorTest.java index 8722d6b06..b38ab5f26 100644 --- a/hg-store-test/src/main/java/common/ByteBufferAllocatorTest.java +++ b/hg-store-test/src/main/java/common/ByteBufferAllocatorTest.java @@ -17,12 +17,13 @@ package common; -import com.baidu.hugegraph.store.buffer.ByteBufferAllocator; -import lombok.extern.slf4j.Slf4j; +import java.util.concurrent.CountDownLatch; + +import org.apache.hugegraph.store.buffer.ByteBufferAllocator; import org.junit.Assert; import org.junit.Test; -import java.util.concurrent.CountDownLatch; +import lombok.extern.slf4j.Slf4j; @Slf4j public class ByteBufferAllocatorTest extends BaseCommonTest { diff --git a/hg-store-test/src/main/java/common/KVByteBufferTest.java b/hg-store-test/src/main/java/common/KVByteBufferTest.java index 621186e81..460c49523 100644 --- a/hg-store-test/src/main/java/common/KVByteBufferTest.java +++ b/hg-store-test/src/main/java/common/KVByteBufferTest.java @@ -17,19 +17,20 @@ package common; -import com.baidu.hugegraph.store.buffer.KVByteBuffer; -import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.nio.ByteBuffer; import java.util.Arrays; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.apache.hugegraph.store.buffer.KVByteBuffer; +import org.junit.Test; public class KVByteBufferTest { @Test - public void testOps(){ + public void testOps() { KVByteBuffer buffer1 = new KVByteBuffer(10); buffer1.put((byte) 10); // just put a byte @@ -50,14 +51,14 @@ public class KVByteBufferTest { byte[] bytes = new byte[]{10, 20, 30}; KVByteBuffer buffer2 = new KVByteBuffer(bytes); - assertTrue(Arrays.equals(buffer2.array(), bytes)); + assertArrayEquals(buffer2.array(), bytes); ByteBuffer bb = ByteBuffer.allocate(10); KVByteBuffer buffer3 = new KVByteBuffer(bb); buffer3.put(bytes); buffer3.clear(); - assertTrue(Arrays.equals(buffer3.getBytes(), bytes)); + assertArrayEquals(buffer3.getBytes(), bytes); // int (4) + byte(3) assertEquals(7, buffer3.getBuffer().position()); diff --git a/hg-store-test/src/main/java/core/HgBusinessImplTest.java b/hg-store-test/src/main/java/core/HgBusinessImplTest.java index 3eac87018..983e32162 100644 --- a/hg-store-test/src/main/java/core/HgBusinessImplTest.java +++ b/hg-store-test/src/main/java/core/HgBusinessImplTest.java @@ -17,22 +17,23 @@ package core; -import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.junit.Test; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.apache.hugegraph.rocksdb.access.RocksDBFactory; import org.apache.hugegraph.rocksdb.access.RocksDBSession; import org.apache.hugegraph.rocksdb.access.ScanIterator; -import com.baidu.hugegraph.store.business.BusinessHandler; +import org.apache.hugegraph.store.business.BusinessHandler; +import org.junit.Assert; +import org.junit.Test; + import com.baidu.hugegraph.util.Bytes; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; @Slf4j -public class HgBusinessImplTest extends StoreEngineTestBase{ +public class HgBusinessImplTest extends StoreEngineTestBase { public BusinessHandler getBusinessHandler() { return getStoreEngine().getBusinessHandler(); @@ -84,7 +85,8 @@ public class HgBusinessImplTest extends StoreEngineTestBase{ Assert.assertEquals(4, count); System.out.println("--------------------dump scan range key1 key4 -------"); - iterator = handler.scan(graph1, 0, table, "key1".getBytes(), "key4".getBytes(), ScanIterator.Trait.SCAN_LT_END); + iterator = handler.scan(graph1, 0, table, "key1".getBytes(), "key4".getBytes(), + ScanIterator.Trait.SCAN_LT_END); count = 0; while (iterator.hasNext()) { RocksDBSession.BackendColumn entry = iterator.next(); @@ -131,27 +133,30 @@ public class HgBusinessImplTest extends StoreEngineTestBase{ } CountDownLatch latch = new CountDownLatch(1); - RocksDBFactory.getInstance().addRocksdbChangedListener(new RocksDBFactory.RocksdbChangedListener() { - @Override - public void onCompacted(String dbName) { - RocksDBFactory.RocksdbChangedListener.super.onCompacted(dbName); - } + RocksDBFactory.getInstance() + .addRocksdbChangedListener(new RocksDBFactory.RocksdbChangedListener() { + @Override + public void onCompacted(String dbName) { + RocksDBFactory.RocksdbChangedListener.super.onCompacted(dbName); + } - @Override - public void onDBDeleteBegin(String dbName, String filePath) { - RocksDBFactory.RocksdbChangedListener.super.onDBDeleteBegin(dbName, filePath); - } + @Override + public void onDBDeleteBegin(String dbName, String filePath) { + RocksDBFactory.RocksdbChangedListener.super.onDBDeleteBegin(dbName, + filePath); + } - @Override - public void onDBDeleted(String dbName, String filePath) { - latch.countDown(); - } + @Override + public void onDBDeleted(String dbName, String filePath) { + latch.countDown(); + } - @Override - public void onDBSessionReleased(RocksDBSession dbSession) { - RocksDBFactory.RocksdbChangedListener.super.onDBSessionReleased(dbSession); - } - }); + @Override + public void onDBSessionReleased(RocksDBSession dbSession) { + RocksDBFactory.RocksdbChangedListener.super.onDBSessionReleased( + dbSession); + } + }); latch.await(1, TimeUnit.SECONDS); diff --git a/hg-store-test/src/main/java/core/HgCmdClientTest.java b/hg-store-test/src/main/java/core/HgCmdClientTest.java index 199b93e94..9a5421ea9 100644 --- a/hg-store-test/src/main/java/core/HgCmdClientTest.java +++ b/hg-store-test/src/main/java/core/HgCmdClientTest.java @@ -17,28 +17,9 @@ package core; -import com.alipay.sofa.jraft.JRaftUtils; -import com.alipay.sofa.jraft.option.RpcOptions; -import com.alipay.sofa.jraft.util.Endpoint; -import com.baidu.hugegraph.pd.client.PDClient; -import com.baidu.hugegraph.pd.client.PDConfig; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgStoreClient; -import com.baidu.hugegraph.store.HgStoreSession; -import com.baidu.hugegraph.store.cmd.BatchPutRequest; -import com.baidu.hugegraph.store.cmd.BatchPutResponse; -import com.baidu.hugegraph.store.cmd.CleanDataRequest; -import com.baidu.hugegraph.store.cmd.CleanDataResponse; -import com.baidu.hugegraph.store.cmd.HgCmdClient; -import com.baidu.hugegraph.store.meta.Store; -import com.baidu.hugegraph.store.pd.DefaultPdProvider; -import com.baidu.hugegraph.store.pd.PdProvider; -import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.junit.Test; +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toIntBytes; +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toOwnerKey; +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toStr; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -49,9 +30,30 @@ import java.util.LinkedList; import java.util.List; import java.util.concurrent.atomic.AtomicLong; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toIntBytes; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toOwnerKey; -import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toStr; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreClient; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.cmd.BatchPutRequest; +import org.apache.hugegraph.store.cmd.BatchPutResponse; +import org.apache.hugegraph.store.cmd.CleanDataRequest; +import org.apache.hugegraph.store.cmd.CleanDataResponse; +import org.apache.hugegraph.store.cmd.HgCmdClient; +import org.apache.hugegraph.store.meta.Store; +import org.apache.hugegraph.store.pd.DefaultPdProvider; +import org.apache.hugegraph.store.pd.PdProvider; +import org.junit.Assert; +import org.junit.Test; + +import com.alipay.sofa.jraft.JRaftUtils; +import com.alipay.sofa.jraft.option.RpcOptions; +import com.alipay.sofa.jraft.util.Endpoint; +import com.baidu.hugegraph.pd.client.PDClient; +import com.baidu.hugegraph.pd.client.PDConfig; +import com.baidu.hugegraph.pd.grpc.Metapb; + +import lombok.extern.slf4j.Slf4j; @Slf4j public class HgCmdClientTest { @@ -59,9 +61,9 @@ public class HgCmdClientTest { private HgCmdClient hgCmdClient; private HgStoreClient storeClient; private PDClient pdClient; - private String pdAddress = "127.0.0.1:8686"; - private String graphName = "hugegraph"; - private String tableName = "table-1"; + private final String pdAddress = "127.0.0.1:8686"; + private final String graphName = "hugegraph"; + private final String tableName = "table-1"; private static AtomicLong id; @Test @@ -75,8 +77,8 @@ public class HgCmdClientTest { } -// @Test - public void testBatchPut(){ + // @Test + public void testBatchPut() { hgCmdClient = new HgCmdClient(); pdProvider = new DefaultPdProvider(pdAddress); @@ -90,7 +92,7 @@ public class HgCmdClientTest { }); storeClient = HgStoreClient.create(PDConfig.of(pdAddress) - .setEnableCache(true)); + .setEnableCache(true)); HgStoreSession session = storeClient.openSession(graphName); pdClient = storeClient.getPdClient(); session.createTable(tableName); @@ -98,7 +100,7 @@ public class HgCmdClientTest { HgOwnerKey hgOwnerKey = toOwnerKey(createGraph); // 需要写数据，才会创建图 session.put(tableName, - hgOwnerKey, createGraph.getBytes(StandardCharsets.UTF_8)); + hgOwnerKey, createGraph.getBytes(StandardCharsets.UTF_8)); Assert.assertEquals(createGraph, toStr(session.get(tableName, hgOwnerKey))); Integer partId = 0; @@ -108,7 +110,8 @@ public class HgCmdClientTest { for (int i = 1; i <= 3; i++) { key = "key-" + i; BatchPutRequest.KV kv = BatchPutRequest.KV.of(tableName, 1, - key.getBytes(StandardCharsets.UTF_8), key.getBytes(StandardCharsets.UTF_8)); + key.getBytes(StandardCharsets.UTF_8), + key.getBytes(StandardCharsets.UTF_8)); kvs.add(kv); BatchPutRequest request = new BatchPutRequest(); @@ -118,9 +121,9 @@ public class HgCmdClientTest { try { BatchPutResponse response = hgCmdClient.batchPut(request); - if (response == null){ + if (response == null) { log.error("response is null "); - }else if (response.getStatus() == null){ + } else if (response.getStatus() == null) { log.error("response status is null"); } @@ -133,10 +136,10 @@ public class HgCmdClientTest { } - HgKvIterator hgKvIterator = session.scanIterator(tableName); + HgKvIterator hgKvIterator = session.scanIterator(tableName); Assert.assertTrue(hgKvIterator.hasNext()); boolean findKey = false; - while (hgKvIterator.hasNext()){ + while (hgKvIterator.hasNext()) { HgKvEntry entry = hgKvIterator.next(); if (toStr(entry.key()).equals(key) && toStr(entry.value()).equals(key)) { log.info("key={} value={}", toStr(entry.key()), toStr(entry.value())); @@ -146,8 +149,8 @@ public class HgCmdClientTest { Assert.assertTrue(findKey); } -// @Test - public void testCleanData(){ + // @Test + public void testCleanData() { hgCmdClient = new HgCmdClient(); pdProvider = new DefaultPdProvider(pdAddress); @@ -161,7 +164,7 @@ public class HgCmdClientTest { }); storeClient = HgStoreClient.create(PDConfig.of(pdAddress) - .setEnableCache(true)); + .setEnableCache(true)); HgStoreSession session = storeClient.openSession(graphName); pdClient = storeClient.getPdClient(); session.createTable(tableName); @@ -169,7 +172,7 @@ public class HgCmdClientTest { HgOwnerKey hgOwnerKey = toOwnerKey(createGraph); // 需要写数据，才会创建图 session.put(tableName, - hgOwnerKey, createGraph.getBytes(StandardCharsets.UTF_8)); + hgOwnerKey, createGraph.getBytes(StandardCharsets.UTF_8)); Assert.assertEquals(createGraph, toStr(session.get(tableName, hgOwnerKey))); Integer partId = 0; @@ -181,9 +184,9 @@ public class HgCmdClientTest { try { CleanDataResponse response = hgCmdClient.cleanData(request); - if (response == null){ + if (response == null) { log.error("response is null "); - }else if (response.getStatus() == null){ + } else if (response.getStatus() == null) { log.error("response status is null"); } @@ -196,7 +199,7 @@ public class HgCmdClientTest { } @Test - public void testUpdatePartitionLeader(){ + public void testUpdatePartitionLeader() { hgCmdClient = new HgCmdClient(); pdProvider = new DefaultPdProvider(pdAddress); hgCmdClient.init(new RpcOptions(), new HgCmdClient.PartitionAgent() { @@ -209,7 +212,7 @@ public class HgCmdClientTest { }); storeClient = HgStoreClient.create(PDConfig.of(pdAddress) - .setEnableCache(true)); + .setEnableCache(true)); HgStoreSession session = storeClient.openSession(graphName); pdClient = storeClient.getPdClient(); session.createTable(tableName); @@ -217,13 +220,13 @@ public class HgCmdClientTest { HgOwnerKey hgOwnerKey = toOwnerKey(createGraph); // 需要写数据，才会创建图 session.put(tableName, - hgOwnerKey, createGraph.getBytes(StandardCharsets.UTF_8)); + hgOwnerKey, createGraph.getBytes(StandardCharsets.UTF_8)); Assert.assertEquals(createGraph, toStr(session.get(tableName, hgOwnerKey))); } - @Test - public void testData(){ + @Test + public void testData() { hgCmdClient = new HgCmdClient(); pdProvider = new DefaultPdProvider(pdAddress); hgCmdClient.init(new RpcOptions(), new HgCmdClient.PartitionAgent() { @@ -236,23 +239,23 @@ public class HgCmdClientTest { }); storeClient = HgStoreClient.create(PDConfig.of(pdAddress) - .setEnableCache(true)); + .setEnableCache(true)); HgStoreSession session = storeClient.openSession("hugegraphtest"); pdClient = storeClient.getPdClient(); - session.truncate(); + session.truncate(); String tableName = "cli-table"; int loop = 3; - for (int i=0; i < loop; i++) { + for (int i = 0; i < loop; i++) { HgOwnerKey hgOwnerKey = toOwnerKey(i + "owner:" + i, i + "k:" + i); session.put(tableName, hgOwnerKey, toIntBytes(i)); } try { - HgKvIterator iterable = session.scanIterator(tableName); + HgKvIterator iterable = session.scanIterator(tableName); int x = 0; - while (iterable.hasNext()){ + while (iterable.hasNext()) { HgKvEntry entry = iterable.next(); log.info("data:{}-{}", toStr(entry.key()), entry.value()); x++; @@ -264,8 +267,8 @@ public class HgCmdClientTest { } } - @Test - public void testComPressionData(){ + @Test + public void testComPressionData() { hgCmdClient = new HgCmdClient(); pdProvider = new DefaultPdProvider(pdAddress); @@ -279,7 +282,7 @@ public class HgCmdClientTest { }); storeClient = HgStoreClient.create(PDConfig.of(pdAddress) - .setEnableCache(true)); + .setEnableCache(true)); HgStoreSession session = storeClient.openSession("hugegraphtest"); pdClient = storeClient.getPdClient(); session.truncate(); @@ -287,7 +290,7 @@ public class HgCmdClientTest { String tableName = "cli-table"; int loop = 10; - for (int i=0; i < loop; i++) { + for (int i = 0; i < loop; i++) { String key = "d41d8cd98f00b204e9800998ecf8427e" + getMd5("a" + i) + getId(); String value = "10000" + getId() + getId(); HgOwnerKey hgOwnerKey = toOwnerKey("d41d8cd98f00b204e9800998ecf8427e", key); @@ -295,9 +298,9 @@ public class HgCmdClientTest { } try { - HgKvIterator iterable = session.scanIterator(tableName); + HgKvIterator iterable = session.scanIterator(tableName); int x = 0; - while (iterable.hasNext()){ + while (iterable.hasNext()) { HgKvEntry entry = iterable.next(); x++; } @@ -309,12 +312,12 @@ public class HgCmdClientTest { } - - public static String getMd5(String txt){ + public static String getMd5(String txt) { String rs = ""; - String[] hexDigits = { "0", "1", "2", "3", "4", "5", "6", "7", "8","9", "a", "b", "c", "d", "e", "f" }; + String[] hexDigits = + {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"}; try { - MessageDigest messageDigest = MessageDigest.getInstance("MD5"); + MessageDigest messageDigest = MessageDigest.getInstance("MD5"); byte[] b = messageDigest.digest(txt.getBytes()); StringBuffer resultSb = new StringBuffer(); for (int i = 0; i < b.length; i++) { @@ -327,7 +330,7 @@ public class HgCmdClientTest { resultSb.append(hexDigits[d1] + hexDigits[d2]); } rs = resultSb.toString(); - }catch (NoSuchAlgorithmException e) { + } catch (NoSuchAlgorithmException e) { e.printStackTrace(); } return rs; @@ -337,8 +340,9 @@ public class HgCmdClientTest { // 如果需要更长 或者更大冗余空间, 只需要 time * 10^n 即可 // 当前可保证1毫秒 生成 10000条不重复 Long time = Long.valueOf(new SimpleDateFormat("HHmmssSSS").format(new Date())) * 10000 + - (long) (Math.random() * 100); -// Long time = Long.valueOf(new SimpleDateFormat("MMddhhmmssSSS").format(new Date()).toString()); + (long) (Math.random() * 100); +// Long time = Long.valueOf(new SimpleDateFormat("MMddhhmmssSSS").format(new Date()) +// .toString()); // System.out.println(time); if (id == null) { id = new AtomicLong(time); diff --git a/hg-store-test/src/main/java/core/MetricServiceTest.java b/hg-store-test/src/main/java/core/MetricServiceTest.java index f1d9c8835..691a10e54 100644 --- a/hg-store-test/src/main/java/core/MetricServiceTest.java +++ b/hg-store-test/src/main/java/core/MetricServiceTest.java @@ -19,13 +19,13 @@ package core; import java.util.Map; +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.metric.HgMetricService; +import org.apache.hugegraph.store.metric.SystemMetricService; import org.junit.Before; import org.junit.Test; import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.metric.HgMetricService; -import com.baidu.hugegraph.store.metric.SystemMetricService; public class MetricServiceTest { @@ -47,7 +47,7 @@ public class MetricServiceTest { @Test public void testGetSystemMetrics() { - try{ + try { Map systemMetrics = service.getSystemMetrics(); Thread.sleep(1000); systemMetrics = service.getSystemMetrics(); diff --git a/hg-store-test/src/main/java/core/StoreEngineTestBase.java b/hg-store-test/src/main/java/core/StoreEngineTestBase.java index ff71cb10a..402edde7f 100644 --- a/hg-store-test/src/main/java/core/StoreEngineTestBase.java +++ b/hg-store-test/src/main/java/core/StoreEngineTestBase.java @@ -17,46 +17,49 @@ package core; -import com.alipay.sofa.jraft.util.StorageOptionsFactory; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.PartitionEngine; -import com.baidu.hugegraph.store.business.DefaultDataMover; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.meta.ShardGroup; -import com.baidu.hugegraph.store.options.HgStoreEngineOptions; -import com.baidu.hugegraph.store.options.RaftRocksdbOptions; -import com.baidu.hugegraph.store.pd.FakePdServiceProvider; -import lombok.extern.slf4j.Slf4j; +import java.io.File; +import java.util.HashMap; + +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.PartitionEngine; +import org.apache.hugegraph.store.business.DefaultDataMover; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.meta.ShardGroup; +import org.apache.hugegraph.store.options.HgStoreEngineOptions; +import org.apache.hugegraph.store.options.RaftRocksdbOptions; +import org.apache.hugegraph.store.pd.FakePdServiceProvider; import org.junit.AfterClass; import org.junit.BeforeClass; -import util.UnitTestBase; -import java.io.File; -import java.util.ArrayList; -import java.util.HashMap; +import com.alipay.sofa.jraft.util.StorageOptionsFactory; +import com.baidu.hugegraph.pd.grpc.Metapb; + +import lombok.extern.slf4j.Slf4j; +import util.UnitTestBase; /** * 使用FakePd 和 FakePdOptions，初始化HgStoreEngine，该类的各项get函数可用 */ @Slf4j public class StoreEngineTestBase { - private static final String DB_PATH="/tmp/junit"; + private static final String DB_PATH = "/tmp/junit"; @BeforeClass - public static void initEngine(){ + public static void initEngine() { UnitTestBase.deleteDir(new File(DB_PATH)); HgStoreEngineOptions options = new HgStoreEngineOptions(); options.setDataPath(DB_PATH); options.setRaftPath(DB_PATH); options.setFakePD(true); - options.setRocksdbConfig(new HashMap<>(){{put("rocksdb.write_buffer_size", "1048576");}}); + options.setRocksdbConfig(new HashMap<>() {{ + put("rocksdb.write_buffer_size", "1048576"); + }}); options.setGrpcAddress("127.0.0.1:6511"); options.setRaftAddress("127.0.0.1:6510"); options.setDataTransfer(new DefaultDataMover()); - options.setFakePdOptions(new HgStoreEngineOptions.FakePdOptions(){{ + options.setFakePdOptions(new HgStoreEngineOptions.FakePdOptions() {{ setStoreList("127.0.0.1"); setPeersList("127.0.0.1"); setPartitionCount(1); @@ -69,11 +72,11 @@ public class StoreEngineTestBase { HgStoreEngine.getInstance().init(options); } - public static Partition getPartition(int partitionId){ + public static Partition getPartition(int partitionId) { return getPartition(partitionId, "graph0"); } - public static Partition getPartition(int partitionId, String graphName){ + public static Partition getPartition(int partitionId, String graphName) { Partition partition = new Partition(); partition.setId(partitionId); partition.setGraphName(graphName); @@ -86,43 +89,46 @@ public class StoreEngineTestBase { /** * 创建 分区为0的partition engine. 该分区1个shard，为leader, graph name: graph0 + * * @return */ - public static PartitionEngine createPartitionEngine(int partitionId){ + public static PartitionEngine createPartitionEngine(int partitionId) { return createPartitionEngine(partitionId, "graph0"); } - public static PartitionEngine createPartitionEngine(int partitionId, String graphName){ + public static PartitionEngine createPartitionEngine(int partitionId, String graphName) { Metapb.Shard shard = Metapb.Shard.newBuilder() - .setStoreId(FakePdServiceProvider.makeStoreId("127.0.0.1:6511")) - .setRole(Metapb.ShardRole.Leader) - .build(); + .setStoreId(FakePdServiceProvider.makeStoreId( + "127.0.0.1:6511")) + .setRole(Metapb.ShardRole.Leader) + .build(); Metapb.ShardGroup shardGroup = Metapb.ShardGroup.newBuilder() - .setId(partitionId) - .setConfVer(1) - .setVersion(1) - .setState(Metapb.PartitionState.PState_Normal) - .addShards(shard) - .build(); + .setId(partitionId) + .setConfVer(1) + .setVersion(1) + .setState( + Metapb.PartitionState.PState_Normal) + .addShards(shard) + .build(); getStoreEngine().getPartitionManager().updateShardGroup(ShardGroup.from(shardGroup)); - var engine = getStoreEngine().createPartitionEngine(getPartition(partitionId, graphName)); + var engine = getStoreEngine().createPartitionEngine(getPartition(partitionId, graphName)); engine.waitForLeader(2000); return engine; } - public static HgStoreEngine getStoreEngine(){ + public static HgStoreEngine getStoreEngine() { return HgStoreEngine.getInstance(); } @AfterClass - public static void shutDownEngine(){ + public static void shutDownEngine() { try { HgStoreEngine.getInstance().shutdown(); - }catch (Exception e){ + } catch (Exception e) { log.error("shut down engine error: {}", e.getMessage()); } } diff --git a/hg-store-test/src/main/java/core/raft/HgStoreStateMachineTest.java b/hg-store-test/src/main/java/core/raft/HgStoreStateMachineTest.java index 2f67f1097..0b1aba27c 100644 --- a/hg-store-test/src/main/java/core/raft/HgStoreStateMachineTest.java +++ b/hg-store-test/src/main/java/core/raft/HgStoreStateMachineTest.java @@ -17,6 +17,26 @@ package core.raft; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hugegraph.store.raft.HgStoreStateMachine; +import org.apache.hugegraph.store.raft.RaftClosure; +import org.apache.hugegraph.store.raft.RaftOperation; +import org.apache.hugegraph.store.raft.RaftStateListener; +import org.apache.hugegraph.store.raft.RaftTaskHandler; +import org.apache.hugegraph.store.snapshot.HgSnapshotHandler; +import org.apache.hugegraph.store.util.HgStoreException; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.Iterator; import com.alipay.sofa.jraft.Status; @@ -26,25 +46,6 @@ import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.entity.Task; import com.alipay.sofa.jraft.error.RaftError; import com.alipay.sofa.jraft.error.RaftException; -import com.baidu.hugegraph.store.raft.HgStoreStateMachine; -import com.baidu.hugegraph.store.raft.RaftClosure; -import com.baidu.hugegraph.store.raft.RaftOperation; -import com.baidu.hugegraph.store.raft.RaftStateListener; -import com.baidu.hugegraph.store.raft.RaftTaskHandler; -import com.baidu.hugegraph.store.snapshot.HgSnapshotHandler; -import com.baidu.hugegraph.store.util.HgStoreException; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; @RunWith(MockitoJUnitRunner.class) public class HgStoreStateMachineTest { @@ -64,7 +65,8 @@ public class HgStoreStateMachineTest { // Setup final RaftTaskHandler handler = new RaftTaskHandler() { @Override - public boolean invoke(int groupId, byte[] request, RaftClosure response) throws HgStoreException { + public boolean invoke(int groupId, byte[] request, RaftClosure response) throws + HgStoreException { return false; } @@ -117,7 +119,7 @@ public class HgStoreStateMachineTest { RaftOperation op = RaftOperation.create((byte) 0b0); final Task task = new Task(); task.setData(ByteBuffer.wrap(op.getValues())); - task.setDone(new HgStoreStateMachine.RaftClosureAdapter(op, closure->{ + task.setDone(new HgStoreStateMachine.RaftClosureAdapter(op, closure -> { })); @@ -125,8 +127,9 @@ public class HgStoreStateMachineTest { tasks.add(task); // Setup final Iterator inter = new Iterator() { - java.util.Iterator iterator = tasks.iterator(); + final java.util.Iterator iterator = tasks.iterator(); Task task; + @Override public ByteBuffer getData() { return task.getData(); @@ -204,8 +207,9 @@ public class HgStoreStateMachineTest { @Test public void testOnStartFollowing() { // Setup - final LeaderChangeContext ctx = new LeaderChangeContext(new PeerId("ip", 0, 0, 0), "groupId", 0L, - new Status(RaftError.UNKNOWN, "fmt", "args")); + final LeaderChangeContext ctx = + new LeaderChangeContext(new PeerId("ip", 0, 0, 0), "groupId", 0L, + new Status(RaftError.UNKNOWN, "fmt", "args")); // Run the test hgStoreStateMachineUnderTest.onStartFollowing(ctx); @@ -216,8 +220,9 @@ public class HgStoreStateMachineTest { @Test public void testOnStopFollowing() { // Setup - final LeaderChangeContext ctx = new LeaderChangeContext(new PeerId("ip", 0, 0, 0), "groupId", 0L, - new Status(RaftError.UNKNOWN, "fmt", "args")); + final LeaderChangeContext ctx = + new LeaderChangeContext(new PeerId("ip", 0, 0, 0), "groupId", 0L, + new Status(RaftError.UNKNOWN, "fmt", "args")); // Run the test hgStoreStateMachineUnderTest.onStopFollowing(ctx); @@ -229,7 +234,7 @@ public class HgStoreStateMachineTest { public void testOnConfigurationCommitted() { // Setup final Configuration conf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)), - List.of(new PeerId("ip", 0, 0, 0))); + List.of(new PeerId("ip", 0, 0, 0))); // Run the test hgStoreStateMachineUnderTest.onConfigurationCommitted(conf); diff --git a/hg-store-test/src/main/java/core/raft/RaftOperationTest.java b/hg-store-test/src/main/java/core/raft/RaftOperationTest.java index 81cea63b3..04098851e 100644 --- a/hg-store-test/src/main/java/core/raft/RaftOperationTest.java +++ b/hg-store-test/src/main/java/core/raft/RaftOperationTest.java @@ -17,14 +17,12 @@ package core.raft; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.raft.RaftOperation; -import com.google.protobuf.GeneratedMessageV3; +import org.apache.hugegraph.store.raft.RaftOperation; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import com.baidu.hugegraph.pd.grpc.Metapb; +import com.google.protobuf.GeneratedMessageV3; public class RaftOperationTest { @@ -36,7 +34,6 @@ public class RaftOperationTest { } - @Test public void testCreate1() { // Run the test diff --git a/hg-store-test/src/main/java/core/raft/RaftUtilsTest.java b/hg-store-test/src/main/java/core/raft/RaftUtilsTest.java index 28005f0df..e9c66eba6 100644 --- a/hg-store-test/src/main/java/core/raft/RaftUtilsTest.java +++ b/hg-store-test/src/main/java/core/raft/RaftUtilsTest.java @@ -17,6 +17,17 @@ package core.raft; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.util.Collections; +import java.util.List; + +import org.apache.hugegraph.store.raft.util.RaftUtils; +import org.junit.Before; +import org.junit.Test; + import com.alipay.sofa.jraft.Node; import com.alipay.sofa.jraft.NodeManager; import com.alipay.sofa.jraft.conf.Configuration; @@ -24,25 +35,16 @@ import com.alipay.sofa.jraft.core.NodeImpl; import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.option.NodeOptions; import com.alipay.sofa.jraft.util.Endpoint; -import com.baidu.hugegraph.store.raft.util.RaftUtils; -import org.junit.Before; -import org.junit.Test; + import util.UnitTestBase; -import java.io.File; -import java.util.Collections; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - public class RaftUtilsTest { final String ip = "127.0.0.1"; final int port = 12456; final String dataPath = "tmp/raftUtils"; @Before - public void setUp(){ + public void setUp() { UnitTestBase.deleteDir(new File(dataPath)); new File(dataPath).mkdirs(); @@ -113,7 +115,7 @@ public class RaftUtilsTest { public void testGetPeerEndpoints2() { // Setup final Configuration conf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)), - List.of(new PeerId("ip", 0, 0, 0))); + List.of(new PeerId("ip", 0, 0, 0))); // Run the test final List result = RaftUtils.getPeerEndpoints(conf); @@ -135,7 +137,7 @@ public class RaftUtilsTest { nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta"); nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot"); nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer), - Collections.singletonList(peer2))); + Collections.singletonList(peer2))); final Node node = new NodeImpl("unittest", new PeerId(addr, 0)); assertTrue(node.init(nodeOptions)); node.isLeader(true); @@ -152,7 +154,7 @@ public class RaftUtilsTest { public void testGetLearnerEndpoints2() { // Setup final Configuration conf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)), - List.of(new PeerId("ip", 0, 0, 0))); + List.of(new PeerId("ip", 0, 0, 0))); // Run the test final List result = RaftUtils.getLearnerEndpoints(conf); @@ -165,9 +167,9 @@ public class RaftUtilsTest { public void testConfigurationEquals() { // Setup final Configuration oldConf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)), - List.of(new PeerId("ip", 0, 0, 0))); + List.of(new PeerId("ip", 0, 0, 0))); final Configuration newConf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)), - List.of(new PeerId("ip", 0, 0, 0))); + List.of(new PeerId("ip", 0, 0, 0))); // Run the test final boolean result = RaftUtils.configurationEquals(oldConf, newConf); diff --git a/hg-store-test/src/main/java/core/snapshot/HgSnapshotHandlerTest.java b/hg-store-test/src/main/java/core/snapshot/HgSnapshotHandlerTest.java index c0fbf7c38..efd4a6ca3 100644 --- a/hg-store-test/src/main/java/core/snapshot/HgSnapshotHandlerTest.java +++ b/hg-store-test/src/main/java/core/snapshot/HgSnapshotHandlerTest.java @@ -17,16 +17,7 @@ package core.snapshot; -import com.alipay.sofa.jraft.entity.RaftOutter; -import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; -import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.snapshot.HgSnapshotHandler; -import com.google.protobuf.Message; -import core.StoreEngineTestBase; -import org.apache.commons.io.FileUtils; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.assertEquals; import java.io.File; import java.io.IOException; @@ -34,7 +25,18 @@ import java.util.List; import java.util.Map; import java.util.Set; -import static org.junit.Assert.assertEquals; +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.snapshot.HgSnapshotHandler; +import org.junit.Before; +import org.junit.Test; + +import com.alipay.sofa.jraft.entity.RaftOutter; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; +import com.google.protobuf.Message; + +import core.StoreEngineTestBase; public class HgSnapshotHandlerTest extends StoreEngineTestBase { @@ -119,7 +121,8 @@ public class HgSnapshotHandlerTest extends StoreEngineTestBase { // Setup final SnapshotReader reader = new SnapshotReader() { - String path = "/tmp/snapshot"; + final String path = "/tmp/snapshot"; + @Override public RaftOutter.SnapshotMeta load() { return null; @@ -166,7 +169,6 @@ public class HgSnapshotHandlerTest extends StoreEngineTestBase { } - @Test public void testTrimStartPath() { assertEquals("str", HgSnapshotHandler.trimStartPath("str", "prefix")); diff --git a/hg-store-test/src/main/java/core/store/HgStoreEngineTest.java b/hg-store-test/src/main/java/core/store/HgStoreEngineTest.java index fb7f5efb0..ac2ed9f9d 100644 --- a/hg-store-test/src/main/java/core/store/HgStoreEngineTest.java +++ b/hg-store-test/src/main/java/core/store/HgStoreEngineTest.java @@ -17,30 +17,33 @@ package core.store; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.pd.FakePdServiceProvider; -import com.baidu.hugegraph.testutil.Assert; -import core.StoreEngineTestBase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.Arrays; +import java.util.List; + +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.pd.FakePdServiceProvider; import org.junit.Before; import org.junit.Test; -import java.util.Arrays; +import com.baidu.hugegraph.pd.grpc.Metapb; +import com.baidu.hugegraph.testutil.Assert; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertEquals; +import core.StoreEngineTestBase; public class HgStoreEngineTest extends StoreEngineTestBase { private HgStoreEngine engine; @Before - public void setup(){ + public void setup() { engine = HgStoreEngine.getInstance(); } @Test - public void testGetInstance(){ + public void testGetInstance() { assertNotNull(HgStoreEngine.getInstance()); } @@ -54,26 +57,26 @@ public class HgStoreEngineTest extends StoreEngineTestBase { } @Test - public void testCreatePartitionEngine(){ + public void testCreatePartitionEngine() { var partition = getPartition(0); assertNotNull(engine.createPartitionEngine(partition)); } @Test - public void testCreatePartitionGroups(){ + public void testCreatePartitionGroups() { var partition = getPartition(0); engine.createPartitionGroups(partition); } @Test - public void testDestroyPartitionEngine(){ + public void testDestroyPartitionEngine() { createPartitionEngine(16); - engine.destroyPartitionEngine(16, Arrays.asList("graph0")); + engine.destroyPartitionEngine(16, List.of("graph0")); // assertEquals(engine.getPartitionEngines().size(), 0); } @Test - public void testDeletePartition(){ + public void testDeletePartition() { createPartitionEngine(0); engine.deletePartition(0, "graph0"); // TODO: check logic @@ -93,7 +96,7 @@ public class HgStoreEngineTest extends StoreEngineTestBase { } @Test - public void testGetLeaderTerm(){ + public void testGetLeaderTerm() { createPartitionEngine(0); // no vote assertEquals(engine.getLeaderTerm(0), -1); @@ -107,32 +110,32 @@ public class HgStoreEngineTest extends StoreEngineTestBase { } @Test - public void testGetRaftRpcServer(){ + public void testGetRaftRpcServer() { assertNotNull(engine.getRaftRpcServer()); } @Test - public void testGetPartitionManager(){ + public void testGetPartitionManager() { assertNotNull(engine.getPartitionManager()); } @Test - public void testGetDataMover(){ + public void testGetDataMover() { assertNotNull(engine.getDataMover()); } @Test - public void testGetPdProvider(){ + public void testGetPdProvider() { assertNotNull(engine.getPdProvider()); } @Test - public void testGetCmdClient(){ + public void testGetCmdClient() { assertNotNull(engine.getHgCmdClient()); } @Test - public void testGetHeartbeatService(){ + public void testGetHeartbeatService() { assertNotNull(engine.getHeartbeatService()); } @@ -144,29 +147,29 @@ public class HgStoreEngineTest extends StoreEngineTestBase { } @Test - public void testGetDataLocations(){ + public void testGetDataLocations() { assertEquals(engine.getDataLocations().size(), 1); } @Test - public void testGetPartitionEngine(){ + public void testGetPartitionEngine() { createPartitionEngine(0); assertNotNull(engine.getPartitionEngine(0)); } @Test - public void testGetPartitionEngines(){ + public void testGetPartitionEngines() { createPartitionEngine(0); assertEquals(engine.getPartitionEngines().size(), 1); } @Test - public void testGetNodeMetrics(){ + public void testGetNodeMetrics() { assertNotNull(engine.getNodeMetrics()); } @Test - public void testGetRaftGroupCount(){ + public void testGetRaftGroupCount() { createPartitionEngine(0); assertEquals(engine.getRaftGroupCount(), 1); } diff --git a/hg-store-test/src/main/java/core/store/PartitionEngineTest.java b/hg-store-test/src/main/java/core/store/PartitionEngineTest.java index 01b0daa63..63162d4d3 100644 --- a/hg-store-test/src/main/java/core/store/PartitionEngineTest.java +++ b/hg-store-test/src/main/java/core/store/PartitionEngineTest.java @@ -17,76 +17,78 @@ package core.store; -import com.baidu.hugegraph.store.PartitionEngine; -import core.StoreEngineTestBase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import org.apache.hugegraph.store.PartitionEngine; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import core.StoreEngineTestBase; + public class PartitionEngineTest extends StoreEngineTestBase { PartitionEngine engine; @Before - public void setup(){ + public void setup() { engine = createPartitionEngine(0); } @Test - public void testHasPartition(){ + public void testHasPartition() { assertTrue(engine.hasPartition("graph0")); } @Test - public void testGetGroupId(){ + public void testGetGroupId() { assertEquals(engine.getGroupId().intValue(), 0); } @Test - public void testGetShardGroup(){ + public void testGetShardGroup() { assertEquals(engine.getShardGroup().getShards().size(), 1); } @Test - public void testIsLeader(){ + public void testIsLeader() { System.out.println(engine.isLeader()); } @Test - public void testGetLeader(){ + public void testGetLeader() { assertEquals(engine.getLeader().toString(), engine.getOptions().getRaftAddress()); } @Test - public void testGetAlivePeers(){ + public void testGetAlivePeers() { try { System.out.println(engine.getAlivePeers().size()); - }catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); } } @Test - public void testGetRaftNode(){ + public void testGetRaftNode() { assertNotNull(engine.getRaftNode()); } @Test - public void testGetPartitions(){ + public void testGetPartitions() { assertEquals(engine.getPartitions().get("graph0").getId(), 0); } @Test - public void testGetPartition(){ + public void testGetPartition() { assertEquals(engine.getPartition("graph0").getId(), 0); } @Test public void testGetCommittedIndex() throws InterruptedException { Thread.sleep(1000); - System.out.println(engine.getCommittedIndex() ); + System.out.println(engine.getCommittedIndex()); } } diff --git a/hg-store-test/src/main/java/core/store/PartitionInstructionProcessorTest.java b/hg-store-test/src/main/java/core/store/PartitionInstructionProcessorTest.java index 36772f98a..2847054af 100644 --- a/hg-store-test/src/main/java/core/store/PartitionInstructionProcessorTest.java +++ b/hg-store-test/src/main/java/core/store/PartitionInstructionProcessorTest.java @@ -17,42 +17,44 @@ package core.store; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.pd.grpc.pulse.DbCompaction; -import com.baidu.hugegraph.pd.grpc.pulse.SplitPartition; -import com.baidu.hugegraph.pd.grpc.pulse.TransferLeader; -import com.baidu.hugegraph.store.PartitionInstructionProcessor; -import com.baidu.hugegraph.store.pd.FakePdServiceProvider; -import core.StoreEngineTestBase; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.assertEquals; import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.assertEquals; +import org.apache.hugegraph.store.PartitionInstructionProcessor; +import org.apache.hugegraph.store.pd.FakePdServiceProvider; +import org.junit.Before; +import org.junit.Test; + +import com.baidu.hugegraph.pd.grpc.Metapb; +import com.baidu.hugegraph.pd.grpc.pulse.DbCompaction; +import com.baidu.hugegraph.pd.grpc.pulse.SplitPartition; +import com.baidu.hugegraph.pd.grpc.pulse.TransferLeader; + +import core.StoreEngineTestBase; public class PartitionInstructionProcessorTest extends StoreEngineTestBase { PartitionInstructionProcessor processor; @Before - public void init(){ + public void init() { processor = new PartitionInstructionProcessor(getStoreEngine()); } @Test - public void testTransferLeader(){ + public void testTransferLeader() { var engine = createPartitionEngine(0); engine.waitForLeader(1000); var shard = Metapb.Shard.newBuilder() - .setStoreId(FakePdServiceProvider.makeStoreId("127.0.0.1:6511")) - .setRole(Metapb.ShardRole.Leader) - .build(); + .setStoreId(FakePdServiceProvider.makeStoreId("127.0.0.1:6511")) + .setRole(Metapb.ShardRole.Leader) + .build(); TransferLeader trans = TransferLeader.newBuilder() - .setShard(shard) - .build(); + .setShard(shard) + .build(); processor.onTransferLeader(1, getPartition(0), trans, integer -> { assertEquals(0, integer.intValue()); }); @@ -63,8 +65,8 @@ public class PartitionInstructionProcessorTest extends StoreEngineTestBase { var engine = createPartitionEngine(0); engine.waitForLeader(1000); DbCompaction dbCompaction = DbCompaction.newBuilder() - .setTableName("test") - .build(); + .setTableName("test") + .build(); processor.onDbCompaction(2, getPartition(0), dbCompaction, integer -> { assertEquals(0, integer.intValue()); }); @@ -82,17 +84,17 @@ public class PartitionInstructionProcessorTest extends StoreEngineTestBase { List list = new ArrayList<>(); list.add(Metapb.Partition.newBuilder(partition.getProtoObj()) - .setStartKey(0) - .setEndKey(20000) - .build()); + .setStartKey(0) + .setEndKey(20000) + .build()); list.add(Metapb.Partition.newBuilder(partition.getProtoObj()) - .setStartKey(20000) - .setEndKey(65535) - .setId(1).build()); + .setStartKey(20000) + .setEndKey(65535) + .setId(1).build()); SplitPartition splitPartition = SplitPartition.newBuilder() - .addAllNewPartition(list) - .build(); + .addAllNewPartition(list) + .build(); processor.onSplitPartition(3, partition, splitPartition, integer -> { assertEquals(0, integer.intValue()); @@ -103,7 +105,7 @@ public class PartitionInstructionProcessorTest extends StoreEngineTestBase { } @Test - public void testChangeShard(){ + public void testChangeShard() { } } diff --git a/hg-store-test/src/main/java/core/store/meta/MetadataKeyHelperTest.java b/hg-store-test/src/main/java/core/store/meta/MetadataKeyHelperTest.java index 6c8aa843a..e9a44269c 100644 --- a/hg-store-test/src/main/java/core/store/meta/MetadataKeyHelperTest.java +++ b/hg-store-test/src/main/java/core/store/meta/MetadataKeyHelperTest.java @@ -17,20 +17,21 @@ package core.store.meta; -import com.baidu.hugegraph.store.meta.MetadataKeyHelper; -import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertTrue; import java.util.Arrays; -import static org.junit.Assert.assertTrue; +import org.apache.hugegraph.store.meta.MetadataKeyHelper; +import org.junit.Test; public class MetadataKeyHelperTest { @Test - public void testKey(){ - assertTrue(Arrays.equals("HUGEGRAPH/TASK/".getBytes(), MetadataKeyHelper.getTaskPrefix())); - assertTrue(Arrays.equals("HUGEGRAPH/TASK/0/".getBytes(), MetadataKeyHelper.getTaskPrefix(0))); - assertTrue(Arrays.equals("HUGEGRAPH/TASK_DONE/0000000000000000".getBytes(), - MetadataKeyHelper.getDoneTaskKey(0))); + public void testKey() { + assertArrayEquals("HUGEGRAPH/TASK/".getBytes(), MetadataKeyHelper.getTaskPrefix()); + assertArrayEquals("HUGEGRAPH/TASK/0/".getBytes(), MetadataKeyHelper.getTaskPrefix(0)); + assertArrayEquals("HUGEGRAPH/TASK_DONE/0000000000000000".getBytes(), + MetadataKeyHelper.getDoneTaskKey(0)); } } diff --git a/hg-store-test/src/main/java/core/store/meta/PartitionManagerTest.java b/hg-store-test/src/main/java/core/store/meta/PartitionManagerTest.java index 45756bbbb..14a95e7d2 100644 --- a/hg-store-test/src/main/java/core/store/meta/PartitionManagerTest.java +++ b/hg-store-test/src/main/java/core/store/meta/PartitionManagerTest.java @@ -17,64 +17,63 @@ package core.store.meta; -import com.baidu.hugegraph.pd.common.PDException; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.cmd.UpdatePartitionRequest; -import com.baidu.hugegraph.store.meta.Graph; -import com.baidu.hugegraph.store.meta.GraphManager; -import com.baidu.hugegraph.store.meta.Partition; -import com.baidu.hugegraph.store.meta.PartitionManager; -import com.baidu.hugegraph.store.pd.FakePdServiceProvider; -import core.StoreEngineTestBase; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; import java.util.ArrayList; import java.util.List; -import java.util.function.Consumer; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import org.apache.hugegraph.store.cmd.UpdatePartitionRequest; +import org.apache.hugegraph.store.meta.Graph; +import org.apache.hugegraph.store.meta.GraphManager; +import org.apache.hugegraph.store.meta.PartitionManager; +import org.apache.hugegraph.store.pd.FakePdServiceProvider; +import org.junit.Before; +import org.junit.Test; + +import com.baidu.hugegraph.pd.common.PDException; +import com.baidu.hugegraph.pd.grpc.Metapb; + +import core.StoreEngineTestBase; public class PartitionManagerTest extends StoreEngineTestBase { private PartitionManager manager; @Before - public void setup(){ + public void setup() { manager = getStoreEngine().getPartitionManager(); } @Test - public void testGetDeletedFileManager(){ + public void testGetDeletedFileManager() { assertNotNull(manager.getDeletedFileManager()); } @Test - public void testGetPdProvider(){ + public void testGetPdProvider() { assertEquals(manager.getPdProvider().getClass(), FakePdServiceProvider.class); } @Test - public void testGetStoreMetadata(){ + public void testGetStoreMetadata() { assertNotNull(manager.getStoreMetadata()); } @Test - public void testSetStore(){ + public void testSetStore() { var storeId = FakePdServiceProvider.makeStoreId("127.0.0.1:6511"); var store = manager.getStore(storeId); manager.setStore(store); var store2 = manager.getStoreMetadata().getStore(); - assertTrue(store.getId() == store2.getId()); + assertSame(store.getId(), store2.getId()); } @Test - public void testUpdatePartition(){ + public void testUpdatePartition() { var partition = getPartition(5); manager.updatePartition(partition.getProtoObj(), true); @@ -86,7 +85,7 @@ public class PartitionManagerTest extends StoreEngineTestBase { } @Test - public void testChangeState(){ + public void testChangeState() { createPartitionEngine(4); var partition = getPartition(4); manager.changeState(partition, Metapb.PartitionState.PState_Offline); @@ -95,7 +94,7 @@ public class PartitionManagerTest extends StoreEngineTestBase { } @Test - public void testChangeKeyRange(){ + public void testChangeKeyRange() { createPartitionEngine(4); var partition = getPartition(4); manager.changeKeyRange(partition, 1000, 2000); @@ -107,7 +106,7 @@ public class PartitionManagerTest extends StoreEngineTestBase { @Test - public void testUpdatePartitionRangeOrState(){ + public void testUpdatePartitionRangeOrState() { createPartitionEngine(4); UpdatePartitionRequest request = new UpdatePartitionRequest(); request.setPartitionId(4); @@ -124,7 +123,7 @@ public class PartitionManagerTest extends StoreEngineTestBase { } @Test - public void testGetLeaderPartitionIds(){ + public void testGetLeaderPartitionIds() { createPartitionEngine(0); createPartitionEngine(4); createPartitionEngine(5); @@ -133,7 +132,7 @@ public class PartitionManagerTest extends StoreEngineTestBase { } @Test - public void testisLocal(){ + public void testisLocal() { createPartitionEngine(0); assertTrue(manager.isLocalPartition(0)); assertTrue(manager.isLocalPartition(getPartition(0))); @@ -152,35 +151,35 @@ public class PartitionManagerTest extends StoreEngineTestBase { manager.reportTask(null); var partitions = manager.changePartitionToOnLine(list); - assertTrue(partitions.get(0).getState() == Metapb.PartitionState.PState_Normal); + assertSame(partitions.get(0).getState(), Metapb.PartitionState.PState_Normal); // fake pd - assertNotNull(manager.findPartition("graph0",1000)); + assertNotNull(manager.findPartition("graph0", 1000)); } @Test - public void testShards2Peers(){ + public void testShards2Peers() { var storeId = FakePdServiceProvider.makeStoreId("127.0.0.1:6511"); Metapb.Shard shard = Metapb.Shard.newBuilder() - .setStoreId(storeId) - .setRole(Metapb.ShardRole.Leader) - .build(); + .setStoreId(storeId) + .setRole(Metapb.ShardRole.Leader) + .build(); List list = new ArrayList<>(); list.add(shard); var peers = manager.shards2Peers(list); - assertEquals("127.0.0.1:6510" , peers.get(0)); + assertEquals("127.0.0.1:6510", peers.get(0)); } @Test - public void testLoad(){ + public void testLoad() { createPartitionEngine(0); var graphManager = new GraphManager(manager.getOptions(), manager.getPdProvider()); var graph = Metapb.Graph.newBuilder() - .setGraphName("graph0") - .setPartitionCount(12) - .build(); + .setGraphName("graph0") + .setPartitionCount(12) + .build(); graphManager.updateGraph(new Graph(graph)); manager.load(); @@ -191,7 +190,8 @@ public class PartitionManagerTest extends StoreEngineTestBase { public void testSyncPartitionsFromPD() throws PDException { createPartitionEngine(0); // from fake pd - manager.syncPartitionsFromPD(partition -> { }); + manager.syncPartitionsFromPD(partition -> { + }); assertTrue(manager.getPartitions().isEmpty()); } diff --git a/hg-store-test/src/main/java/core/store/meta/TaskManagerTest.java b/hg-store-test/src/main/java/core/store/meta/TaskManagerTest.java index 3e2b4d19a..0a289d050 100644 --- a/hg-store-test/src/main/java/core/store/meta/TaskManagerTest.java +++ b/hg-store-test/src/main/java/core/store/meta/TaskManagerTest.java @@ -17,32 +17,34 @@ package core.store.meta; -import com.baidu.hugegraph.pd.grpc.MetaTask; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.meta.TaskManager; -import com.baidu.hugegraph.store.meta.asynctask.AsyncTaskState; -import com.baidu.hugegraph.store.meta.asynctask.CleanTask; -import core.StoreEngineTestBase; -import org.junit.Before; -import org.junit.Test; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import org.apache.hugegraph.store.meta.TaskManager; +import org.apache.hugegraph.store.meta.asynctask.AsyncTaskState; +import org.apache.hugegraph.store.meta.asynctask.CleanTask; +import org.junit.Before; +import org.junit.Test; + +import com.baidu.hugegraph.pd.grpc.MetaTask; +import com.baidu.hugegraph.pd.grpc.Metapb; + +import core.StoreEngineTestBase; + public class TaskManagerTest extends StoreEngineTestBase { private TaskManager manager; @Before - public void setup(){ + public void setup() { createPartitionEngine(0, "graph0"); manager = getStoreEngine().getPartitionEngine(0).getTaskManager(); } @Test - public void testAsyncTaskReadAndWrite(){ + public void testAsyncTaskReadAndWrite() { var task1 = new CleanTask(0, "graph0", AsyncTaskState.START, null); var task2 = new CleanTask(0, "graph0", AsyncTaskState.FAILED, null); var task3 = new CleanTask(0, "graph0", AsyncTaskState.SUCCESS, null); @@ -68,37 +70,40 @@ public class TaskManagerTest extends StoreEngineTestBase { } @Test - public void testTaskOp(){ + public void testTaskOp() { MetaTask.Task task1 = MetaTask.Task.newBuilder() - .setId(1) - .setState(MetaTask.TaskState.Task_Ready) - .setType(MetaTask.TaskType.Split_Partition) - .setPartition(Metapb.Partition.newBuilder().setGraphName("graph0").setId(0).build()) - .build(); + .setId(1) + .setState(MetaTask.TaskState.Task_Ready) + .setType(MetaTask.TaskType.Split_Partition) + .setPartition(Metapb.Partition.newBuilder() + .setGraphName("graph0") + .setId(0).build()) + .build(); manager.updateTask(task1); assertTrue(manager.taskExists(task1)); assertFalse(manager.taskExists(0, "graph0", - MetaTask.TaskType.Split_Partition.name())); + MetaTask.TaskType.Split_Partition.name())); assertFalse(manager.partitionTaskRepeat(0, "graph0", - MetaTask.TaskType.Split_Partition.name())); + MetaTask.TaskType.Split_Partition.name())); MetaTask.Task task2 = MetaTask.Task.newBuilder(task1).setId(2).build(); manager.updateTask(task2); assertTrue(manager.taskExists(0, "graph0", - MetaTask.TaskType.Split_Partition.name())); + MetaTask.TaskType.Split_Partition.name())); assertTrue(manager.partitionTaskRepeat(0, "graph0", - MetaTask.TaskType.Split_Partition.name())); + MetaTask.TaskType.Split_Partition.name())); MetaTask.Task task3 = MetaTask.Task.newBuilder(task1) - .setId(3) - .setState(MetaTask.TaskState.Task_Success) - .setPartition(Metapb.Partition.newBuilder(task1.getPartition()) - .setGraphName("graph1") - .setId(1) - .build()) - .build(); + .setId(3) + .setState(MetaTask.TaskState.Task_Success) + .setPartition( + Metapb.Partition.newBuilder(task1.getPartition()) + .setGraphName("graph1") + .setId(1) + .build()) + .build(); manager.updateTask(task3); assertFalse(manager.taskExists(task3)); } diff --git a/hg-store-test/src/main/java/core/store/meta/asynctask/CleanTaskTest.java b/hg-store-test/src/main/java/core/store/meta/asynctask/CleanTaskTest.java index 169e061bc..e98c7fd29 100644 --- a/hg-store-test/src/main/java/core/store/meta/asynctask/CleanTaskTest.java +++ b/hg-store-test/src/main/java/core/store/meta/asynctask/CleanTaskTest.java @@ -17,19 +17,20 @@ package core.store.meta.asynctask; -import com.baidu.hugegraph.store.meta.asynctask.AbstractAsyncTask; -import com.baidu.hugegraph.store.meta.asynctask.AsyncTask; -import com.baidu.hugegraph.store.meta.asynctask.AsyncTaskState; -import com.baidu.hugegraph.store.meta.asynctask.CleanTask; -import core.StoreEngineTestBase; +import static org.junit.Assert.assertEquals; + +import org.apache.hugegraph.store.meta.asynctask.AbstractAsyncTask; +import org.apache.hugegraph.store.meta.asynctask.AsyncTask; +import org.apache.hugegraph.store.meta.asynctask.AsyncTaskState; +import org.apache.hugegraph.store.meta.asynctask.CleanTask; import org.junit.Test; -import static org.junit.Assert.assertEquals; +import core.StoreEngineTestBase; public class CleanTaskTest extends StoreEngineTestBase { @Test - public void testSerialize(){ + public void testSerialize() { CleanTask task = new CleanTask(0, "graph0", AsyncTaskState.SUCCESS, null); byte[] bytes = task.toBytes(); diff --git a/hg-store-test/src/main/java/core/store/util/MiscUtilClassTest.java b/hg-store-test/src/main/java/core/store/util/MiscUtilClassTest.java index 74d5ecb15..d12026be4 100644 --- a/hg-store-test/src/main/java/core/store/util/MiscUtilClassTest.java +++ b/hg-store-test/src/main/java/core/store/util/MiscUtilClassTest.java @@ -17,33 +17,32 @@ package core.store.util; -import com.alipay.sofa.jraft.Status; -import com.baidu.hugegraph.store.HgStoreEngine; -import com.baidu.hugegraph.store.util.Asserts; -import com.baidu.hugegraph.store.util.HgRaftError; -import com.baidu.hugegraph.store.util.HgStoreException; -import com.baidu.hugegraph.store.util.ManualResetEvent; -import org.apache.kafka.common.metrics.Stat; -import org.junit.Test; - -import java.util.concurrent.TimeUnit; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.util.concurrent.TimeUnit; + +import org.apache.hugegraph.store.util.Asserts; +import org.apache.hugegraph.store.util.HgRaftError; +import org.apache.hugegraph.store.util.HgStoreException; +import org.apache.hugegraph.store.util.ManualResetEvent; +import org.junit.Test; + +import com.alipay.sofa.jraft.Status; + public class MiscUtilClassTest { @Test - public void testHgRaftError(){ + public void testHgRaftError() { HgRaftError error = HgRaftError.forNumber(0); assertEquals(0, error.getNumber()); assertEquals("OK", error.getMsg()); assertEquals(Status.OK().getCode(), error.toStatus().getCode()); } - @Test (expected = NullPointerException.class) - public void testAsserts(){ + @Test(expected = NullPointerException.class) + public void testAsserts() { assertTrue(Asserts.isInvalid(null)); assertTrue(Asserts.isInvalid()); assertTrue(Asserts.isInvalid(null)); @@ -54,15 +53,15 @@ public class MiscUtilClassTest { Asserts.isNonNull(null, "msg"); } - @Test (expected = IllegalArgumentException.class) - public void testAsserts2(){ + @Test(expected = IllegalArgumentException.class) + public void testAsserts2() { Asserts.isTrue(false, ""); Asserts.isFalse(true, ""); Asserts.isTrue(true, null); } @Test - public void testHgStoreException(){ + public void testHgStoreException() { var exception = new HgStoreException(); assertEquals(0, exception.getCode()); exception = new HgStoreException("invalid"); @@ -91,5 +90,4 @@ public class MiscUtilClassTest { } - } diff --git a/hg-store-test/src/main/java/core/store/util/PartitionMetaStoreWrapperTest.java b/hg-store-test/src/main/java/core/store/util/PartitionMetaStoreWrapperTest.java index 20401d4a9..26c7437e5 100644 --- a/hg-store-test/src/main/java/core/store/util/PartitionMetaStoreWrapperTest.java +++ b/hg-store-test/src/main/java/core/store/util/PartitionMetaStoreWrapperTest.java @@ -17,17 +17,17 @@ package core.store.util; -import com.baidu.hugegraph.pd.grpc.Metapb; -import com.baidu.hugegraph.store.meta.MetadataKeyHelper; -import com.baidu.hugegraph.store.util.PartitionMetaStoreWrapper; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; -import core.StoreEngineTestBase; -import org.junit.Assert; +import org.apache.hugegraph.store.meta.MetadataKeyHelper; +import org.apache.hugegraph.store.util.PartitionMetaStoreWrapper; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import com.baidu.hugegraph.pd.grpc.Metapb; + +import core.StoreEngineTestBase; public class PartitionMetaStoreWrapperTest extends StoreEngineTestBase { @@ -36,23 +36,23 @@ public class PartitionMetaStoreWrapperTest extends StoreEngineTestBase { private Metapb.Partition partition; @Before - public void setup(){ + public void setup() { wrapper = new PartitionMetaStoreWrapper(); partition = Metapb.Partition.newBuilder() - .setId(1) - .setGraphName("graph0") - .setStartKey(0L) - .setEndKey(65535L) - .build(); + .setId(1) + .setGraphName("graph0") + .setStartKey(0L) + .setEndKey(65535L) + .build(); } - public static void putToDb(Metapb.Partition partition, PartitionMetaStoreWrapper wrapper){ + public static void putToDb(Metapb.Partition partition, PartitionMetaStoreWrapper wrapper) { byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId()); wrapper.put(partition.getId(), key, partition.toByteArray()); } @Test - public void testGet(){ + public void testGet() { putToDb(partition, wrapper); byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId()); assertEquals(partition, wrapper.get(1, key, Metapb.Partition.parser())); @@ -61,7 +61,7 @@ public class PartitionMetaStoreWrapperTest extends StoreEngineTestBase { } @Test - public void testPut(){ + public void testPut() { putToDb(partition, wrapper); byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId()); var list = wrapper.scan(partition.getId(), Metapb.Partition.parser(), key); @@ -70,7 +70,7 @@ public class PartitionMetaStoreWrapperTest extends StoreEngineTestBase { } @Test - public void testDelete(){ + public void testDelete() { putToDb(partition, wrapper); byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId()); wrapper.delete(partition.getId(), key); @@ -79,7 +79,7 @@ public class PartitionMetaStoreWrapperTest extends StoreEngineTestBase { } @Test - public void testScan(){ + public void testScan() { putToDb(partition, wrapper); byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId()); var list = wrapper.scan(partition.getId(), Metapb.Partition.parser(), key); diff --git a/hg-store-test/src/main/java/core/store/util/ZipUtilsTest.java b/hg-store-test/src/main/java/core/store/util/ZipUtilsTest.java index 867189508..420266778 100644 --- a/hg-store-test/src/main/java/core/store/util/ZipUtilsTest.java +++ b/hg-store-test/src/main/java/core/store/util/ZipUtilsTest.java @@ -17,11 +17,7 @@ package core.store.util; -import com.baidu.hugegraph.store.util.ZipUtils; -import org.apache.logging.log4j.core.util.FileUtils; -import org.junit.Before; -import org.junit.Test; -import util.UnitTestBase; +import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; @@ -29,7 +25,12 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.zip.CRC32; -import static org.junit.Assert.assertTrue; +import org.apache.hugegraph.store.util.ZipUtils; +import org.apache.logging.log4j.core.util.FileUtils; +import org.junit.Before; +import org.junit.Test; + +import util.UnitTestBase; public class ZipUtilsTest { private static final String ZIP_TEST_PATH = "/tmp/zip_util_test"; diff --git a/hg-store-test/src/main/java/util/CopyOnWriteCacheTest.java b/hg-store-test/src/main/java/util/CopyOnWriteCacheTest.java index 195891a6b..dd399d164 100644 --- a/hg-store-test/src/main/java/util/CopyOnWriteCacheTest.java +++ b/hg-store-test/src/main/java/util/CopyOnWriteCacheTest.java @@ -23,11 +23,10 @@ import static org.junit.Assert.assertTrue; import java.util.Map; import java.util.Set; +import org.apache.hugegraph.store.util.CopyOnWriteCache; import org.junit.Before; import org.junit.Test; -import com.baidu.hugegraph.store.util.CopyOnWriteCache; - public class CopyOnWriteCacheTest { private CopyOnWriteCache writeCache; @@ -43,7 +42,8 @@ public class CopyOnWriteCacheTest { // Run the test writeCache.put("k", "v"); final boolean result = writeCache.containsKey("k"); - Map allKeys = Map.ofEntries(Map.entry("key1", "value1")); + Map allKeys = + Map.ofEntries(Map.entry("key1", "value1")); writeCache.putAll(allKeys); // Verify the results assertTrue(result); diff --git a/hg-store-test/src/main/java/util/HgStoreTestUtil.java b/hg-store-test/src/main/java/util/HgStoreTestUtil.java index e1752a8d9..347e8b0e4 100644 --- a/hg-store-test/src/main/java/util/HgStoreTestUtil.java +++ b/hg-store-test/src/main/java/util/HgStoreTestUtil.java @@ -17,13 +17,6 @@ package util; -import com.baidu.hugegraph.store.HgKvEntry; -import com.baidu.hugegraph.store.HgKvIterator; -import com.baidu.hugegraph.store.HgOwnerKey; -import com.baidu.hugegraph.store.HgStoreSession; -import com.baidu.hugegraph.store.client.util.HgStoreClientConst; -import com.baidu.hugegraph.store.client.util.MetricX; - import java.io.Closeable; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -42,8 +35,15 @@ import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.MetricX; + public class HgStoreTestUtil { - public static final String GRAPH_NAME = "default/hugegraph/g"; + public static final String GRAPH_NAME = "default/hugegraph/g"; public static final String GRAPH_NAME2 = "default/hugegraph2/g"; public static final String TABLE_NAME = "unit-table"; public static final String TABLE_NAME2 = "unit-table-2"; @@ -52,37 +52,40 @@ public class HgStoreTestUtil { return batchPut(session, keyPrefix, 100); } - public static Map batchPut(HgStoreSession session, String keyPrefix, int loop) { + public static Map batchPut(HgStoreSession session, String keyPrefix, + int loop) { return batchPut(session, TABLE_NAME, keyPrefix, loop); } public static Map batchPut( - HgStoreSession session, String tableName, String keyPrefix, int loop) { + HgStoreSession session, String tableName, String keyPrefix, int loop) { return batchPut(session, tableName, keyPrefix, loop, 1, key -> toOwnerKey(key)); } public static Map batchPut( - HgStoreSession session, String tableName, byte[] keyPrefix, int loop) { - return batchPut(session, tableName, keyPrefix, loop, 1, (prefix, key) -> toOwnerKey(prefix, key)); + HgStoreSession session, String tableName, byte[] keyPrefix, int loop) { + return batchPut(session, tableName, keyPrefix, loop, 1, + (prefix, key) -> toOwnerKey(prefix, key)); } public static Map batchPut( - HgStoreSession session, String tableName, String keyPrefix, int loop, int start) { + HgStoreSession session, String tableName, String keyPrefix, int loop, int start) { return batchPut(session, tableName, keyPrefix, loop, start, key -> toOwnerKey(key)); } public static Map batchPut( - HgStoreSession session, String tableName, String keyPrefix, int loop, Function f) { + HgStoreSession session, String tableName, String keyPrefix, int loop, + Function f) { return batchPut(session, tableName, keyPrefix, loop, 1, f); } public static Map batchPut( - HgStoreSession session, - String tableName, - String keyPrefix, - int loop, - int start, - Function f) { + HgStoreSession session, + String tableName, + String keyPrefix, + int loop, + int start, + Function f) { Map res = new LinkedHashMap<>(); @@ -111,12 +114,12 @@ public class HgStoreTestUtil { } public static Map batchPut( - HgStoreSession session, - String tableName, - byte[] keyPrefix, - int loop, - int start, - BiFunction f) { + HgStoreSession session, + String tableName, + byte[] keyPrefix, + int loop, + int start, + BiFunction f) { Map res = new LinkedHashMap<>(); @@ -400,7 +403,7 @@ public class HgStoreTestUtil { break; } } - if (Closeable.class.isInstance(iterator)) { + if (iterator instanceof Closeable) { try { ((Closeable) iterator).close(); } catch (Exception e) { @@ -423,21 +426,23 @@ public class HgStoreTestUtil { } } - public static void parallelTest(int threads, Runnable runner, Consumer throwableConsumer) { + public static void parallelTest(int threads, Runnable runner, + Consumer throwableConsumer) { int threadsAmount = threads; CountDownLatch countDownLatch = new CountDownLatch(threadsAmount); ExecutorService pool = new ThreadPoolExecutor(threadsAmount, threadsAmount + 20, - 200, TimeUnit.SECONDS, new ArrayBlockingQueue( 1000)); + 200, TimeUnit.SECONDS, + new ArrayBlockingQueue(1000)); for (int i = 0; i < threadsAmount; i++) { pool.submit( - () -> { - try { - runner.run(); - } catch (Throwable t) { - throwableConsumer.accept(t); - } - countDownLatch.countDown(); - }); + () -> { + try { + runner.run(); + } catch (Throwable t) { + throwableConsumer.accept(t); + } + countDownLatch.countDown(); + }); } try { diff --git a/hg-store-test/src/main/java/util/UnitTestBase.java b/hg-store-test/src/main/java/util/UnitTestBase.java index cf5159cf1..349de8e16 100644 --- a/hg-store-test/src/main/java/util/UnitTestBase.java +++ b/hg-store-test/src/main/java/util/UnitTestBase.java @@ -23,13 +23,13 @@ import java.util.Map; import org.apache.hugegraph.rocksdb.access.RocksDBFactory; import org.apache.hugegraph.rocksdb.access.RocksDBSession; -import com.baidu.hugegraph.store.business.BusinessHandler; -import com.baidu.hugegraph.store.business.BusinessHandlerImpl; -import com.baidu.hugegraph.store.meta.PartitionManager; -import com.baidu.hugegraph.store.options.HgStoreEngineOptions; -import com.baidu.hugegraph.store.options.RaftRocksdbOptions; -import com.baidu.hugegraph.store.pd.FakePdServiceProvider; -import com.baidu.hugegraph.store.pd.PdProvider; +import org.apache.hugegraph.store.business.BusinessHandler; +import org.apache.hugegraph.store.business.BusinessHandlerImpl; +import org.apache.hugegraph.store.meta.PartitionManager; +import org.apache.hugegraph.store.options.HgStoreEngineOptions; +import org.apache.hugegraph.store.options.RaftRocksdbOptions; +import org.apache.hugegraph.store.pd.FakePdServiceProvider; +import org.apache.hugegraph.store.pd.PdProvider; public class UnitTestBase { public static boolean deleteDir(File dir) { diff --git a/hg-store-test/src/main/java/util/UnsafeUtilTest.java b/hg-store-test/src/main/java/util/UnsafeUtilTest.java index abaa0ddb6..a8ef6dfe0 100644 --- a/hg-store-test/src/main/java/util/UnsafeUtilTest.java +++ b/hg-store-test/src/main/java/util/UnsafeUtilTest.java @@ -23,12 +23,11 @@ import static org.junit.Assert.assertTrue; import java.lang.reflect.Field; -import lombok.Data; +import org.apache.hugegraph.store.util.UnsafeUtf8Util; +import org.apache.hugegraph.store.util.UnsafeUtil; import org.junit.Test; -import com.baidu.hugegraph.store.util.UnsafeUtf8Util; -import com.baidu.hugegraph.store.util.UnsafeUtil; - +import lombok.Data; import sun.misc.Unsafe; public class UnsafeUtilTest { @@ -36,7 +35,7 @@ public class UnsafeUtilTest { @Test public void testHasUnsafe() { - assertFalse(!UnsafeUtil.hasUnsafe()); + assertTrue(UnsafeUtil.hasUnsafe()); } @Test @@ -66,10 +65,10 @@ public class UnsafeUtilTest { } @Test - public void testUnsafeUtf8Util(){ + public void testUnsafeUtf8Util() { String content = "content"; UnsafeUtf8Util.decodeUtf8(content.getBytes(), 0, content.length()); - byte[] out= new byte[content.length()]; + byte[] out = new byte[content.length()]; UnsafeUtf8Util.encodeUtf8(content, out, 0, content.length()); UnsafeUtf8Util.encodedLength(content); } @@ -80,11 +79,11 @@ public class UnsafeUtilTest { private Object o; private byte b; - public TestObject(){ + public TestObject() { } - public TestObject(Object o, byte b){ + public TestObject(Object o, byte b) { this.o = o; this.b = b; } @@ -105,74 +104,72 @@ public class UnsafeUtilTest { offset = acc.objectFieldOffset(TestObject.class.getDeclaredField("b")); acc.putByte(to, offset, byteValue); byte b = acc.getByte(to, offset); - assertTrue(byteValue == b); + assertEquals(byteValue, b); short shortValue = 1; acc.putShort(to, offset, shortValue); short shortResult = acc.getShort(to, offset); - assertTrue(shortValue == shortResult); + assertEquals(shortValue, shortResult); int intValue = 99; acc.putInt(to, offset, intValue); int i = acc.getInt(to, offset); - assertTrue(intValue == i); + assertEquals(intValue, i); long longValue = 11L; acc.putLong(to, offset, longValue); long l = acc.getLong(to, offset); - assertTrue(longValue == l); + assertEquals(longValue, l); acc.putBoolean(to, offset, false); - assertTrue(!acc.getBoolean(to, offset)); + assertFalse(acc.getBoolean(to, offset)); float f = 1; acc.putFloat(to, offset, f); float v = acc.getFloat(to, offset); - assertTrue(f == v); + assertEquals(f, v, 0.0); double d = 2; acc.putDouble(to, offset, d); double v1 = acc.getDouble(to, offset); - assertTrue(d == v1); + assertEquals(d, v1, 0.0); TestObject o1 = new TestObject(); acc.putObject(to, offset, o1); Object o2 = acc.getObject(to, offset); - assertTrue(o1.equals(o2)); + assertEquals(o1, o2); offset = unsafe.allocateMemory(1024); acc.putByte(offset, byteValue); byte bResult = acc.getByte(offset); - assertTrue(byteValue == bResult); + assertEquals(byteValue, bResult); acc.putShort(offset, shortValue); short aShort1 = acc.getShort(offset); - assertTrue(aShort1 == shortValue); + assertEquals(aShort1, shortValue); acc.putInt(offset, intValue); - assertTrue(acc.getInt(offset) == intValue); + assertEquals(acc.getInt(offset), intValue); acc.putLong(offset, longValue); - assertTrue(acc.getLong(offset) == longValue); + assertEquals(acc.getLong(offset), longValue); unsafe.freeMemory(offset); offset = acc.objectFieldOffset(TestObject.class.getDeclaredField("b")); acc.putByteVolatile(to, offset, byteValue); b = acc.getByteVolatile(to, offset); - assertTrue(byteValue == b); + assertEquals(byteValue, b); acc.putShortVolatile(to, offset, shortValue); shortResult = acc.getShortVolatile(to, offset); - assertTrue(shortValue == shortResult); + assertEquals(shortValue, shortResult); acc.putIntVolatile(to, offset, intValue); i = acc.getIntVolatile(to, offset); - assertTrue(intValue == i); + assertEquals(intValue, i); acc.putLongVolatile(to, offset, longValue); l = acc.getLongVolatile(to, offset); - assertTrue(longValue == l); + assertEquals(longValue, l); acc.putBooleanVolatile(to, offset, false); - assertTrue(!acc.getBooleanVolatile(to, offset)); + assertFalse(acc.getBooleanVolatile(to, offset)); acc.putFloatVolatile(to, offset, f); v = acc.getFloatVolatile(to, offset); - assertTrue(f == v); + assertEquals(f, v, 0.0); acc.putDoubleVolatile(to, offset, d); v1 = acc.getDoubleVolatile(to, offset); - assertTrue(d == v1); + assertEquals(d, v1, 0.0); acc.putObjectVolatile(to, offset, o1); o2 = acc.getObjectVolatile(to, offset); - assertTrue(o1.equals(o2)); + assertEquals(o1, o2); } catch (Exception e) { e.printStackTrace(); - } finally { - } } diff --git a/pom.xml b/pom.xml index 445465441..1d180d7f9 100644 --- a/pom.xml +++ b/pom.xml @@ -1,13 +1,30 @@ + + 4.0.0 - com.baidu.hugegraph + org.apache.hugegraph hugegraph-store-root - 3.6.3-SNAPSHOT + ${revision} pom @@ -29,6 +46,7 @@ + 1.5.0 11 11 3.6.4-SNAPSHOT @@ -38,32 +56,32 @@ - com.baidu.hugegraph + org.apache.hugegraph hg-store-common ${project.version} - com.baidu.hugegraph + org.apache.hugegraph hg-store-grpc ${project.version} - com.baidu.hugegraph + org.apache.hugegraph hg-store-rocksdb ${project.version} - com.baidu.hugegraph + org.apache.hugegraph hg-store-client ${project.version} - com.baidu.hugegraph + org.apache.hugegraph hg-store-core ${project.version} - com.baidu.hugegraph + org.apache.hugegraph hg-store-transfer ${project.version}