Merge branch 'pd-store' into merge-store

hugegraph-pd/.gitattributes

@@ -0,0 +1 @@
+*  text=auto eol=lf

hugegraph-pd/.gitignore

@@ -0,0 +1,13 @@
+/hg-pd-grpc/src/main/java/
+/.idea/
+/hg-pd-grpc/target/
+/dist/
+**/target/
+.DS_Store
+**/tmp/
+*.log
+*.iml
+/hg-pd-common/target_B000000405016P_Oct-28-114458-2021_conflict_parent/
+
+dist/
+.flattened-pom.xml

hugegraph-pd/.mvn/wrapper/MavenWrapperDownloader.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2007-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.net.*;
+import java.io.*;
+import java.nio.channels.*;
+import java.util.Properties;
+
+public class MavenWrapperDownloader {
+
+    private static final String WRAPPER_VERSION = "0.5.6";
+    /**
+     * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
+     */
+    private static final String DEFAULT_DOWNLOAD_URL =
+            "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
+            + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
+
+    /**
+     * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
+     * use instead of the default one.
+     */
+    private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
+            ".mvn/wrapper/maven-wrapper.properties";
+
+    /**
+     * Path where the maven-wrapper.jar will be saved to.
+     */
+    private static final String MAVEN_WRAPPER_JAR_PATH =
+            ".mvn/wrapper/maven-wrapper.jar";
+
+    /**
+     * Name of the property which should be used to override the default download url for the
+     * wrapper.
+     */
+    private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
+
+    public static void main(String args[]) {
+        System.out.println("- Downloader started");
+        File baseDirectory = new File(args[0]);
+        System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
+
+        // If the maven-wrapper.properties exists, read it and check if it contains a custom
+        // wrapperUrl parameter.
+        File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
+        String url = DEFAULT_DOWNLOAD_URL;
+        if (mavenWrapperPropertyFile.exists()) {
+            FileInputStream mavenWrapperPropertyFileInputStream = null;
+            try {
+                mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
+                Properties mavenWrapperProperties = new Properties();
+                mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
+                url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
+            } catch (IOException e) {
+                System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
+            } finally {
+                try {
+                    if (mavenWrapperPropertyFileInputStream != null) {
+                        mavenWrapperPropertyFileInputStream.close();
+                    }
+                } catch (IOException e) {
+                    // Ignore ...
+                }
+            }
+        }
+        System.out.println("- Downloading from: " + url);
+
+        File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
+        if (!outputFile.getParentFile().exists()) {
+            if (!outputFile.getParentFile().mkdirs()) {
+                System.out.println(
+                        "- ERROR creating output directory '" +
+                        outputFile.getParentFile().getAbsolutePath() + "'");
+            }
+        }
+        System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
+        try {
+            downloadFileFromURL(url, outputFile);
+            System.out.println("Done");
+            System.exit(0);
+        } catch (Throwable e) {
+            System.out.println("- Error downloading");
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+
+    private static void downloadFileFromURL(String urlString, File destination) throws Exception {
+        if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
+            String username = System.getenv("MVNW_USERNAME");
+            char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
+            Authenticator.setDefault(new Authenticator() {
+                @Override
+                protected PasswordAuthentication getPasswordAuthentication() {
+                    return new PasswordAuthentication(username, password);
+                }
+            });
+        }
+        URL website = new URL(urlString);
+        ReadableByteChannel rbc;
+        rbc = Channels.newChannel(website.openStream());
+        FileOutputStream fos = new FileOutputStream(destination);
+        fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+        fos.close();
+        rbc.close();
+    }
+
+}

hugegraph-pd/.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to You under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.0/apache-maven-3.6.0-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar

hugegraph-pd/README.md

@@ -0,0 +1,940 @@
+# PD部署说明
+
+## PD配置
+
+- 配置文件在application.yml
+
+````
+license:
+  # 验证使用的配置文件所在目录，包括主题、密码等
+  verify-path: 'conf/verify-license.json'
+  # license文件所在目录，通过hugegraph-signature项目生成
+  license-path: 'conf/hugegraph.license'
+pd:
+  # 存储路径
+  data-path: ./pd_data
+  # 自动扩容的检查周期，定时检查每个Store的分区数量，自动进行分区数量平衡
+  patrol-interval: 1800
+  # 是否允许批量单副本入库
+  enable-batch-load: false
+store:
+  # store下线时间。超过该时间，认为store永久不可用，分配副本到其他机器，单位秒
+  max-down-time: 172800
+partition:
+  # 默认每个分区副本数
+  default-shard-count: 3
+  # 默认每机器最大副本数,初始分区数= store-max-shard-count * store-number / default-shard-count
+  store-max-shard-count: 12
+````
+
+##store配置
+-配置文件在application.yml,配置pdserver的address
+
+````
+pdserver:
+  # pd服务地址，多个pd地址用逗号分割
+  address: pdserver ip:端口
+````
+
+## Hugegraph配置
+
+- 配置项在hugegraph的启动脚本start-hugegraph.sh中
+
+````
+if [ -z "$META_SERVERS" ];then
+  META_SERVERS="pdserver ip:端口"
+fi
+if [ -z "$PD_PEERS" ];then
+  PD_PEERS="pdserver ip:端口"
+fi
+````
+
+## RESTFUL API
+
+- pd提供了一些restful API可以获取集群分区，图，存储节点等一系列信息
+
+###获取集群统计信息
+
+#### 获取集群统计信息
+
+###### Method & Url
+
+```
+GET http://localhost:8620/v1/cluster
+```
+
+###### Response Status
+
+```json
+200
+```
+
+###### Response Body
+
+```json
+{
+  "message": "OK",
+  "data": {
+    "state": "Cluster_OK",
+    "pdList": [
+      {
+        "raftUrl": "10.232.132.38:8610",
+        "grpcUrl": "10.232.132.38:8686",
+        "restUrl": "10.232.132.38:8620",
+        "state": "Up",
+        "dataPath": "./pd_data",
+        "role": "Leader",
+        "serviceName": "10.232.132.38:8686-PD",
+        "serviceVersion": "",
+        "startTimeStamp": 0
+      }
+    ],
+    "pdLeader": {
+      "raftUrl": "10.232.132.38:8610",
+      "grpcUrl": "10.232.132.38:8686",
+      "restUrl": "10.232.132.38:8620",
+      "state": "Up",
+      "dataPath": "./pd_data",
+      "role": "Leader",
+      "serviceName": "10.232.132.38:8686-PD",
+      "serviceVersion": "",
+      "startTimeStamp": 0
+    },
+    "memberSize": 1,
+    "stores": [
+      {
+        "storeId": 110645464809417136,
+        "address": "10.232.132.38:8500",
+        "raftAddress": "10.232.132.38:8510",
+        "version": "3.6.3",
+        "state": "Up"
+      }
+    ],
+    "storeSize": 1,
+    "onlineStoreSize": 1,
+    "offlineStoreSize": 0,
+    "graphSize": 3,
+    "partitionSize": 4,
+    "shardCount": 3,
+    "keyCount": 1707,
+    "dataSize": 19
+  },
+  "status": 0
+}
+```
+
+#### 获取pd集群成员信息
+
+###### Method & Url
+
+```
+GET http://localhost:8620/v1/member
+```
+
+###### Response Status
+
+```json
+200
+```
+
+###### Response Body
+
+```json
+{
+  "message": "OK",
+  "data": {
+    "pdLeader": {
+      "raftUrl": "10.232.132.38:8610",
+      "grpcUrl": "10.232.132.38:8686",
+      "restUrl": "10.232.132.38:8620",
+      "state": "Up",
+      "dataPath": "./pd_data",
+      "role": "Leader",
+      "serviceName": "10.232.132.38:8686-PD",
+      "serviceVersion": "",
+      "startTimeStamp": 0
+    },
+    "pdList": [
+      {
+        "raftUrl": "10.232.132.38:8610",
+        "grpcUrl": "10.232.132.38:8686",
+        "restUrl": "10.232.132.38:8620",
+        "state": "Up",
+        "dataPath": "./pd_data",
+        "role": "Leader",
+        "serviceName": "10.232.132.38:8686-PD",
+        "serviceVersion": "",
+        "startTimeStamp": 0
+      }
+    ],
+    "state": "Cluster_OK"
+  },
+  "status": 0
+}
+```
+
+###存储节点相关
+
+#### 获取集群所有的store的信息
+
+###### Method & Url
+
+```
+GET http://localhost:8620/v1/stores
+```
+
+###### Response Status
+
+```json
+200
+```
+
+###### Request Body
+
+```json
+{
+  "message": "OK",
+  "data": {
+    "stores": [
+      {
+        "storeId": 110645464809417136,
+        "address": "10.232.132.38:8500",
+        "raftAddress": "10.232.132.38:8510",
+        "version": "3.6.3",
+        "state": "Up",
+        "deployPath": "",
+        "startTimeStamp": 1658491024,
+        "lastHeatBeat": 1658491748560,
+        "capacity": 1968740712448,
+        "available": 1959665557504,
+        "partitionCount": 4,
+        "graphSize": 3,
+        "keyCount": 1128,
+        "leaderCount": 4,
+        "serviceName": "10.232.132.38:8500-store",
+        "serviceVersion": "3.6.3",
+        "partitions": [
+          {
+            "partitionId": 0,
+            "graphName": "DEFAULT/hugegraph/s",
+            "role": "Leader",
+            "workState": "PState_Normal"
+          },
+          {
+            "partitionId": 0,
+            "graphName": "DEFAULT/hugegraph/g",
+            "role": "Leader",
+            "workState": "PState_Normal"
+          },
+          {
+            "partitionId": 1,
+            "graphName": "DEFAULT/hugegraph/g",
+            "role": "Leader",
+            "workState": "PState_Normal"
+          },
+          {
+            "partitionId": 2,
+            "graphName": "DEFAULT/hugegraph/g",
+            "role": "Leader",
+            "workState": "PState_Normal"
+          },
+          {
+            "partitionId": 3,
+            "graphName": "DEFAULT/hugegraph/g",
+            "role": "Leader",
+            "workState": "PState_Normal"
+          },
+          {
+            "partitionId": 0,
+            "graphName": "DEFAULT/hugegraph/m",
+            "role": "Leader",
+            "workState": "PState_Normal"
+          }
+        ]
+      }
+    ]
+  },
+  "status": 0
+}
+```
+
+#### 获取单个store的信息
+
+###### Method & Url
+
+```
+GET http://localhost:8620/v1/store/{storeId}
+```
+
+###### Response Status
+
+```json
+200
+```
+
+###### Request Body
+
+```json
+{
+  "message": "OK",
+  "data": {
+    "storeId": 110645464809417136,
+    "address": "10.232.132.38:8500",
+    "raftAddress": "10.232.132.38:8510",
+    "version": "3.6.3",
+    "state": "Up",
+    "deployPath": "",
+    "startTimeStamp": 1658491024,
+    "lastHeatBeat": 1658491838632,
+    "capacity": 1968740712448,
+    "available": 1959665549312,
+    "partitionCount": 4,
+    "graphSize": 3,
+    "keyCount": 1128,
+    "leaderCount": 4,
+    "serviceName": "10.232.132.38:8500-store",
+    "serviceVersion": "3.6.3",
+    "partitions": [
+      {
+        "partitionId": 0,
+        "graphName": "DEFAULT/hugegraph/s",
+        "role": "Leader",
+        "workState": "PState_Normal"
+      },
+      {
+        "partitionId": 0,
+        "graphName": "DEFAULT/hugegraph/g",
+        "role": "Leader",
+        "workState": "PState_Normal"
+      },
+      {
+        "partitionId": 1,
+        "graphName": "DEFAULT/hugegraph/g",
+        "role": "Leader",
+        "workState": "PState_Normal"
+      },
+      {
+        "partitionId": 2,
+        "graphName": "DEFAULT/hugegraph/g",
+        "role": "Leader",
+        "workState": "PState_Normal"
+      },
+      {
+        "partitionId": 3,
+        "graphName": "DEFAULT/hugegraph/g",
+        "role": "Leader",
+        "workState": "PState_Normal"
+      },
+      {
+        "partitionId": 0,
+        "graphName": "DEFAULT/hugegraph/m",
+        "role": "Leader",
+        "workState": "PState_Normal"
+      }
+    ]
+  },
+  "status": 0
+}
+```
+
+### 分区相关
+
+#### 获取分区信息
+
+###### Method & Url
+
+```
+GET http://localhost:8620/v1/highLevelPartitions
+```
+
+###### Response Status
+
+```json
+200
+```
+
+###### Response Body
+
+```json
+{
+  "message": "OK",
+  "data": {
+    "partitions": [
+      {
+        "partitionId": 0,
+        "state": "PState_Normal",
+        "leaderAddress": "10.232.132.38:8500",
+        "keyCount": 0,
+        "dataSize": 0,
+        "shardState": "SState_Normal",
+        "graphs": [
+          {
+            "graphName": "DEFAULT/hugegraph/g",
+            "keyCount": 361,
+            "startKey": 0,
+            "endKey": 0,
+            "dataSize": 8,
+            "workState": "PState_Normal",
+            "partitionId": 0
+          },
+          {
+            "graphName": "DEFAULT/hugegraph/m",
+            "keyCount": 361,
+            "startKey": 0,
+            "endKey": 0,
+            "dataSize": 13,
+            "workState": "PState_Normal",
+            "partitionId": 0
+          },
+          {
+            "graphName": "DEFAULT/hugegraph/s",
+            "keyCount": 361,
+            "startKey": 0,
+            "endKey": 65535,
+            "dataSize": 6,
+            "workState": "PState_Normal",
+            "partitionId": 0
+          }
+        ],
+        "shards": [
+          {
+            "storeId": 110645464809417136,
+            "role": "Leader",
+            "state": "SState_Normal",
+            "progress": 0,
+            "partitionId": 0,
+            "address": "10.232.132.38:8500"
+          }
+        ]
+      },
+      {
+        "partitionId": 1,
+        "state": "PState_Normal",
+        "leaderAddress": "10.232.132.38:8500",
+        "keyCount": 0,
+        "dataSize": 0,
+        "shardState": "SState_Normal",
+        "graphs": [
+          {
+            "graphName": "DEFAULT/hugegraph/g",
+            "keyCount": 8,
+            "startKey": 16384,
+            "endKey": 32768,
+            "dataSize": 5,
+            "workState": "PState_Normal",
+            "partitionId": 1
+          }
+        ],
+        "shards": [
+          {
+            "storeId": 110645464809417136,
+            "role": "Leader",
+            "state": "SState_Normal",
+            "progress": 0,
+            "partitionId": 1,
+            "address": "10.232.132.38:8500"
+          }
+        ]
+      },
+      {
+        "partitionId": 2,
+        "state": "PState_Normal",
+        "leaderAddress": "10.232.132.38:8500",
+        "keyCount": 0,
+        "dataSize": 0,
+        "shardState": "SState_Normal",
+        "graphs": [
+          {
+            "graphName": "DEFAULT/hugegraph/g",
+            "keyCount": 18,
+            "startKey": 32768,
+            "endKey": 49152,
+            "dataSize": 8,
+            "workState": "PState_Normal",
+            "partitionId": 2
+          }
+        ],
+        "shards": [
+          {
+            "storeId": 110645464809417136,
+            "role": "Leader",
+            "state": "SState_Normal",
+            "progress": 0,
+            "partitionId": 2,
+            "address": "10.232.132.38:8500"
+          }
+        ]
+      },
+      {
+        "partitionId": 3,
+        "state": "PState_Normal",
+        "leaderAddress": "10.232.132.38:8500",
+        "keyCount": 0,
+        "dataSize": 0,
+        "shardState": "SState_Normal",
+        "graphs": [
+          {
+            "graphName": "DEFAULT/hugegraph/g",
+            "keyCount": 19,
+            "startKey": 49152,
+            "endKey": 65536,
+            "dataSize": 8,
+            "workState": "PState_Normal",
+            "partitionId": 3
+          }
+        ],
+        "shards": [
+          {
+            "storeId": 110645464809417136,
+            "role": "Leader",
+            "state": "SState_Normal",
+            "progress": 0,
+            "partitionId": 3,
+            "address": "10.232.132.38:8500"
+          }
+        ]
+      }
+    ]
+  },
+  "status": 0
+}
+```
+
+###获取图信息
+
+#### 获取所有的图信息
+
+###### Method & Url
+
+```
+GET http://localhost:8620/v1/graphs
+```
+
+###### Response Status
+
+```json
+200
+```
+
+###### Response Body
+
+```json
+{
+  "message": "OK",
+  "data": {
+    "graphs": [
+      {
+        "graphName": "DEFAULT/hugegraph/g",
+        "partitionCount": 4,
+        "state": "PState_Normal",
+        "partitions": [
+          {
+            "partitionId": 0,
+            "graphName": "DEFAULT/hugegraph/g",
+            "workState": "PState_Normal",
+            "startKey": 0,
+            "endKey": 16384,
+            "shards": [
+              {
+                "partitionId": 0,
+                "storeId": 110645464809417136,
+                "state": "SState_Normal",
+                "role": "Leader",
+                "progress": 0
+              }
+            ]
+          },
+          {
+            "partitionId": 1,
+            "graphName": "DEFAULT/hugegraph/g",
+            "workState": "PState_Normal",
+            "startKey": 16384,
+            "endKey": 32768,
+            "shards": [
+              {
+                "partitionId": 1,
+                "storeId": 110645464809417136,
+                "state": "SState_Normal",
+                "role": "Leader",
+                "progress": 0
+              }
+            ]
+          },
+          {
+            "partitionId": 2,
+            "graphName": "DEFAULT/hugegraph/g",
+            "workState": "PState_Normal",
+            "startKey": 32768,
+            "endKey": 49152,
+            "shards": [
+              {
+                "partitionId": 2,
+                "storeId": 110645464809417136,
+                "state": "SState_Normal",
+                "role": "Leader",
+                "progress": 0
+              }
+            ]
+          },
+          {
+            "partitionId": 3,
+            "graphName": "DEFAULT/hugegraph/g",
+            "workState": "PState_Normal",
+            "startKey": 49152,
+            "endKey": 65536,
+            "shards": [
+              {
+                "partitionId": 3,
+                "storeId": 110645464809417136,
+                "state": "SState_Normal",
+                "role": "Leader",
+                "progress": 0
+              }
+            ]
+          }
+        ],
+        "dataSize": 48,
+        "keyCount": 1128,
+        "nodeCount": 0,
+        "edgeCount": 0
+      },
+      {
+        "graphName": "DEFAULT/hugegraph/m",
+        "partitionCount": 1,
+        "state": "PState_Normal",
+        "partitions": [
+          {
+            "partitionId": 0,
+            "graphName": "DEFAULT/hugegraph/m",
+            "workState": "PState_Normal",
+            "startKey": 0,
+            "endKey": 65535,
+            "shards": [
+              {
+                "partitionId": 0,
+                "storeId": 110645464809417136,
+                "state": "SState_Normal",
+                "role": "Leader",
+                "progress": 0
+              }
+            ]
+          }
+        ],
+        "dataSize": 48,
+        "keyCount": 1128,
+        "nodeCount": 0,
+        "edgeCount": 0
+      },
+      {
+        "graphName": "DEFAULT/hugegraph/s",
+        "partitionCount": 1,
+        "state": "PState_Normal",
+        "partitions": [
+          {
+            "partitionId": 0,
+            "graphName": "DEFAULT/hugegraph/s",
+            "workState": "PState_Normal",
+            "startKey": 0,
+            "endKey": 65535,
+            "shards": [
+              {
+                "partitionId": 0,
+                "storeId": 110645464809417136,
+                "state": "SState_Normal",
+                "role": "Leader",
+                "progress": 0
+              }
+            ]
+          }
+        ],
+        "dataSize": 48,
+        "keyCount": 1128,
+        "nodeCount": 0,
+        "edgeCount": 0
+      }
+    ]
+  },
+  "status": 0
+}
+```
+
+#### 获取单个图信息
+
+###### Method & Url
+
+```
+GET http://localhost:8620/v1/graph/{graphName}
+```
+
+###### Response Status
+
+```json
+200
+```
+
+###### Response Body
+
+```json
+{
+  "message": "OK",
+  "data": {
+    "graphName": "DEFAULT/hugegraph/g",
+    "partitionCount": 4,
+    "state": "PState_Normal",
+    "partitions": [
+      {
+        "partitionId": 0,
+        "graphName": "DEFAULT/hugegraph/g",
+        "workState": "PState_Normal",
+        "startKey": 0,
+        "endKey": 16384,
+        "shards": [
+          {
+            "partitionId": 0,
+            "storeId": 110645464809417136,
+            "state": "SState_Normal",
+            "role": "Leader",
+            "progress": 0
+          }
+        ]
+      },
+      {
+        "partitionId": 1,
+        "graphName": "DEFAULT/hugegraph/g",
+        "workState": "PState_Normal",
+        "startKey": 16384,
+        "endKey": 32768,
+        "shards": [
+          {
+            "partitionId": 1,
+            "storeId": 110645464809417136,
+            "state": "SState_Normal",
+            "role": "Leader",
+            "progress": 0
+          }
+        ]
+      },
+      {
+        "partitionId": 2,
+        "graphName": "DEFAULT/hugegraph/g",
+        "workState": "PState_Normal",
+        "startKey": 32768,
+        "endKey": 49152,
+        "shards": [
+          {
+            "partitionId": 2,
+            "storeId": 110645464809417136,
+            "state": "SState_Normal",
+            "role": "Leader",
+            "progress": 0
+          }
+        ]
+      },
+      {
+        "partitionId": 3,
+        "graphName": "DEFAULT/hugegraph/g",
+        "workState": "PState_Normal",
+        "startKey": 49152,
+        "endKey": 65536,
+        "shards": [
+          {
+            "partitionId": 3,
+            "storeId": 110645464809417136,
+            "state": "SState_Normal",
+            "role": "Leader",
+            "progress": 0
+          }
+        ]
+      }
+    ],
+    "dataSize": 48,
+    "keyCount": 1128,
+    "nodeCount": 0,
+    "edgeCount": 0
+  },
+  "status": 0
+}
+```
+
+###获取shard的信息
+
+#### 获取所有shard的信息
+
+###### Method & Url
+
+```
+GET http://localhost:8620/v1/shards
+```
+
+###### Response Status
+
+```json
+200
+```
+
+###### Response Body
+
+```json
+{
+  "message": "OK",
+  "data": {
+    "shards": [
+      {
+        "storeId": 110645464809417136,
+        "partitionId": 0,
+        "role": "Leader",
+        "state": "SState_Normal",
+        "graphName": "DEFAULT/hugegraph/g",
+        "progress": 0
+      },
+      {
+        "storeId": 110645464809417136,
+        "partitionId": 1,
+        "role": "Leader",
+        "state": "SState_Normal",
+        "graphName": "DEFAULT/hugegraph/g",
+        "progress": 0
+      },
+      {
+        "storeId": 110645464809417136,
+        "partitionId": 2,
+        "role": "Leader",
+        "state": "SState_Normal",
+        "graphName": "DEFAULT/hugegraph/g",
+        "progress": 0
+      },
+      {
+        "storeId": 110645464809417136,
+        "partitionId": 3,
+        "role": "Leader",
+        "state": "SState_Normal",
+        "graphName": "DEFAULT/hugegraph/g",
+        "progress": 0
+      },
+      {
+        "storeId": 110645464809417136,
+        "partitionId": 0,
+        "role": "Leader",
+        "state": "SState_Normal",
+        "graphName": "DEFAULT/hugegraph/m",
+        "progress": 0
+      },
+      {
+        "storeId": 110645464809417136,
+        "partitionId": 0,
+        "role": "Leader",
+        "state": "SState_Normal",
+        "graphName": "DEFAULT/hugegraph/s",
+        "progress": 0
+      }
+    ]
+  },
+  "status": 0
+}
+```
+
+###服务注册
+
+#### 注册服务
+
+###### Method & Url
+
+```
+POST http://127.0.0.1:8620/v1/registry
+```
+
+###### Request Body
+
+```json
+200
+```
+
+###### Response Status
+
+```json
+{
+  "appName":"aaaa",
+  "version":"version1",
+  "address":"address1",
+  "interval":"9223372036854775807",
+  "labels": {
+    "aaa": "aaaavalue"
+  }
+}
+```
+
+appName：所属服务名  
+version：所属服务版本号  
+address：服务实例地址+端口  
+interval：实例心跳间隔，字符串，最大9223372036854775807  
+labels: 自定义标签，若服务名为'hg'即hugeserver时，需要提供key为cores的项，进行cpu核数的验证
+
+###### Response Body
+
+```json
+{
+  "errorType": "OK",
+  "message": "",
+  "data": null
+}
+```
+
+errorType：状态码  
+message：状态码为错误时的具体出错信息  
+data：无返回数据
+
+#### 服务实例获取
+
+###### Method & Url
+
+```
+POST http://127.0.0.1:8620/v1/registryInfo
+```
+
+###### Request Body
+
+```json
+200
+```
+
+###### Response Status
+
+```json
+{
+  "appName":"aaaa",
+  "version":"version1",
+  "labels": {
+    "aaa": "aaaavalue"
+  }
+}
+```
+
+以下三项可全部为空，则获取所有服务节点的信息:  
+-- appName：过滤所属服务名的条件   
+-- version：过滤所属服务版本号的条件，此项有值，则appName不能为空  
+-- labels: 过滤自定义标签的条件
+
+###### Response Body
+
+```json
+{
+  "errorType": "OK",
+  "message": null,
+  "data": [
+    {
+      "id": null,
+      "appName": "aaaa",
+      "version": "version1",
+      "address": "address1",
+      "interval": "9223372036854775807",
+      "labels": {
+        "aaa": "aaaavalue"
+      }
+    }
+  ]
+}
+```
+
+errorType：状态码  
+message：状态码为错误时的具体出错信息  
+data：获取的服务节点信息  

hugegraph-pd/build-pre.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to You under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+mkdir output
+touch output/1
+export JAVA_HOME=$ORACLEJDK_1_8_0_HOME
+
+readonly VER=3.6.3
+readonly REPO_URL=http://127.0.0.1:8081/artifactory/star-local
+
+$MAVEN_3_5_3_BIN/mvn -DremoveSnapshot=true  -DprocessAllModules=true -DgenerateBackupPoms=true versions:set
+$MAVEN_3_5_3_BIN/mvn --settings ./settings.xml -Dmaven.test.skip=true -DaltDeploymentRepository=star-local::default::${REPO_URL} clean deploy
+$MAVEN_3_5_3_BIN/mvn versions:revert
+
+
+#------------------repo-----------------------
+readonly FILE_NAME=hugegraph-pd-3.6.3.tar.gz
+readonly REPO_URL_FILE=http://127.0.0.1:8081/artifactory/star-file
+
+localFilePath=dist/${FILE_NAME}
+targetFolder="${REPO_URL_FILE}/dist/$(date '+%Y-%m-%d')/"
+artifactoryUser="admin"
+artifactoryPassword="JFrog12345"
+
+md5Value="$(md5sum "$localFilePath")"
+md5Value="${md5Value:0:32}"
+sha1Value="$(sha1sum "$localFilePath")"
+sha1Value="${sha1Value:0:40}"
+sha256Value="$(sha256sum "$localFilePath")"
+sha256Value="${sha256Value:0:65}"
+
+#curl -X PUT -u admin:JFrog12345 -T ${localFilePath} "${REPO_URL_FILE}/dist/${data_folder}/"
+echo "INFO: Uploading $localFilePath to $targetFolder"
+curl -i -X PUT -u "$artifactoryUser:$artifactoryPassword" \
+ -H "X-Checksum-Md5: $md5Value" \
+ -H "X-Checksum-Sha1: $sha1Value" \
+ -H "X-Checksum-Sha256: $sha256Value" \
+ -T "$localFilePath" \
+ "$targetFolder"

hugegraph-pd/build.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to You under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+export PATH=$MAVEN_3_5_3_BIN:$ORACLEJDK_11_0_7_BIN:$PATH
+export JAVA_HOME=$ORACLEJDK_11_0_7_HOME
+export MAVEN_HOME=$MAVEN_3_5_3_HOME
+
+# TODO: remove later
+readonly REPO_URL=http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots
+
+if [ ! -n "$1" ] ;then
+    GOAL=package
+else
+    GOAL=$1
+fi
+
+$MAVEN_3_5_3_BIN/mvn -Dmaven.test.skip=true -DaltDeploymentRepository=Baidu_Local_Snapshots::default::${REPO_URL} clean ${GOAL}
+echo "mv dist...."
+mv dist output
+ls output
+echo "mv dist done"
+echo "show output...."
+ls output
+echo "show output done"

hugegraph-pd/ci.yml

@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+Global:
+  tool: build_submitter
+  languages:
+    # Java Version: 1.7, 1.8, 9, 10
+    - language: java
+      version: 11
+  envs:
+    # Maven Version: 3.0.4, 3.1.1, 3.2.5, 3.3.9, 3.5.3
+    - env: maven
+      version: 3.5.3
+Default:
+  profile: [ dev ]
+Profiles:
+  - profile:
+    name: dev
+    tool: build_submitter
+    env: DECK_CENTOS7U5_K3
+    command: sh build.sh
+    release: true
+
+  - profile:
+    name: deploy
+    tool: build_submitter
+    env: DECK_CENTOS7U5_K3
+    command: sh build.sh deploy
+    release: true

hugegraph-pd/conf/verify-license.json

@@ -0,0 +1,6 @@
+{
+  "subject": "hugegraph-license",
+  "public_alias": "publiccert",
+  "store_ticket": "803b6cc3-d144-47e8-948f-ec8b39c8881e",
+  "publickey_path": "/public-certs.store"
+}

hugegraph-pd/deploy-release.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to You under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+readonly VER=3.6.3
+readonly REPO_URL=http://127.0.0.1:8081/artifactory/star-local
+#mvn -DnewVersion=${VER}-SNAPSHOT  -DprocessAllModules=true -DgenerateBackupPoms=false versions:set
+
+./mvnw -DremoveSnapshot=true  -DprocessAllModules=true -DgenerateBackupPoms=true versions:set
+./mvnw --settings ./settings.xml -Dmaven.test.skip=true -DaltDeploymentRepository=star-local::default::${REPO_URL} clean deploy
+./mvnw versions:revert

hugegraph-pd/deploy-snapshot.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to You under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+readonly VER=3.6.3
+readonly REPO_URL=http://127.0.0.1:8081/artifactory/star-local
+#mvn -DnewVersion=${VER}-SNAPSHOT  -DprocessAllModules=true -DgenerateBackupPoms=false versions:set
+
+./mvnw -DremoveSnapshot=true  -DprocessAllModules=true -DgenerateBackupPoms=true versions:set
+./mvnw --settings ./settings.xml -Dmaven.test.skip=true -DaltDeploymentRepository=star-local::default::${REPO_URL} clean deploy
+./mvnw versions:revert

hugegraph-pd/hg-pd-client/pom.xml

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements. See the NOTICE file distributed with this
+  work for additional information regarding copyright ownership. The ASF
+  licenses this file to You under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  License for the specific language governing permissions and limitations
+  under the License.
+  -->
+
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.apache.hugegraph</groupId>
+        <artifactId>hugegraph-pd</artifactId>
+        <version>${revision}</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <artifactId>hg-pd-client</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.20</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j-impl</artifactId>
+            <version>2.17.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hugegraph</groupId>
+            <artifactId>hg-pd-grpc</artifactId>
+            <version>${revision}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hugegraph</groupId>
+            <artifactId>hg-pd-common</artifactId>
+            <version>${revision}</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.13.2</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>2.8.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+            <version>1.28</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+</project>

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClient.java

@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.io.Closeable;
+import java.util.LinkedList;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+import org.apache.hugegraph.pd.common.KVPair;
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.apache.hugegraph.pd.grpc.PDGrpc;
+import org.apache.hugegraph.pd.grpc.PDGrpc.PDBlockingStub;
+import org.apache.hugegraph.pd.grpc.Pdpb;
+import org.apache.hugegraph.pd.grpc.Pdpb.GetMembersRequest;
+import org.apache.hugegraph.pd.grpc.Pdpb.GetMembersResponse;
+
+import io.grpc.Channel;
+import io.grpc.ClientCall;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.MethodDescriptor;
+import io.grpc.StatusRuntimeException;
+import io.grpc.stub.AbstractBlockingStub;
+import io.grpc.stub.AbstractStub;
+import io.grpc.stub.ClientCalls;
+import io.grpc.stub.StreamObserver;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public abstract class AbstractClient implements Closeable {
+
+    private static final ConcurrentHashMap<String, ManagedChannel> chs = new ConcurrentHashMap<>();
+    public static Pdpb.ResponseHeader okHeader = Pdpb.ResponseHeader.newBuilder().setError(
+            Pdpb.Error.newBuilder().setType(Pdpb.ErrorType.OK)).build();
+    protected final Pdpb.RequestHeader header;
+    protected final AbstractClientStubProxy stubProxy;
+    protected final PDConfig config;
+    protected ManagedChannel channel = null;
+    protected volatile ConcurrentMap<String, AbstractBlockingStub> stubs = null;
+
+    protected AbstractClient(PDConfig config) {
+        String[] hosts = config.getServerHost().split(",");
+        this.stubProxy = new AbstractClientStubProxy(hosts);
+        this.header = Pdpb.RequestHeader.getDefaultInstance();
+        this.config = config;
+    }
+
+    public static Pdpb.ResponseHeader newErrorHeader(int errorCode, String errorMsg) {
+        Pdpb.ResponseHeader header = Pdpb.ResponseHeader.newBuilder().setError(
+                Pdpb.Error.newBuilder().setTypeValue(errorCode).setMessage(errorMsg)).build();
+        return header;
+    }
+
+    protected static void handleErrors(Pdpb.ResponseHeader header) throws PDException {
+        if (header.hasError() && header.getError().getType() != Pdpb.ErrorType.OK) {
+            throw new PDException(header.getError().getTypeValue(),
+                                  String.format("PD request error, error code = %d, msg = %s",
+                                                header.getError().getTypeValue(),
+                                                header.getError().getMessage()));
+        }
+    }
+
+    protected AbstractBlockingStub getBlockingStub() throws PDException {
+        if (stubProxy.getBlockingStub() == null) {
+            synchronized (this) {
+                if (stubProxy.getBlockingStub() == null) {
+                    String host = resetStub();
+                    if (host.isEmpty()) {
+                        throw new PDException(Pdpb.ErrorType.PD_UNREACHABLE_VALUE,
+                                              "PD unreachable, pd.peers=" +
+                                              config.getServerHost());
+                    }
+                }
+            }
+        }
+        return (AbstractBlockingStub) stubProxy.getBlockingStub()
+                                               .withDeadlineAfter(config.getGrpcTimeOut(),
+                                                                  TimeUnit.MILLISECONDS);
+    }
+
+    protected AbstractStub getStub() throws PDException {
+        if (stubProxy.getStub() == null) {
+            synchronized (this) {
+                if (stubProxy.getStub() == null) {
+                    String host = resetStub();
+                    if (host.isEmpty()) {
+                        throw new PDException(Pdpb.ErrorType.PD_UNREACHABLE_VALUE,
+                                              "PD unreachable, pd.peers=" +
+                                              config.getServerHost());
+                    }
+                }
+            }
+        }
+        return stubProxy.getStub();
+    }
+
+    protected abstract AbstractStub createStub();
+
+    protected abstract AbstractBlockingStub createBlockingStub();
+
+    private String resetStub() {
+        String leaderHost = "";
+        for (int i = 0; i < stubProxy.getHostCount(); i++) {
+            String host = stubProxy.nextHost();
+            channel = ManagedChannelBuilder.forTarget(host).usePlaintext().build();
+            PDBlockingStub blockingStub = PDGrpc.newBlockingStub(channel)
+                                                .withDeadlineAfter(config.getGrpcTimeOut(),
+                                                                   TimeUnit.MILLISECONDS);
+            try {
+                GetMembersRequest request = Pdpb.GetMembersRequest.newBuilder()
+                                                                  .setHeader(header).build();
+                GetMembersResponse members = blockingStub.getMembers(request);
+                Metapb.Member leader = members.getLeader();
+                leaderHost = leader.getGrpcUrl();
+                close();
+                channel = ManagedChannelBuilder.forTarget(leaderHost).usePlaintext().build();
+                stubProxy.setBlockingStub(createBlockingStub());
+                stubProxy.setStub(createStub());
+                log.info("PDClient connect to host = {} success", leaderHost);
+                break;
+            } catch (Exception e) {
+                log.error("PDClient connect to {} exception {}, {}", host, e.getMessage(),
+                          e.getCause() != null ? e.getCause().getMessage() : "");
+            }
+        }
+        return leaderHost;
+    }
+
+    protected <ReqT, RespT, StubT extends AbstractBlockingStub<StubT>> RespT blockingUnaryCall(
+            MethodDescriptor<ReqT, RespT> method, ReqT req) throws PDException {
+        return blockingUnaryCall(method, req, 5);
+    }
+
+    protected <ReqT, RespT, StubT extends AbstractBlockingStub<StubT>> RespT blockingUnaryCall(
+            MethodDescriptor<ReqT, RespT> method, ReqT req, int retry) throws PDException {
+        AbstractBlockingStub stub = getBlockingStub();
+        try {
+            RespT resp =
+                    ClientCalls.blockingUnaryCall(stub.getChannel(), method, stub.getCallOptions(),
+                                                  req);
+            return resp;
+        } catch (Exception e) {
+            log.error(method.getFullMethodName() + " exception, {}", e.getMessage());
+            if (e instanceof StatusRuntimeException) {
+                if (retry < stubProxy.getHostCount()) {
+                    // 网络不通，关掉之前连接，换host重新连接
+                    synchronized (this) {
+                        stubProxy.setBlockingStub(null);
+                    }
+                    return blockingUnaryCall(method, req, ++retry);
+                }
+            }
+        }
+        return null;
+    }
+
+    // this.stubs = new ConcurrentHashMap<String,AbstractBlockingStub>(hosts.length);
+    private AbstractBlockingStub getConcurrentBlockingStub(String address) {
+        AbstractBlockingStub stub = stubs.get(address);
+        if (stub != null) {
+            return stub;
+        }
+        Channel ch = ManagedChannelBuilder.forTarget(address).usePlaintext().build();
+        PDBlockingStub blockingStub =
+                PDGrpc.newBlockingStub(ch).withDeadlineAfter(config.getGrpcTimeOut(),
+                                                             TimeUnit.MILLISECONDS);
+        stubs.put(address, blockingStub);
+        return blockingStub;
+
+    }
+
+    protected <ReqT, RespT> KVPair<Boolean, RespT> concurrentBlockingUnaryCall(
+            MethodDescriptor<ReqT, RespT> method, ReqT req, Predicate<RespT> predicate) {
+        LinkedList<String> hostList = this.stubProxy.getHostList();
+        if (this.stubs == null) {
+            synchronized (this) {
+                if (this.stubs == null) {
+                    this.stubs = new ConcurrentHashMap<>(hostList.size());
+                }
+            }
+        }
+        Stream<RespT> respTStream = hostList.parallelStream().map((address) -> {
+            AbstractBlockingStub stub = getConcurrentBlockingStub(address);
+            RespT resp = ClientCalls.blockingUnaryCall(stub.getChannel(),
+                                                       method, stub.getCallOptions(), req);
+            return resp;
+        });
+        KVPair<Boolean, RespT> pair;
+        AtomicReference<RespT> response = new AtomicReference<>();
+        boolean result = respTStream.anyMatch((r) -> {
+            response.set(r);
+            return predicate.test(r);
+        });
+        if (result) {
+            pair = new KVPair<>(true, null);
+        } else {
+            pair = new KVPair<>(false, response.get());
+        }
+        return pair;
+    }
+
+    protected <ReqT, RespT> void streamingCall(MethodDescriptor<ReqT, RespT> method, ReqT request,
+                                               StreamObserver<RespT> responseObserver,
+                                               int retry) throws PDException {
+        AbstractStub stub = getStub();
+        try {
+            ClientCall<ReqT, RespT> call = stub.getChannel().newCall(method, stub.getCallOptions());
+            ClientCalls.asyncServerStreamingCall(call, request, responseObserver);
+        } catch (Exception e) {
+            if (e instanceof StatusRuntimeException) {
+                if (retry < stubProxy.getHostCount()) {
+                    synchronized (this) {
+                        stubProxy.setStub(null);
+                    }
+                    streamingCall(method, request, responseObserver, ++retry);
+                    return;
+                }
+            }
+            log.error("rpc call with exception, {}", e.getMessage());
+        }
+    }
+
+    @Override
+    public void close() {
+        closeChannel(channel);
+        if (stubs != null) {
+            for (AbstractBlockingStub stub : stubs.values()) {
+                closeChannel((ManagedChannel) stub.getChannel());
+            }
+        }
+
+    }
+
+    private void closeChannel(ManagedChannel channel) {
+        try {
+            while (channel != null &&
+                   !channel.shutdownNow().awaitTermination(100, TimeUnit.MILLISECONDS)) {
+                continue;
+            }
+        } catch (Exception e) {
+            log.info("Close channel with error : ", e);
+        }
+    }
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClientStubProxy.java

@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.util.LinkedList;
+
+import io.grpc.stub.AbstractBlockingStub;
+import io.grpc.stub.AbstractStub;
+
+public class AbstractClientStubProxy {
+
+    private final LinkedList<String> hostList = new LinkedList<>();
+    private AbstractBlockingStub blockingStub;
+    private AbstractStub stub;
+
+    public AbstractClientStubProxy(String[] hosts) {
+        for (String host : hosts) {
+            if (!host.isEmpty()) {
+                hostList.offer(host);
+            }
+        }
+    }
+
+    public LinkedList<String> getHostList() {
+        return hostList;
+    }
+
+    public String nextHost() {
+        String host = hostList.poll();
+        hostList.offer(host);   //移到尾部
+        return host;
+    }
+
+    public AbstractBlockingStub getBlockingStub() {
+        return this.blockingStub;
+    }
+
+    public void setBlockingStub(AbstractBlockingStub stub) {
+        this.blockingStub = stub;
+    }
+
+    public String getHost() {
+        return hostList.peek();
+    }
+
+    public int getHostCount() {
+        return hostList.size();
+    }
+
+    public AbstractStub getStub() {
+        return stub;
+    }
+
+    public void setStub(AbstractStub stub) {
+        this.stub = stub;
+    }
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/Channels.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+
+public class Channels {
+
+    private static final ConcurrentHashMap<String, ManagedChannel> chs = new ConcurrentHashMap<>();
+
+    public static ManagedChannel getChannel(String target) {
+
+        ManagedChannel channel;
+        if ((channel = chs.get(target)) == null || channel.isShutdown() || channel.isTerminated()) {
+            synchronized (chs) {
+                if ((channel = chs.get(target)) == null || channel.isShutdown() ||
+                    channel.isTerminated()) {
+                    channel = ManagedChannelBuilder.forTarget(target).usePlaintext().build();
+                    chs.put(target, channel);
+                }
+            }
+        }
+
+        return channel;
+    }
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/ClientCache.java

@@ -0,0 +1,338 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hugegraph.pd.common.GraphCache;
+import org.apache.hugegraph.pd.common.KVPair;
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.common.PartitionUtils;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.apache.hugegraph.pd.grpc.Metapb.Partition;
+import org.apache.hugegraph.pd.grpc.Metapb.Shard;
+import org.apache.hugegraph.pd.grpc.Metapb.ShardGroup;
+import org.apache.hugegraph.pd.grpc.Pdpb.CachePartitionResponse;
+import org.apache.hugegraph.pd.grpc.Pdpb.CacheResponse;
+
+import com.google.common.collect.Range;
+import com.google.common.collect.RangeMap;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class ClientCache {
+
+    private final AtomicBoolean initialized = new AtomicBoolean(false);
+    private final org.apache.hugegraph.pd.client.PDClient client;
+    private volatile Map<Integer, KVPair<ShardGroup, Shard>> groups;
+    private volatile Map<Long, Metapb.Store> stores;
+    private volatile Map<String, GraphCache> caches = new ConcurrentHashMap<>();
+
+    public ClientCache(org.apache.hugegraph.pd.client.PDClient pdClient) {
+        groups = new ConcurrentHashMap<>();
+        stores = new ConcurrentHashMap<>();
+        client = pdClient;
+    }
+
+    private GraphCache getGraphCache(String graphName) {
+        GraphCache graph;
+        if ((graph = caches.get(graphName)) == null) {
+            synchronized (caches) {
+                if ((graph = caches.get(graphName)) == null) {
+                    graph = new GraphCache();
+                    caches.put(graphName, graph);
+                }
+            }
+        }
+        return graph;
+    }
+
+    public KVPair<Partition, Shard> getPartitionById(String graphName, int partId) {
+        try {
+            GraphCache graph = initGraph(graphName);
+            Partition partition = graph.getPartition(partId);
+            Shard shard = groups.get(partId).getValue();
+            if (partition == null || shard == null) {
+                return null;
+            }
+            return new KVPair<>(partition, shard);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private KVPair<Partition, Shard> getPair(int partId, GraphCache graph) {
+        Partition p = graph.getPartition(partId);
+        KVPair<ShardGroup, Shard> pair = groups.get(partId);
+        if (p != null && pair != null) {
+            Shard s = pair.getValue();
+            if (s == null) {
+                pair.setValue(getLeader(partId));
+                return new KVPair<>(p, pair.getValue());
+            } else {
+                return new KVPair<>(p, s);
+            }
+        }
+        return null;
+    }
+
+    /**
+     * 根据key的hashcode返回分区信息
+     *
+     * @param graphName
+     * @param code
+     * @return
+     */
+    public KVPair<Partition, Shard> getPartitionByCode(String graphName, long code) {
+        try {
+            GraphCache graph = initGraph(graphName);
+            RangeMap<Long, Integer> range = graph.getRange();
+            Integer pId = range.get(code);
+            if (pId != null) {
+                return getPair(pId, graph);
+            }
+            return null;
+        } catch (PDException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private GraphCache initGraph(String graphName) throws PDException {
+        initCache();
+        GraphCache graph = getGraphCache(graphName);
+        if (!graph.getInitialized().get()) {
+            synchronized (graph) {
+                if (!graph.getInitialized().get()) {
+                    CachePartitionResponse pc = client.getPartitionCache(graphName);
+                    RangeMap<Long, Integer> range = graph.getRange();
+                    List<Partition> ps = pc.getPartitionsList();
+                    HashMap<Integer, Partition> gps = new HashMap<>(ps.size(), 1);
+                    for (Partition p : ps) {
+                        gps.put(p.getId(), p);
+                        range.put(Range.closedOpen(p.getStartKey(), p.getEndKey()), p.getId());
+                    }
+                    graph.setPartitions(gps);
+                    graph.getInitialized().set(true);
+                }
+            }
+        }
+        return graph;
+    }
+
+    private void initCache() throws PDException {
+        if (!initialized.get()) {
+            synchronized (this) {
+                if (!initialized.get()) {
+                    CacheResponse cache = client.getClientCache();
+                    List<ShardGroup> shardGroups = cache.getShardsList();
+                    for (ShardGroup s : shardGroups) {
+                        this.groups.put(s.getId(), new KVPair<>(s, getLeader(s.getId())));
+                    }
+                    List<Metapb.Store> stores = cache.getStoresList();
+                    for (Metapb.Store store : stores) {
+                        this.stores.put(store.getId(), store);
+                    }
+                    List<Metapb.Graph> graphs = cache.getGraphsList();
+                    for (Metapb.Graph g : graphs) {
+                        GraphCache c = new GraphCache(g);
+                        caches.put(g.getGraphName(), c);
+                    }
+                    initialized.set(true);
+                }
+            }
+        }
+    }
+
+    /**
+     * 返回key所在的分区信息
+     *
+     * @param key
+     * @return
+     */
+    public KVPair<Partition, Shard> getPartitionByKey(String graphName, byte[] key) {
+        int code = PartitionUtils.calcHashcode(key);
+        return getPartitionByCode(graphName, code);
+    }
+
+    public boolean update(String graphName, int partId, Partition partition) {
+        GraphCache graph = getGraphCache(graphName);
+        try {
+            Partition p = graph.getPartition(partId);
+            if (p != null && p.equals(partition)) {
+                return false;
+            }
+            RangeMap<Long, Integer> range = graph.getRange();
+            graph.addPartition(partId, partition);
+            if (p != null) {
+                // old [1-3) 被 [2-3)覆盖了。当 [1-3) 变成[1-2) 不应该删除原先的[1-3)
+                // 当确认老的 start, end 都是自己的时候，才可以删除老的. (即还没覆盖）
+                if (Objects.equals(partition.getId(), range.get(partition.getStartKey())) &&
+                    Objects.equals(partition.getId(), range.get(partition.getEndKey() - 1))) {
+                    range.remove(range.getEntry(partition.getStartKey()).getKey());
+                }
+            }
+            range.put(Range.closedOpen(partition.getStartKey(), partition.getEndKey()), partId);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        return true;
+    }
+
+    public void removePartition(String graphName, int partId) {
+        GraphCache graph = getGraphCache(graphName);
+        Partition p = graph.removePartition(partId);
+        if (p != null) {
+            RangeMap<Long, Integer> range = graph.getRange();
+            if (Objects.equals(p.getId(), range.get(p.getStartKey())) &&
+                Objects.equals(p.getId(), range.get(p.getEndKey() - 1))) {
+                range.remove(range.getEntry(p.getStartKey()).getKey());
+            }
+        }
+    }
+
+    /**
+     * remove all partitions
+     */
+    public void removePartitions() {
+        for (Entry<String, GraphCache> entry : caches.entrySet()) {
+            removePartitions(entry.getValue());
+        }
+    }
+
+    private void removePartitions(GraphCache graph) {
+        graph.getState().clear();
+        graph.getRange().clear();
+    }
+
+    /**
+     * remove partition cache of graphName
+     *
+     * @param graphName
+     */
+    public void removeAll(String graphName) {
+        GraphCache graph = caches.get(graphName);
+        if (graph != null) {
+            removePartitions(graph);
+        }
+    }
+
+    public boolean updateShardGroup(ShardGroup shardGroup) {
+        KVPair<ShardGroup, Shard> old = groups.get(shardGroup.getId());
+        Shard leader = getLeader(shardGroup);
+        if (old != null) {
+            old.setKey(shardGroup);
+            old.setValue(leader);
+            return false;
+        }
+        groups.put(shardGroup.getId(), new KVPair<>(shardGroup, leader));
+        return true;
+    }
+
+    public void deleteShardGroup(int shardGroupId) {
+        groups.remove(shardGroupId);
+    }
+
+    public ShardGroup getShardGroup(int groupId) {
+        KVPair<ShardGroup, Shard> pair = groups.get(groupId);
+        if (pair != null) {
+            return pair.getKey();
+        }
+        return null;
+    }
+
+    public boolean addStore(Long storeId, Metapb.Store store) {
+        Metapb.Store oldStore = stores.get(storeId);
+        if (oldStore != null && oldStore.equals(store)) {
+            return false;
+        }
+        stores.put(storeId, store);
+        return true;
+    }
+
+    public Metapb.Store getStoreById(Long storeId) {
+        return stores.get(storeId);
+    }
+
+    public void removeStore(Long storeId) {
+        stores.remove(storeId);
+    }
+
+    public void reset() {
+        groups = new ConcurrentHashMap<>();
+        stores = new ConcurrentHashMap<>();
+        caches = new ConcurrentHashMap<>();
+    }
+
+    public Shard getLeader(int partitionId) {
+        KVPair<ShardGroup, Shard> pair = groups.get(partitionId);
+        if (pair != null) {
+            if (pair.getValue() != null) {
+                return pair.getValue();
+            }
+            for (Shard shard : pair.getKey().getShardsList()) {
+                if (shard.getRole() == Metapb.ShardRole.Leader) {
+                    pair.setValue(shard);
+                    return shard;
+                }
+            }
+        }
+
+        return null;
+    }
+
+    public Shard getLeader(ShardGroup shardGroup) {
+        if (shardGroup != null) {
+            for (Shard shard : shardGroup.getShardsList()) {
+                if (shard.getRole() == Metapb.ShardRole.Leader) {
+                    return shard;
+                }
+            }
+        }
+
+        return null;
+    }
+
+    public void updateLeader(int partitionId, Shard leader) {
+        KVPair<ShardGroup, Shard> pair = groups.get(partitionId);
+        if (pair != null && leader != null) {
+            Shard l = getLeader(partitionId);
+            if (l == null || leader.getStoreId() != l.getStoreId()) {
+                ShardGroup shardGroup = pair.getKey();
+                ShardGroup.Builder builder = ShardGroup.newBuilder(shardGroup).clearShards();
+                for (var shard : shardGroup.getShardsList()) {
+                    builder.addShards(
+                            Shard.newBuilder()
+                                 .setStoreId(shard.getStoreId())
+                                 .setRole(shard.getStoreId() == leader.getStoreId() ?
+                                          Metapb.ShardRole.Leader : Metapb.ShardRole.Follower)
+                                 .build()
+                    );
+                }
+                pair.setKey(builder.build());
+                pair.setValue(leader);
+            }
+        }
+    }
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/Discoverable.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import org.apache.hugegraph.pd.grpc.discovery.NodeInfos;
+import org.apache.hugegraph.pd.grpc.discovery.Query;
+
+public interface Discoverable {
+
+    NodeInfos getNodeInfos(Query query);
+
+    void scheduleTask();
+
+    void cancelTask();
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClient.java

@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.io.Closeable;
+import java.util.LinkedList;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.grpc.discovery.DiscoveryServiceGrpc;
+import org.apache.hugegraph.pd.grpc.discovery.NodeInfo;
+import org.apache.hugegraph.pd.grpc.discovery.NodeInfos;
+import org.apache.hugegraph.pd.grpc.discovery.Query;
+import org.apache.hugegraph.pd.grpc.discovery.RegisterInfo;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public abstract class DiscoveryClient implements Closeable, Discoverable {
+
+    private final Timer timer = new Timer("serverHeartbeat", true);
+    private final AtomicBoolean requireResetStub = new AtomicBoolean(false);
+    protected int period; //心跳周期
+    LinkedList<String> pdAddresses = new LinkedList<>();
+    ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+    private volatile int currentIndex; // 当前在用pd地址位置
+    private int maxTime = 6;
+    private ManagedChannel channel = null;
+    private DiscoveryServiceGrpc.DiscoveryServiceBlockingStub registerStub;
+    private DiscoveryServiceGrpc.DiscoveryServiceBlockingStub blockingStub;
+
+    public DiscoveryClient(String centerAddress, int delay) {
+        String[] addresses = centerAddress.split(",");
+        for (int i = 0; i < addresses.length; i++) {
+            String singleAddress = addresses[i];
+            if (singleAddress == null || singleAddress.length() <= 0) {
+                continue;
+            }
+            pdAddresses.add(addresses[i]);
+        }
+        this.period = delay;
+        if (maxTime < addresses.length) {
+            maxTime = addresses.length;
+        }
+    }
+
+    private <V, R> R tryWithTimes(Function<V, R> function, V v) {
+        R r;
+        Exception ex = null;
+        for (int i = 0; i < maxTime; i++) {
+            try {
+                r = function.apply(v);
+                return r;
+            } catch (Exception e) {
+                requireResetStub.set(true);
+                resetStub();
+                ex = e;
+            }
+        }
+        if (ex != null) {
+            log.error("Try discovery method with error: {}", ex.getMessage());
+        }
+        return null;
+    }
+
+    /***
+     * 按照pd列表重置stub
+     */
+    private void resetStub() {
+        String errLog = null;
+        for (int i = currentIndex + 1; i <= pdAddresses.size() + currentIndex; i++) {
+            currentIndex = i % pdAddresses.size();
+            String singleAddress = pdAddresses.get(currentIndex);
+            try {
+                if (requireResetStub.get()) {
+                    resetChannel(singleAddress);
+                }
+                errLog = null;
+                break;
+            } catch (Exception e) {
+                requireResetStub.set(true);
+                if (errLog == null) {
+                    errLog = e.getMessage();
+                }
+                continue;
+            }
+        }
+        if (errLog != null) {
+            log.error(errLog);
+        }
+    }
+
+    /***
+     * 按照某个pd的地址重置channel和stub
+     * @param singleAddress
+     * @throws PDException
+     */
+    private void resetChannel(String singleAddress) throws PDException {
+
+        readWriteLock.writeLock().lock();
+        try {
+            if (requireResetStub.get()) {
+                while (channel != null && !channel.shutdownNow().awaitTermination(
+                    100, TimeUnit.MILLISECONDS)) {
+                    continue;
+                }
+                channel = ManagedChannelBuilder.forTarget(
+                    singleAddress).usePlaintext().build();
+                this.registerStub = DiscoveryServiceGrpc.newBlockingStub(
+                    channel);
+                this.blockingStub = DiscoveryServiceGrpc.newBlockingStub(
+                    channel);
+                requireResetStub.set(false);
+            }
+        } catch (Exception e) {
+            throw new PDException(-1, String.format(
+                "Reset channel with error : %s.", e.getMessage()));
+        } finally {
+            readWriteLock.writeLock().unlock();
+        }
+    }
+
+    /***
+     * 获取注册节点信息
+     * @param query
+     * @return
+     */
+    @Override
+    public NodeInfos getNodeInfos(Query query) {
+        return tryWithTimes((q) -> {
+            this.readWriteLock.readLock().lock();
+            NodeInfos nodes;
+            try {
+                nodes = this.blockingStub.getNodes(q);
+            } catch (Exception e) {
+                throw e;
+            } finally {
+                this.readWriteLock.readLock().unlock();
+            }
+            return nodes;
+        }, query);
+    }
+
+    /***
+     * 启动心跳任务
+     */
+    @Override
+    public void scheduleTask() {
+        timer.schedule(new TimerTask() {
+            @Override
+            public void run() {
+                NodeInfo nodeInfo = getRegisterNode();
+                tryWithTimes((t) -> {
+                    RegisterInfo register;
+                    readWriteLock.readLock().lock();
+                    try {
+                        register = registerStub.register(t);
+                        log.debug("Discovery Client work done.");
+                        Consumer<RegisterInfo> consumer = getRegisterConsumer();
+                        if (consumer != null) {
+                            consumer.accept(register);
+                        }
+                    } catch (Exception e) {
+                        throw e;
+                    } finally {
+                        readWriteLock.readLock().unlock();
+                    }
+                    return register;
+                }, nodeInfo);
+            }
+        }, 0, period);
+    }
+
+    abstract NodeInfo getRegisterNode();
+
+    abstract Consumer<RegisterInfo> getRegisterConsumer();
+
+    @Override
+    public void cancelTask() {
+        this.timer.cancel();
+    }
+
+    @Override
+    public void close() {
+        this.timer.cancel();
+        readWriteLock.writeLock().lock();
+        try {
+            while (channel != null && !channel.shutdownNow().awaitTermination(
+                100, TimeUnit.MILLISECONDS)) {
+                continue;
+            }
+        } catch (Exception e) {
+            log.info("Close channel with error : {}.", e);
+        } finally {
+            readWriteLock.writeLock().unlock();
+        }
+    }
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImpl.java

@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.util.Map;
+import java.util.function.Consumer;
+
+import org.apache.hugegraph.pd.grpc.discovery.NodeInfo;
+import org.apache.hugegraph.pd.grpc.discovery.RegisterType;
+
+public class DiscoveryClientImpl extends DiscoveryClient {
+
+    private final String id;
+    private final RegisterType type; // 心跳类型，备用
+    private final String version;
+    private final String appName;
+    private final int times; // 心跳过期次数，备用
+    private final String address;
+    private final Map labels;
+    private final Consumer registerConsumer;
+
+
+    private DiscoveryClientImpl(Builder builder) {
+        super(builder.centerAddress, builder.delay);
+        period = builder.delay;
+        id = builder.id;
+        type = builder.type;
+        version = builder.version;
+        appName = builder.appName;
+        times = builder.times;
+        address = builder.address;
+        labels = builder.labels;
+        registerConsumer = builder.registerConsumer;
+    }
+
+    public static Builder newBuilder() {
+        return new Builder();
+    }
+
+
+    @Override
+    NodeInfo getRegisterNode() {
+        return NodeInfo.newBuilder().setAddress(this.address)
+                       .setVersion(this.version)
+                       .setAppName(this.appName).setInterval(this.period)
+                       .setId(this.id).putAllLabels(labels).build();
+    }
+
+    @Override
+    Consumer getRegisterConsumer() {
+        return registerConsumer;
+    }
+
+
+    public static final class Builder {
+
+        private int delay;
+        private String centerAddress;
+        private String id;
+        private RegisterType type;
+        private String address;
+        private Map labels;
+        private String version;
+        private String appName;
+        private int times;
+        private Consumer registerConsumer;
+
+        private Builder() {
+        }
+
+        public Builder setDelay(int val) {
+            delay = val;
+            return this;
+        }
+
+        public Builder setCenterAddress(String val) {
+            centerAddress = val;
+            return this;
+        }
+
+        public Builder setId(String val) {
+            id = val;
+            return this;
+        }
+
+        public Builder setType(RegisterType val) {
+            type = val;
+            return this;
+        }
+
+        public Builder setAddress(String val) {
+            address = val;
+            return this;
+        }
+
+        public Builder setLabels(Map val) {
+            labels = val;
+            return this;
+        }
+
+        public Builder setVersion(String val) {
+            version = val;
+            return this;
+        }
+
+        public Builder setAppName(String val) {
+            appName = val;
+            return this;
+        }
+
+        public Builder setTimes(int val) {
+            times = val;
+            return this;
+        }
+
+        public Builder setRegisterConsumer(Consumer registerConsumer) {
+            this.registerConsumer = registerConsumer;
+            return this;
+        }
+
+        public DiscoveryClientImpl build() {
+            return new DiscoveryClientImpl(this);
+        }
+    }
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/KvClient.java

@@ -0,0 +1,352 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.io.Closeable;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+import org.apache.hugegraph.pd.client.AbstractClient;
+import org.apache.hugegraph.pd.client.PDConfig;
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.grpc.kv.K;
+import org.apache.hugegraph.pd.grpc.kv.KResponse;
+import org.apache.hugegraph.pd.grpc.kv.Kv;
+import org.apache.hugegraph.pd.grpc.kv.KvResponse;
+import org.apache.hugegraph.pd.grpc.kv.KvServiceGrpc;
+import org.apache.hugegraph.pd.grpc.kv.LockRequest;
+import org.apache.hugegraph.pd.grpc.kv.LockResponse;
+import org.apache.hugegraph.pd.grpc.kv.ScanPrefixResponse;
+import org.apache.hugegraph.pd.grpc.kv.TTLRequest;
+import org.apache.hugegraph.pd.grpc.kv.TTLResponse;
+import org.apache.hugegraph.pd.grpc.kv.WatchEvent;
+import org.apache.hugegraph.pd.grpc.kv.WatchKv;
+import org.apache.hugegraph.pd.grpc.kv.WatchRequest;
+import org.apache.hugegraph.pd.grpc.kv.WatchResponse;
+import org.apache.hugegraph.pd.grpc.kv.WatchType;
+
+import io.grpc.stub.AbstractBlockingStub;
+import io.grpc.stub.AbstractStub;
+import io.grpc.stub.StreamObserver;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class KvClient<T extends WatchResponse> extends AbstractClient implements Closeable {
+
+    private final AtomicLong clientId = new AtomicLong(0);
+    private final Semaphore semaphore = new Semaphore(1);
+    private final ConcurrentHashMap<Long, StreamObserver> observers = new ConcurrentHashMap<>();
+
+    public KvClient(PDConfig pdConfig) {
+        super(pdConfig);
+    }
+
+    @Override
+    protected AbstractStub createStub() {
+        return KvServiceGrpc.newStub(channel);
+    }
+
+    @Override
+    protected AbstractBlockingStub createBlockingStub() {
+        return KvServiceGrpc.newBlockingStub(channel);
+    }
+
+    public KvResponse put(String key, String value) throws PDException {
+        Kv kv = Kv.newBuilder().setKey(key).setValue(value).build();
+        KvResponse response = blockingUnaryCall(KvServiceGrpc.getPutMethod(), kv);
+        handleErrors(response.getHeader());
+        return response;
+    }
+
+
+    public KResponse get(String key) throws PDException {
+        K k = K.newBuilder().setKey(key).build();
+        KResponse response = blockingUnaryCall(KvServiceGrpc.getGetMethod(), k);
+        handleErrors(response.getHeader());
+        return response;
+    }
+
+
+    public KvResponse delete(String key) throws PDException {
+        K k = K.newBuilder().setKey(key).build();
+        KvResponse response = blockingUnaryCall(KvServiceGrpc.getDeleteMethod(), k);
+        handleErrors(response.getHeader());
+        return response;
+    }
+
+
+    public KvResponse deletePrefix(String prefix) throws PDException {
+        K k = K.newBuilder().setKey(prefix).build();
+        KvResponse response = blockingUnaryCall(KvServiceGrpc.getDeletePrefixMethod(), k);
+        handleErrors(response.getHeader());
+        return response;
+    }
+
+
+    public ScanPrefixResponse scanPrefix(String prefix) throws PDException {
+        K k = K.newBuilder().setKey(prefix).build();
+        ScanPrefixResponse response = blockingUnaryCall(KvServiceGrpc.getScanPrefixMethod(), k);
+        handleErrors(response.getHeader());
+        return response;
+    }
+
+    public TTLResponse keepTTLAlive(String key) throws PDException {
+        TTLRequest request = TTLRequest.newBuilder().setKey(key).build();
+        TTLResponse response = blockingUnaryCall(KvServiceGrpc.getKeepTTLAliveMethod(), request);
+        handleErrors(response.getHeader());
+        return response;
+    }
+
+    public TTLResponse putTTL(String key, String value, long ttl) throws PDException {
+        TTLRequest request =
+                TTLRequest.newBuilder().setKey(key).setValue(value).setTtl(ttl).build();
+        TTLResponse response = blockingUnaryCall(KvServiceGrpc.getPutTTLMethod(), request);
+        handleErrors(response.getHeader());
+        return response;
+    }
+
+    private void onEvent(WatchResponse value, Consumer<T> consumer) {
+        log.info("receive message for {},event Count:{}", value, value.getEventsCount());
+        clientId.compareAndSet(0L, value.getClientId());
+        if (value.getEventsCount() != 0) {
+            consumer.accept((T) value);
+        }
+    }
+
+    private StreamObserver<WatchResponse> getObserver(String key, Consumer<T> consumer,
+                                                      BiConsumer<String, Consumer> listenWrapper,
+                                                      long client) {
+        StreamObserver<WatchResponse> observer;
+        if ((observer = observers.get(client)) == null) {
+            synchronized (this) {
+                if ((observer = observers.get(client)) == null) {
+                    observer = getObserver(key, consumer, listenWrapper);
+                    observers.put(client, observer);
+                }
+            }
+        }
+        return observer;
+    }
+
+    private StreamObserver<WatchResponse> getObserver(String key, Consumer<T> consumer,
+                                                      BiConsumer<String, Consumer> listenWrapper) {
+        return new StreamObserver<WatchResponse>() {
+            @Override
+            public void onNext(WatchResponse value) {
+                switch (value.getState()) {
+                    case Starting:
+                        boolean b = clientId.compareAndSet(0, value.getClientId());
+                        if (b) {
+                            observers.put(value.getClientId(), this);
+                            log.info("set watch client id to :{}", value.getClientId());
+                        }
+                        semaphore.release();
+                        break;
+                    case Started:
+                        onEvent(value, consumer);
+                        break;
+                    case Leader_Changed:
+                        listenWrapper.accept(key, consumer);
+                        break;
+                    case Alive:
+                        // only for check client is alive, do nothing
+                        break;
+                    default:
+                        break;
+                }
+            }
+
+            @Override
+            public void onError(Throwable t) {
+                listenWrapper.accept(key, consumer);
+            }
+
+
+            @Override
+            public void onCompleted() {
+
+            }
+        };
+    }
+
+    public void listen(String key, Consumer<T> consumer) throws PDException {
+        long value = clientId.get();
+        StreamObserver<WatchResponse> observer = getObserver(key, consumer, listenWrapper, value);
+        acquire();
+        WatchRequest k = WatchRequest.newBuilder().setClientId(value).setKey(key).build();
+        streamingCall(KvServiceGrpc.getWatchMethod(), k, observer, 1);
+    }
+
+    public void listenPrefix(String prefix, Consumer<T> consumer) throws PDException {
+        long value = clientId.get();
+        StreamObserver<WatchResponse> observer =
+                getObserver(prefix, consumer, prefixListenWrapper, value);
+        acquire();
+        WatchRequest k =
+                WatchRequest.newBuilder().setClientId(clientId.get()).setKey(prefix).build();
+        streamingCall(KvServiceGrpc.getWatchPrefixMethod(), k, observer, 1);
+    }
+
+    private void acquire() {
+        if (clientId.get() == 0L) {
+            try {
+                semaphore.acquire();
+                if (clientId.get() != 0L) {
+                    semaphore.release();
+                }
+            } catch (InterruptedException e) {
+                log.error("get semaphore with error:", e);
+            }
+        }
+    }
+
+    public List<String> getWatchList(T response) {
+        List<String> values = new LinkedList<>();
+        List<WatchEvent> eventsList = response.getEventsList();
+        for (WatchEvent event : eventsList) {
+            if (event.getType() != WatchType.Put) {
+                return null;
+            }
+            String value = event.getCurrent().getValue();
+            values.add(value);
+        }
+        return values;
+    }
+
+    public Map<String, String> getWatchMap(T response) {
+        Map<String, String> values = new HashMap<>();
+        List<WatchEvent> eventsList = response.getEventsList();
+        for (WatchEvent event : eventsList) {
+            if (event.getType() != WatchType.Put) {
+                return null;
+            }
+            WatchKv current = event.getCurrent();
+            String key = current.getKey();
+            String value = current.getValue();
+            values.put(key, value);
+        }
+        return values;
+    }
+
+    public LockResponse lock(String key, long ttl) throws PDException {
+        acquire();
+        LockResponse response;
+        try {
+            LockRequest k =
+                    LockRequest.newBuilder().setKey(key).setClientId(clientId.get()).setTtl(ttl)
+                               .build();
+            response = blockingUnaryCall(KvServiceGrpc.getLockMethod(), k);
+            handleErrors(response.getHeader());
+            if (clientId.compareAndSet(0L, response.getClientId())) {
+                semaphore.release();
+            }
+        } catch (Exception e) {
+            if (clientId.get() == 0L) {
+                semaphore.release();
+            }
+            throw e;
+        }
+        return response;
+    }    BiConsumer<String, Consumer> listenWrapper = (key, consumer) -> {
+        try {
+            listen(key, consumer);
+        } catch (PDException e) {
+            try {
+                log.warn("start listen with warning:", e);
+                Thread.sleep(1000);
+            } catch (InterruptedException ex) {
+            }
+        }
+    };
+
+    public LockResponse lockWithoutReentrant(String key, long ttl) throws PDException {
+        acquire();
+        LockResponse response;
+        try {
+            LockRequest k =
+                    LockRequest.newBuilder().setKey(key).setClientId(clientId.get()).setTtl(ttl)
+                               .build();
+            response = blockingUnaryCall(KvServiceGrpc.getLockWithoutReentrantMethod(), k);
+            handleErrors(response.getHeader());
+            if (clientId.compareAndSet(0L, response.getClientId())) {
+                semaphore.release();
+            }
+        } catch (Exception e) {
+            if (clientId.get() == 0L) {
+                semaphore.release();
+            }
+            throw e;
+        }
+        return response;
+    }
+
+    public LockResponse isLocked(String key) throws PDException {
+        LockRequest k = LockRequest.newBuilder().setKey(key).setClientId(clientId.get()).build();
+        LockResponse response = blockingUnaryCall(KvServiceGrpc.getIsLockedMethod(), k);
+        handleErrors(response.getHeader());
+        return response;
+    }
+
+    public LockResponse unlock(String key) throws PDException {
+        assert clientId.get() != 0;
+        LockRequest k = LockRequest.newBuilder().setKey(key).setClientId(clientId.get()).build();
+        LockResponse response = blockingUnaryCall(KvServiceGrpc.getUnlockMethod(), k);
+        handleErrors(response.getHeader());
+        clientId.compareAndSet(0L, response.getClientId());
+        assert clientId.get() == response.getClientId();
+        return response;
+    }
+
+    public LockResponse keepAlive(String key) throws PDException {
+        assert clientId.get() != 0;
+        LockRequest k = LockRequest.newBuilder().setKey(key).setClientId(clientId.get()).build();
+        LockResponse response = blockingUnaryCall(KvServiceGrpc.getKeepAliveMethod(), k);
+        handleErrors(response.getHeader());
+        clientId.compareAndSet(0L, response.getClientId());
+        assert clientId.get() == response.getClientId();
+        return response;
+    }
+
+    @Override
+    public void close() {
+        super.close();
+    }
+
+
+
+    BiConsumer<String, Consumer> prefixListenWrapper = (key, consumer) -> {
+        try {
+            listenPrefix(key, consumer);
+        } catch (PDException e) {
+            try {
+                log.warn("start listenPrefix with warning:", e);
+                Thread.sleep(1000);
+            } catch (InterruptedException ex) {
+            }
+        }
+    };
+
+
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/LicenseClient.java

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import org.apache.hugegraph.pd.common.KVPair;
+import org.apache.hugegraph.pd.grpc.PDGrpc;
+import org.apache.hugegraph.pd.grpc.Pdpb;
+
+import com.google.protobuf.ByteString;
+
+import io.grpc.stub.AbstractBlockingStub;
+import io.grpc.stub.AbstractStub;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class LicenseClient extends AbstractClient {
+
+    public LicenseClient(PDConfig config) {
+        super(config);
+    }
+
+    @Override
+    protected AbstractStub createStub() {
+        return PDGrpc.newStub(channel);
+    }
+
+    @Override
+    protected AbstractBlockingStub createBlockingStub() {
+        return PDGrpc.newBlockingStub(channel);
+    }
+
+    public Pdpb.PutLicenseResponse putLicense(byte[] content) {
+        Pdpb.PutLicenseRequest request = Pdpb.PutLicenseRequest.newBuilder()
+                                                               .setContent(
+                                                                       ByteString.copyFrom(content))
+                                                               .build();
+        try {
+            KVPair<Boolean, Pdpb.PutLicenseResponse> pair = concurrentBlockingUnaryCall(
+                    PDGrpc.getPutLicenseMethod(), request,
+                    (rs) -> rs.getHeader().getError().getType().equals(Pdpb.ErrorType.OK));
+            if (pair.getKey()) {
+                Pdpb.PutLicenseResponse.Builder builder = Pdpb.PutLicenseResponse.newBuilder();
+                builder.setHeader(okHeader);
+                return builder.build();
+            } else {
+                return pair.getValue();
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.debug("put license with error:{} ", e);
+            Pdpb.ResponseHeader rh =
+                    newErrorHeader(Pdpb.ErrorType.LICENSE_ERROR_VALUE, e.getMessage());
+            return Pdpb.PutLicenseResponse.newBuilder().setHeader(rh).build();
+        }
+    }
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDConfig.java

@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+public final class PDConfig {
+    //TODO multi-server
+    private String serverHost = "localhost:9000";
+    private long grpcTimeOut = 60000;   // grpc调用超时时间 10秒
+
+    // 是否接收PD异步通知
+    private boolean enablePDNotify = false;
+
+    private boolean enableCache = false;
+
+    private PDConfig() {
+    }
+
+    public static PDConfig of() {
+        return new PDConfig();
+    }
+
+    public static PDConfig of(String serverHost) {
+        PDConfig config = new PDConfig();
+        config.serverHost = serverHost;
+        return config;
+    }
+
+    public static PDConfig of(String serverHost, long timeOut) {
+        PDConfig config = new PDConfig();
+        config.serverHost = serverHost;
+        config.grpcTimeOut = timeOut;
+        return config;
+    }
+
+    public String getServerHost() {
+        return serverHost;
+    }
+
+    public long getGrpcTimeOut() {
+        return grpcTimeOut;
+    }
+
+    @Deprecated
+    public PDConfig setEnablePDNotify(boolean enablePDNotify) {
+        this.enablePDNotify = enablePDNotify;
+
+        // TODO 临时代码，hugegraph修改完后删除
+        this.enableCache = enablePDNotify;
+        return this;
+    }
+
+    public boolean isEnableCache() {
+        return enableCache;
+    }
+
+    public PDConfig setEnableCache(boolean enableCache) {
+        this.enableCache = enableCache;
+        return this;
+    }
+
+    @Override
+    public String toString() {
+        return "PDConfig{" +
+               "serverHost='" + serverHost + '\'' +
+               '}';
+    }
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulse.java

@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.io.Closeable;
+import java.util.function.Consumer;
+
+import org.apache.hugegraph.pd.grpc.pulse.PartitionHeartbeatRequest;
+import org.apache.hugegraph.pd.grpc.pulse.PulseResponse;
+import org.apache.hugegraph.pd.pulse.PulseServerNotice;
+
+/**
+ * Bidirectional communication interface of pd-client and pd-server
+ */
+public interface PDPulse {
+
+    /*** inner static methods ***/
+    static <T> Listener<T> listener(Consumer<T> onNext) {
+        return listener(onNext, t -> {
+        }, () -> {
+        });
+    }
+
+    static <T> Listener<T> listener(Consumer<T> onNext, Consumer<Throwable> onError) {
+        return listener(onNext, onError, () -> {
+        });
+    }
+
+    static <T> Listener<T> listener(Consumer<T> onNext, Runnable onCompleted) {
+        return listener(onNext, t -> {
+        }, onCompleted);
+    }
+
+    static <T> Listener<T> listener(Consumer<T> onNext, Consumer<Throwable> onError,
+                                    Runnable onCompleted) {
+        return new Listener<>() {
+            @Override
+            public void onNext(T response) {
+                onNext.accept(response);
+            }
+
+            @Override
+            public void onNotice(PulseServerNotice<T> notice) {
+
+            }
+
+            @Override
+            public void onError(Throwable throwable) {
+                onError.accept(throwable);
+            }
+
+            @Override
+            public void onCompleted() {
+                onCompleted.run();
+            }
+        };
+    }
+
+    /**
+     * @param listener
+     * @return
+     */
+    Notifier<PartitionHeartbeatRequest.Builder> connectPartition(Listener<PulseResponse> listener);
+
+    /**
+     * 切换成新的host。做 channel/host的检查，如果需要关闭，notifier调用close方法。
+     *
+     * @param host     new host
+     * @param notifier notifier
+     * @return true if create new stub, otherwise false
+     */
+    boolean resetStub(String host, Notifier notifier);
+
+    /**
+     * Interface of pulse.
+     */
+    interface Listener<T> {
+        /**
+         * Invoked on new events.
+         *
+         * @param response the response.
+         */
+        @Deprecated
+        default void onNext(T response) {
+        }
+
+        /**
+         * Invoked on new events.
+         *
+         * @param notice a wrapper of response
+         */
+        default void onNotice(PulseServerNotice<T> notice) {
+            notice.ack();
+        }
+
+        /**
+         * Invoked on errors.
+         *
+         * @param throwable the error.
+         */
+        void onError(Throwable throwable);
+
+        /**
+         * Invoked on completion.
+         */
+        void onCompleted();
+
+    }
+
+    /**
+     * Interface of notifier that can send notice to server.
+     *
+     * @param <T>
+     */
+    interface Notifier<T> extends Closeable {
+        /**
+         * closes this watcher and all its resources.
+         */
+        @Override
+        void close();
+
+        /**
+         * Send notice to pd-server.
+         *
+         * @return
+         */
+        void notifyServer(T t);
+
+        /**
+         * Send an error report to pd-server.
+         *
+         * @param error
+         */
+        void crash(String error);
+
+    }
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulseImpl.java

@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.hugegraph.pd.grpc.pulse.HgPdPulseGrpc;
+import org.apache.hugegraph.pd.grpc.pulse.PartitionHeartbeatRequest;
+import org.apache.hugegraph.pd.grpc.pulse.PulseAckRequest;
+import org.apache.hugegraph.pd.grpc.pulse.PulseCreateRequest;
+import org.apache.hugegraph.pd.grpc.pulse.PulseNoticeRequest;
+import org.apache.hugegraph.pd.grpc.pulse.PulseRequest;
+import org.apache.hugegraph.pd.grpc.pulse.PulseResponse;
+import org.apache.hugegraph.pd.grpc.pulse.PulseType;
+import org.apache.hugegraph.pd.pulse.PartitionNotice;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import io.grpc.ManagedChannel;
+import io.grpc.stub.StreamObserver;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public final class PDPulseImpl implements PDPulse {
+
+    private static final ConcurrentHashMap<String, ManagedChannel> chs = new ConcurrentHashMap<>();
+    private final ExecutorService threadPool;
+    private HgPdPulseGrpc.HgPdPulseStub stub;
+    private String pdServerAddress;
+
+    // TODO: support several servers.
+    public PDPulseImpl(String pdServerAddress) {
+        this.pdServerAddress = pdServerAddress;
+        this.stub = HgPdPulseGrpc.newStub(Channels.getChannel(pdServerAddress));
+        var namedThreadFactory =
+                new ThreadFactoryBuilder().setNameFormat("ack-notice-pool-%d").build();
+        threadPool = Executors.newSingleThreadExecutor(namedThreadFactory);
+    }
+
+
+    private String getCurrentHost() {
+        return this.pdServerAddress;
+    }
+
+    private boolean checkChannel() {
+        return stub != null && !((ManagedChannel) stub.getChannel()).isShutdown();
+    }
+
+    /* TODO: handle this override problem */
+    @Override
+    public Notifier<PartitionHeartbeatRequest.Builder> connectPartition(Listener<PulseResponse>
+                                                                                listener) {
+        return new PartitionHeartbeat(listener);
+    }
+
+    @Override
+    public boolean resetStub(String host, Notifier notifier) {
+        log.info("reset stub: current, {}, new: {}, channel state:{}", getCurrentHost(), host,
+                 checkChannel());
+        if (Objects.equals(host, getCurrentHost()) && checkChannel()) {
+            return false;
+        }
+
+        if (notifier != null) {
+            notifier.close();
+        }
+
+        this.stub = HgPdPulseGrpc.newStub(Channels.getChannel(host));
+        log.info("pd pulse connect to {}", host);
+        this.pdServerAddress = host;
+        return true;
+    }
+
+    /*** PartitionHeartbeat's implement  ***/
+    private class PartitionHeartbeat extends
+                                     AbstractConnector<PartitionHeartbeatRequest.Builder,
+                                             PulseResponse> {
+        private long observerId = -1;
+
+        PartitionHeartbeat(Listener<PulseResponse> listener) {
+            super(listener, PulseType.PULSE_TYPE_PARTITION_HEARTBEAT);
+        }
+
+        private void setObserverId(long observerId) {
+            if (this.observerId == -1) {
+                this.observerId = observerId;
+            }
+        }
+
+        @Override
+        public void notifyServer(PartitionHeartbeatRequest.Builder requestBuilder) {
+            this.reqStream.onNext(PulseRequest.newBuilder()
+                                              .setNoticeRequest(
+                                                      PulseNoticeRequest.newBuilder()
+                                                                        .setPartitionHeartbeatRequest(
+                                                                                requestBuilder.build()
+                                                                        ).build()
+                                              ).build()
+            );
+        }
+
+        @Override
+        public void onNext(PulseResponse pulseResponse) {
+            this.setObserverId(pulseResponse.getObserverId());
+            long noticeId = pulseResponse.getNoticeId();
+            this.listener.onNext(pulseResponse);
+            this.listener.onNotice(new PartitionNotice(noticeId,
+                                                       e -> super.ackNotice(e, observerId),
+                                                       pulseResponse));
+        }
+
+    }
+
+    private abstract class AbstractConnector<N, L> implements Notifier<N>,
+                                                              StreamObserver<PulseResponse> {
+        Listener<L> listener;
+        StreamObserver<PulseRequest> reqStream;
+        PulseType pulseType;
+        PulseRequest.Builder reqBuilder = PulseRequest.newBuilder();
+        PulseAckRequest.Builder ackBuilder = PulseAckRequest.newBuilder();
+
+        private AbstractConnector(Listener<L> listener, PulseType pulseType) {
+            this.listener = listener;
+            this.pulseType = pulseType;
+            this.init();
+        }
+
+        void init() {
+            PulseCreateRequest.Builder builder = PulseCreateRequest.newBuilder()
+                                                                   .setPulseType(this.pulseType);
+
+            this.reqStream = PDPulseImpl.this.stub.pulse(this);
+            this.reqStream.onNext(reqBuilder.clear().setCreateRequest(builder).build());
+        }
+
+        /*** notifier ***/
+        @Override
+        public void close() {
+            this.reqStream.onCompleted();
+        }
+
+        @Override
+        public abstract void notifyServer(N t);
+
+        @Override
+        public void crash(String error) {
+            this.reqStream.onError(new Throwable(error));
+        }
+
+        /*** listener  ***/
+        @Override
+        public abstract void onNext(PulseResponse pulseResponse);
+
+        @Override
+        public void onError(Throwable throwable) {
+            this.listener.onError(throwable);
+        }
+
+        @Override
+        public void onCompleted() {
+            this.listener.onCompleted();
+        }
+
+        protected void ackNotice(long noticeId, long observerId) {
+            threadPool.execute(() -> {
+                // log.info("send ack: {}, ts: {}", noticeId, System.currentTimeMillis());
+                this.reqStream.onNext(reqBuilder.clear()
+                                                .setAckRequest(
+                                                        this.ackBuilder.clear()
+                                                                       .setNoticeId(noticeId)
+                                                                       .setObserverId(observerId)
+                                                                       .build()
+                                                ).build()
+                );
+            });
+        }
+    }
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatch.java

@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.io.Closeable;
+import java.util.function.Consumer;
+
+import org.apache.hugegraph.pd.grpc.watch.WatchResponse;
+import org.apache.hugegraph.pd.watch.NodeEvent;
+import org.apache.hugegraph.pd.watch.PartitionEvent;
+
+public interface PDWatch {
+
+    /**
+     * Watch the events of all store-nodes registered in the remote PD-Server.
+     *
+     * @param listener
+     * @return
+     */
+    //PDWatcher watchNode(Listener<NodeEvent> listener);
+
+    /**
+     * Watch the events of the store-nodes assigned to a specified graph.
+     *
+     * @param graph the graph name which you want to watch
+     * @param listener
+     * @return
+     */
+    //PDWatcher watchNode(String graph, Listener<NodeEvent> listener);
+
+    String getCurrentHost();
+
+    boolean checkChannel();
+
+    /*** inner static methods ***/
+    static <T> Listener<T> listener(Consumer<T> onNext) {
+        return listener(onNext, t -> {
+        }, () -> {
+        });
+    }
+
+    static <T> Listener<T> listener(Consumer<T> onNext, Consumer<Throwable> onError) {
+        return listener(onNext, onError, () -> {
+        });
+    }
+
+    static <T> Listener<T> listener(Consumer<T> onNext, Runnable onCompleted) {
+        return listener(onNext, t -> {
+        }, onCompleted);
+    }
+
+    static <T> Listener<T> listener(Consumer<T> onNext, Consumer<Throwable> onError,
+                                    Runnable onCompleted) {
+        return new Listener<T>() {
+            @Override
+            public void onNext(T response) {
+                onNext.accept(response);
+            }
+
+            @Override
+            public void onError(Throwable throwable) {
+                onError.accept(throwable);
+            }
+
+            @Override
+            public void onCompleted() {
+                onCompleted.run();
+            }
+        };
+    }
+
+    /**
+     * @param listener
+     * @return
+     */
+    Watcher watchPartition(Listener<PartitionEvent> listener);
+
+    Watcher watchNode(Listener<NodeEvent> listener);
+
+    Watcher watchGraph(Listener<WatchResponse> listener);
+
+    Watcher watchShardGroup(Listener<WatchResponse> listener);
+
+
+    /**
+     * Interface of Watcher.
+     */
+    interface Listener<T> {
+        /**
+         * Invoked on new events.
+         *
+         * @param response the response.
+         */
+        void onNext(T response);
+
+        /**
+         * Invoked on errors.
+         *
+         * @param throwable the error.
+         */
+        void onError(Throwable throwable);
+
+        /**
+         * Invoked on completion.
+         */
+        default void onCompleted() {};
+    }
+
+    interface Watcher extends Closeable {
+        /**
+         * closes this watcher and all its resources.
+         */
+        @Override
+        void close();
+
+        /**
+         * Requests the latest revision processed and propagates it to listeners
+         */
+        // TODO: what's it for?
+        //void requestProgress();
+    }
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatchImpl.java

@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.util.function.Supplier;
+
+import org.apache.hugegraph.pd.grpc.watch.HgPdWatchGrpc;
+import org.apache.hugegraph.pd.grpc.watch.WatchCreateRequest;
+import org.apache.hugegraph.pd.grpc.watch.WatchNodeResponse;
+import org.apache.hugegraph.pd.grpc.watch.WatchPartitionResponse;
+import org.apache.hugegraph.pd.grpc.watch.WatchRequest;
+import org.apache.hugegraph.pd.grpc.watch.WatchResponse;
+import org.apache.hugegraph.pd.grpc.watch.WatchType;
+import org.apache.hugegraph.pd.watch.NodeEvent;
+import org.apache.hugegraph.pd.watch.PartitionEvent;
+
+import io.grpc.ManagedChannel;
+import io.grpc.stub.StreamObserver;
+
+final class PDWatchImpl implements PDWatch {
+
+    private final HgPdWatchGrpc.HgPdWatchStub stub;
+
+    private final String pdServerAddress;
+
+    // TODO: support several servers.
+    PDWatchImpl(String pdServerAddress) {
+        this.pdServerAddress = pdServerAddress;
+        this.stub = HgPdWatchGrpc.newStub(Channels.getChannel(pdServerAddress));
+    }
+
+    @Override
+    public String getCurrentHost() {
+        return this.pdServerAddress;
+    }
+
+    @Override
+    public boolean checkChannel() {
+        return stub != null && !((ManagedChannel) stub.getChannel()).isShutdown();
+    }
+
+    /**
+     * Get Partition change watcher.
+     *
+     * @param listener
+     * @return
+     */
+    @Override
+    public Watcher watchPartition(Listener<PartitionEvent> listener) {
+        return new PartitionWatcher(listener);
+    }
+
+    /**
+     * Get Store-Node change watcher.
+     *
+     * @param listener
+     * @return
+     */
+    @Override
+    public Watcher watchNode(Listener<NodeEvent> listener) {
+        return new NodeWatcher(listener);
+    }
+
+    @Override
+    public Watcher watchGraph(Listener<WatchResponse> listener) {
+        return new GraphWatcher(listener);
+    }
+
+    @Override
+    public Watcher watchShardGroup(Listener<WatchResponse> listener) {
+        return new ShardGroupWatcher(listener);
+    }
+
+    private class GraphWatcher extends AbstractWatcher<WatchResponse> {
+
+        private GraphWatcher(Listener listener) {
+            super(listener,
+                  () -> WatchCreateRequest
+                      .newBuilder()
+                      .setWatchType(WatchType.WATCH_TYPE_GRAPH_CHANGE)
+                      .build()
+            );
+        }
+
+        @Override
+        public void onNext(WatchResponse watchResponse) {
+            this.listener.onNext(watchResponse);
+        }
+    }
+
+    private class ShardGroupWatcher extends AbstractWatcher<WatchResponse> {
+
+        private ShardGroupWatcher(Listener listener) {
+            super(listener,
+                  () -> WatchCreateRequest
+                      .newBuilder()
+                      .setWatchType(WatchType.WATCH_TYPE_SHARD_GROUP_CHANGE)
+                      .build()
+            );
+        }
+
+        @Override
+        public void onNext(WatchResponse watchResponse) {
+            this.listener.onNext(watchResponse);
+        }
+    }
+
+    private class PartitionWatcher extends AbstractWatcher<PartitionEvent> {
+
+        private PartitionWatcher(Listener listener) {
+            super(listener,
+                  () -> WatchCreateRequest
+                      .newBuilder()
+                      .setWatchType(WatchType.WATCH_TYPE_PARTITION_CHANGE)
+                      .build()
+            );
+        }
+
+        @Override
+        public void onNext(WatchResponse watchResponse) {
+            WatchPartitionResponse res = watchResponse.getPartitionResponse();
+            PartitionEvent event = new PartitionEvent(res.getGraph(), res.getPartitionId(),
+                                                      PartitionEvent.ChangeType.grpcTypeOf(
+                                                          res.getChangeType()));
+            this.listener.onNext(event);
+        }
+    }
+
+    private class NodeWatcher extends AbstractWatcher<NodeEvent> {
+        private NodeWatcher(Listener listener) {
+            super(listener,
+                  () -> WatchCreateRequest
+                      .newBuilder()
+                      .setWatchType(WatchType.WATCH_TYPE_STORE_NODE_CHANGE)
+                      .build()
+            );
+        }
+
+        @Override
+        public void onNext(WatchResponse watchResponse) {
+            WatchNodeResponse res = watchResponse.getNodeResponse();
+            NodeEvent event = new NodeEvent(res.getGraph(), res.getNodeId(),
+                                            NodeEvent.EventType.grpcTypeOf(res.getNodeEventType()));
+            this.listener.onNext(event);
+        }
+    }
+
+    private abstract class AbstractWatcher<T> implements Watcher, StreamObserver<WatchResponse> {
+        Listener<T> listener;
+        StreamObserver<WatchRequest> reqStream;
+        Supplier<WatchCreateRequest> requestSupplier;
+
+        private AbstractWatcher(Listener<T> listener,
+                                Supplier<WatchCreateRequest> requestSupplier) {
+            this.listener = listener;
+            this.requestSupplier = requestSupplier;
+            this.init();
+        }
+
+        void init() {
+            this.reqStream = PDWatchImpl.this.stub.watch(this);
+            this.reqStream.onNext(WatchRequest.newBuilder().setCreateRequest(
+                this.requestSupplier.get()
+            ).build());
+        }
+
+        @Override
+        public void close() {
+            this.reqStream.onCompleted();
+        }
+
+        @Override
+        public abstract void onNext(WatchResponse watchResponse);
+
+        @Override
+        public void onError(Throwable throwable) {
+
+            this.listener.onError(throwable);
+        }
+
+        @Override
+        public void onCompleted() {
+            this.listener.onCompleted();
+        }
+    }
+
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PartitionNotice.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.pulse;
+
+import java.util.function.Consumer;
+
+import org.apache.hugegraph.pd.grpc.pulse.PulseResponse;
+
+public class PartitionNotice implements PulseServerNotice<PulseResponse> {
+    private final long noticeId;
+    private final Consumer<Long> ackConsumer;
+    private final PulseResponse content;
+
+    public PartitionNotice(long noticeId, Consumer<Long> ackConsumer, PulseResponse content) {
+        this.noticeId = noticeId;
+        this.ackConsumer = ackConsumer;
+        this.content = content;
+    }
+
+    @Override
+    public void ack() {
+        this.ackConsumer.accept(this.noticeId);
+    }
+
+    @Override
+    public long getNoticeId() {
+        return this.noticeId;
+    }
+
+    @Override
+    public PulseResponse getContent() {
+        return this.content;
+    }
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PulseServerNotice.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.pulse;
+
+public interface PulseServerNotice<T> {
+    /**
+     * @throws RuntimeException when failed to send ack-message to pd-server
+     */
+    void ack();
+
+    long getNoticeId();
+
+    /**
+     * Return a response object of gRPC stream.
+     *
+     * @return
+     */
+    T getContent();
+
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/NodeEvent.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.watch;
+
+import java.util.Objects;
+
+import org.apache.hugegraph.pd.grpc.watch.NodeEventType;
+
+public class NodeEvent {
+    private final String graph;
+    private final long nodeId;
+    private final EventType eventType;
+
+    public NodeEvent(String graph, long nodeId, EventType eventType) {
+        this.graph = graph;
+        this.nodeId = nodeId;
+        this.eventType = eventType;
+    }
+
+    public String getGraph() {
+        return graph;
+    }
+
+    public long getNodeId() {
+        return nodeId;
+    }
+
+    public EventType getEventType() {
+        return eventType;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        NodeEvent nodeEvent = (NodeEvent) o;
+        return nodeId == nodeEvent.nodeId && Objects.equals(graph,
+                                                            nodeEvent.graph) &&
+               eventType == nodeEvent.eventType;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(graph, nodeId, eventType);
+    }
+
+    @Override
+    public String toString() {
+        return "NodeEvent{" +
+               "graph='" + graph + '\'' +
+               ", nodeId=" + nodeId +
+               ", eventType=" + eventType +
+               '}';
+    }
+
+    public enum EventType {
+        UNKNOWN,
+        NODE_ONLINE,
+        NODE_OFFLINE,
+        NODE_RAFT_CHANGE,
+        NODE_PD_LEADER_CHANGE;
+
+        public static EventType grpcTypeOf(NodeEventType grpcType) {
+            switch (grpcType) {
+                case NODE_EVENT_TYPE_NODE_ONLINE:
+                    return NODE_ONLINE;
+                case NODE_EVENT_TYPE_NODE_OFFLINE:
+                    return NODE_OFFLINE;
+                case NODE_EVENT_TYPE_NODE_RAFT_CHANGE:
+                    return NODE_RAFT_CHANGE;
+                case NODE_EVENT_TYPE_PD_LEADER_CHANGE:
+                    return NODE_PD_LEADER_CHANGE;
+                default:
+                    return UNKNOWN;
+            }
+
+        }
+
+    }
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/PDWatcher.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.watch;
+
+public class PDWatcher {
+
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/PartitionEvent.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.watch;
+
+import java.util.Objects;
+
+import org.apache.hugegraph.pd.grpc.watch.WatchChangeType;
+
+public class PartitionEvent {
+    private final String graph;
+    private final int partitionId;
+    private final ChangeType changeType;
+
+    public PartitionEvent(String graph, int partitionId, ChangeType changeType) {
+        this.graph = graph;
+        this.partitionId = partitionId;
+        this.changeType = changeType;
+    }
+
+    public String getGraph() {
+        return this.graph;
+    }
+
+    public int getPartitionId() {
+        return this.partitionId;
+    }
+
+    public ChangeType getChangeType() {
+        return this.changeType;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        PartitionEvent that = (PartitionEvent) o;
+        return partitionId == that.partitionId && Objects.equals(graph, that.graph) &&
+               changeType == that.changeType;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(graph, partitionId, changeType);
+    }
+
+    @Override
+    public String toString() {
+        return "PartitionEvent{" +
+               "graph='" + graph + '\'' +
+               ", partitionId=" + partitionId +
+               ", changeType=" + changeType +
+               '}';
+    }
+
+    public enum ChangeType {
+        UNKNOWN,
+        ADD,
+        ALTER,
+        DEL;
+
+        public static ChangeType grpcTypeOf(WatchChangeType grpcType) {
+            switch (grpcType) {
+                case WATCH_CHANGE_TYPE_ADD:
+                    return ADD;
+                case WATCH_CHANGE_TYPE_ALTER:
+                    return ALTER;
+                case WATCH_CHANGE_TYPE_DEL:
+                    return DEL;
+                default:
+                    return UNKNOWN;
+            }
+        }
+    }
+}

hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/WatchType.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.watch;
+
+enum WatchType {
+
+    PARTITION_CHANGE(10);
+
+    private final int value;
+
+    WatchType(int value) {
+        this.value = value;
+    }
+
+}

hugegraph-pd/hg-pd-client/src/test/java/org/apache/hugegraph/pd/PartitionCacheTest.java

@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hugegraph.pd.common.KVPair;
+import org.apache.hugegraph.pd.common.PartitionCache;
+import org.apache.hugegraph.pd.grpc.Metapb;
+
+import com.google.common.collect.Range;
+import com.google.common.collect.RangeMap;
+import com.google.common.collect.TreeRangeMap;
+
+public class PartitionCacheTest {
+
+    // @Test
+    public void test() {
+        PartitionCache cache = new PartitionCache();
+        for (int i = 0; i < 10; i++) {
+            KVPair<Metapb.Partition, Metapb.Shard> partShards =
+                    new KVPair<>(Metapb.Partition.newBuilder()
+                                                 .setStartKey(i * 10)
+                                                 .setEndKey((i + 1) * 10)
+                                                 .build(), null);
+            cache.updatePartition("aa", i, partShards.getKey());
+        }
+
+        for (int i = 0; i < 100; i++) {
+            KVPair<Metapb.Partition, Metapb.Shard> partShards = cache.getPartitionByCode("aa", i);
+            System.out.println(" " + i + " " + partShards.getKey().getStartKey());
+        }
+    }
+
+
+    // @Test
+    public void test1() {
+        Map<String, RangeMap<Long, Integer>> keyToPartIdCache = new HashMap<>();
+        // graphName + PartitionID组成key
+        Map<String, KVPair<Metapb.Partition, Metapb.Shard>> partitionCache = new HashMap<>();
+
+        // 缓存全部Store，用于全库查询，需要优化
+        Map<String, List<Metapb.Store>> allStoresCache = new HashMap<>();
+
+        keyToPartIdCache.put("a", TreeRangeMap.create());
+
+        keyToPartIdCache.get("a")
+                        .put(Range.closedOpen(1L, 2L), 1);
+
+        allStoresCache.put("a", new ArrayList<>());
+        allStoresCache.get("a").add(Metapb.Store.newBuilder().setId(34).build());
+
+
+        Map<String, RangeMap<Long, Integer>> keyToPartIdCache2 =
+                cloneKeyToPartIdCache(keyToPartIdCache);
+        System.out.println(keyToPartIdCache2.size());
+    }
+
+    public Map<String, RangeMap<Long, Integer>> cloneKeyToPartIdCache(
+            Map<String, RangeMap<Long, Integer>> cache) {
+        Map<String, RangeMap<Long, Integer>> cacheClone = new HashMap<>();
+        cache.forEach((k1, v1) -> {
+            cacheClone.put(k1, TreeRangeMap.create());
+            v1.asMapOfRanges().forEach((k2, v2) -> {
+                cacheClone.get(k1).put(k2, v2);
+            });
+        });
+        return cacheClone;
+    }
+
+    public Map<String, KVPair<Metapb.Partition, Metapb.Shard>>
+    clonePartitionCache(Map<String, KVPair<Metapb.Partition, Metapb.Shard>> cache) {
+        Map<String, KVPair<Metapb.Partition, Metapb.Shard>> cacheClone = new HashMap<>();
+        cacheClone.putAll(cache);
+        return cacheClone;
+    }
+
+    public Map<String, List<Metapb.Store>>
+    cloneStoreCache(Map<String, List<Metapb.Store>> cache) {
+        Map<String, List<Metapb.Store>> cacheClone = new HashMap<>();
+        cacheClone.putAll(cache);
+        return cacheClone;
+    }
+}

hugegraph-pd/hg-pd-client/src/test/java/org/apache/hugegraph/pd/StoreRegisterTest.java

@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import org.apache.hugegraph.pd.client.PDClient;
+import org.apache.hugegraph.pd.client.PDConfig;
+import org.apache.hugegraph.pd.client.PDPulse;
+import org.apache.hugegraph.pd.client.PDPulseImpl;
+import org.apache.hugegraph.pd.common.KVPair;
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.apache.hugegraph.pd.grpc.pulse.PartitionHeartbeatRequest;
+import org.apache.hugegraph.pd.grpc.pulse.PulseResponse;
+import org.apache.hugegraph.pd.pulse.PulseServerNotice;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+public class StoreRegisterTest {
+    private static PDClient pdClient;
+    private final String storeAddr = "localhost";
+    private final String graphName = "default/hugegraph/g";
+    private long storeId = 0;
+
+    @BeforeClass
+    public static void beforeClass() throws Exception {
+        PDConfig config = PDConfig.of("localhost:8686");
+        config.setEnableCache(true);
+        pdClient = PDClient.create(config);
+    }
+
+    // @Test
+    public void testRegisterStore() throws PDException {
+        Metapb.Store store = Metapb.Store.newBuilder().setAddress(storeAddr).build();
+        try {
+            storeId = pdClient.registerStore(store);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        Assert.assertTrue("RegisterStore store_id = " + storeId, storeId != 0);
+    }
+
+    // @Test
+    public void testGetStore() throws PDException {
+        testRegisterStore();
+        Metapb.Store store = pdClient.getStore(storeId);
+        Assert.assertEquals(storeAddr, store.getAddress());
+        System.out.println(store);
+    }
+
+    // @Test
+    public void testGetActiveStores() throws PDException {
+        testRegisterStore();
+        List<Metapb.Store> stores = pdClient.getActiveStores(graphName);
+        stores.forEach((e) -> {
+            System.out.println("-------------------------------------");
+            System.out.println(e);
+        });
+    }
+
+
+    // @Test
+    public void testStoreHeartbeat() throws PDException {
+        testRegisterStore();
+        Metapb.StoreStats stats = Metapb.StoreStats.newBuilder().setStoreId(storeId).build();
+        pdClient.storeHeartbeat(stats);
+        List<Metapb.Store> stores = pdClient.getActiveStores(graphName);
+        boolean exist = false;
+        for (Metapb.Store store : stores) {
+            if (store.getId() == storeId) {
+                exist = true;
+                break;
+            }
+        }
+        Assert.assertTrue(exist);
+    }
+
+
+    // @Test
+    public void testPartitionHeartbeat() throws InterruptedException, PDException {
+        testRegisterStore();
+        PDPulse pdPulse = new PDPulseImpl(pdClient.getLeaderIp());
+
+        PDPulse.Notifier<PartitionHeartbeatRequest.Builder> notifier = pdPulse.connectPartition(
+                new PDPulse.Listener<PulseResponse>() {
+
+                    @Override
+                    public void onNext(PulseResponse response) {
+
+                    }
+
+                    @Override
+                    public void onNotice(PulseServerNotice<PulseResponse> notice) {
+
+                    }
+
+                    @Override
+                    public void onError(Throwable throwable) {
+
+                    }
+
+                    @Override
+                    public void onCompleted() {
+
+                    }
+                });
+        KVPair<Metapb.Partition, Metapb.Shard> partShard =
+                pdClient.getPartition("test", "1".getBytes(StandardCharsets.UTF_8));
+        notifier.notifyServer(PartitionHeartbeatRequest.newBuilder().setStates(
+                Metapb.PartitionStats.newBuilder().addGraphName("test")
+                                     .setId(partShard.getKey().getId())
+                                     .setLeader(Metapb.Shard.newBuilder().setStoreId(1).build())));
+
+
+        Thread.sleep(10000);
+    }
+
+}

hugegraph-pd/hg-pd-client/src/test/java/org/apache/hugegraph/pd/client/DiscoveryClientImplTest.java

@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Vector;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hugegraph.pd.grpc.discovery.NodeInfos;
+import org.apache.hugegraph.pd.grpc.discovery.Query;
+import org.junit.Assert;
+
+public class DiscoveryClientImplTest {
+
+    private static final AtomicLong label = new AtomicLong();
+    String address = "localhost:80";
+    int delay = 1000;
+    int wait = delay * 3 + 500;
+
+    // @Test
+    public void registerStore() throws InterruptedException {
+
+        HashMap<String, String> labels = new HashMap<>();
+
+        labels.put("metrics", "/actuator/prometheus");
+        labels.put("target", "10.81.116.77:8520");
+        labels.put("scheme", "http");
+        labels.put("__relabeling", "http");
+        labels.put("no_relabeling", "http");
+        getClient("store", "address1", labels);
+
+        labels.put("metrics", "/actuator/prometheus");
+        labels.put("target", "10.81.116.78:8520");
+        labels.put("scheme", "http");
+        getClient("store", "address2", labels);
+
+        labels.put("metrics", "/actuator/prometheus");
+        labels.put("target", "10.81.116.79:8520");
+        labels.put("scheme", "http");
+        getClient("store", "address3", labels);
+
+        labels.put("metrics", "/actuator/prometheus");
+        labels.put("target", "10.81.116.78:8620");
+        labels.put("scheme", "http");
+        getClient("pd", "address1", labels);
+
+        labels.put("metrics", "/graph/metrics");
+        labels.put("target", "10.37.1.1:9200");
+        labels.put("scheme", "https");
+        getClient("hugegraph", "address1", labels);
+    }
+
+    // @Test
+    public void testNodes() throws InterruptedException {
+        String appName = "hugegraph";
+        register(appName, address);
+    }
+
+    // @Test
+    public void testMultiNode() throws InterruptedException {
+        for (int i = 0; i < 2; i++) {
+            register("app" + i, address + i);
+        }
+    }
+
+    // @Test
+    public void testParallelMultiNode() throws InterruptedException {
+        CountDownLatch latch = new CountDownLatch(30);
+        Vector<Exception> exceptions = new Vector<>();
+        for (int i = 0; i < 30; i++) {
+            int finalI = i;
+            new Thread(() -> {
+                try {
+                    for (int j = 0; j < 3; j++) {
+                        register("app" + finalI, address + j);
+                    }
+                } catch (Exception e) {
+                    exceptions.add(e);
+                } finally {
+                    latch.countDown();
+                }
+            }).start();
+        }
+        latch.await();
+        Assert.assertEquals(0, exceptions.size());
+    }
+
+    private void register(String appName, String address) throws InterruptedException {
+
+        HashMap<String, String> labels = new HashMap<>();
+        String labelValue = String.valueOf(label.incrementAndGet());
+        labels.put("address", labelValue);
+        labels.put("address1", labelValue);
+        Query query = Query.newBuilder().setAppName(
+                appName).setVersion("0.13.0").putAllLabels(labels).build();
+        DiscoveryClientImpl discoveryClient = getClient(appName, address, labels);
+        Thread.sleep(10000);
+        NodeInfos nodeInfos1 = discoveryClient.getNodeInfos(query);
+        Assert.assertEquals(1, nodeInfos1.getInfoCount());
+        DiscoveryClientImpl discoveryClient1 = getClient(appName, address + 0, labels);
+        Thread.sleep(10000);
+        Assert.assertEquals(2, discoveryClient.getNodeInfos(query).getInfoCount());
+        Query query1 = Query.newBuilder().setAppName(
+                appName).setVersion("0.12.0").putAllLabels(labels).build();
+        Assert.assertEquals(0, discoveryClient.getNodeInfos(query1).getInfoCount());
+        discoveryClient.cancelTask();
+        discoveryClient1.cancelTask();
+        Thread.sleep(wait);
+        NodeInfos nodeInfos = discoveryClient.getNodeInfos(query);
+        System.out.println(nodeInfos);
+        Assert.assertEquals(0, nodeInfos.getInfoCount());
+        discoveryClient.close();
+        discoveryClient1.close();
+    }
+
+    private DiscoveryClientImpl getClient(String appName, String address, Map labels) {
+        DiscoveryClientImpl discoveryClient = null;
+        try {
+            discoveryClient = DiscoveryClientImpl.newBuilder().setCenterAddress(
+                    "localhost:8687,localhost:8686,localhost:8688").setAddress(address).setAppName(
+                    appName).setDelay(delay).setVersion("0.13.0").setId(
+                    "0").setLabels(labels).build();
+            discoveryClient.scheduleTask();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+        return discoveryClient;
+    }
+}

hugegraph-pd/hg-pd-client/src/test/java/org/apache/hugegraph/pd/client/LicenseClientImplTest.java

@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.io.File;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hugegraph.pd.grpc.Pdpb;
+import org.apache.hugegraph.pd.grpc.kv.KResponse;
+import org.apache.hugegraph.pd.grpc.kv.KvResponse;
+import org.yaml.snakeyaml.Yaml;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class LicenseClientImplTest {
+
+    // @Test
+    public void putLicense() {
+        PDConfig pdConfig = PDConfig.of("localhost:8686,localhost:8687,localhost:8688");
+        //PDConfig pdConfig = PDConfig.of("localhost:8686");
+        pdConfig.setEnableCache(true);
+        try (LicenseClient c = new LicenseClient(pdConfig)) {
+            File file = new File("../conf/hugegraph.license");
+            byte[] bytes = FileUtils.readFileToByteArray(file);
+            Pdpb.PutLicenseResponse putLicenseResponse = c.putLicense(bytes);
+            Pdpb.Error error = putLicenseResponse.getHeader().getError();
+            log.info(error.getMessage());
+            assert error.getType().equals(Pdpb.ErrorType.OK);
+        } catch (Exception e) {
+            log.error("put license with error: ", e);
+        }
+    }
+
+    // @Test
+    public void getKv() {
+        PDConfig pdConfig = PDConfig.of("10.157.12.36:8686");
+        pdConfig.setEnableCache(true);
+        try (KvClient c = new KvClient(pdConfig)) {
+            KResponse kResponse = c.get("S:FS");
+            Pdpb.Error error = kResponse.getHeader().getError();
+            log.info(error.getMessage());
+            assert error.getType().equals(Pdpb.ErrorType.OK);
+            Properties ymlConfig = getYmlConfig(kResponse.getValue());
+            Object property = ymlConfig.get("rocksdb.write_buffer_size");
+            assert property.toString().equals("32000000");
+        } catch (Exception e) {
+            log.error("put license with error: ", e);
+        }
+    }
+
+    // @Test
+    public void putKv() {
+        PDConfig pdConfig = PDConfig.of("127.0.0.1.70:8688");
+        pdConfig.setEnableCache(true);
+        try (KvClient c = new KvClient(pdConfig)) {
+            long l = System.currentTimeMillis();
+            KvResponse kvResponse = c.put("S:Timestamp", String.valueOf(l));
+            Pdpb.Error error = kvResponse.getHeader().getError();
+            log.info(error.getMessage());
+            assert error.getType().equals(Pdpb.ErrorType.OK);
+        } catch (Exception e) {
+            log.error("put license with error: ", e);
+        }
+    }
+
+    // @Test
+    public void putKvLocal() {
+        PDConfig pdConfig = PDConfig.of("localhost:8686");
+        pdConfig.setEnableCache(true);
+        try (KvClient c = new KvClient(pdConfig)) {
+            long l = System.currentTimeMillis();
+            KvResponse kvResponse = c.put("S:Timestamp", String.valueOf(l));
+            Pdpb.Error error = kvResponse.getHeader().getError();
+            log.info(error.getMessage());
+            assert error.getType().equals(Pdpb.ErrorType.OK);
+        } catch (Exception e) {
+            log.error("put license with error: ", e);
+        }
+    }
+
+    private Properties getYmlConfig(String yml) {
+        Yaml yaml = new Yaml();
+        Iterable<Object> load = yaml.loadAll(yml);
+        Iterator<Object> iterator = load.iterator();
+        Properties properties = new Properties();
+        while (iterator.hasNext()) {
+            Map<String, Object> next = (Map<String, Object>) iterator.next();
+            map2Properties(next, "", properties);
+        }
+        return properties;
+    }
+
+    private void map2Properties(Map<String, Object> map, String prefix, Properties properties) {
+
+        for (Map.Entry<String, Object> entry : map.entrySet()) {
+            String key = entry.getKey();
+            String newPrefix = prefix == null || prefix.length() == 0 ? key : prefix + "." + key;
+            Object value = entry.getValue();
+            if (!(value instanceof Map)) {
+                properties.put(newPrefix, value);
+            } else {
+                map2Properties((Map<String, Object>) value, newPrefix, properties);
+            }
+
+        }
+    }
+
+}

hugegraph-pd/hg-pd-client/src/test/java/org/apache/hugegraph/pd/client/PDPulseTest.java

@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hugegraph.pd.client.test.HgPDTestUtil;
+import org.apache.hugegraph.pd.grpc.pulse.PartitionHeartbeatRequest;
+import org.apache.hugegraph.pd.pulse.PulseServerNotice;
+import org.junit.BeforeClass;
+
+public class PDPulseTest {
+    private static PDClient pdClient;
+
+    private final long storeId = 0;
+    private final String storeAddress = "localhost";
+    private final String graphName = "graph1";
+
+    @BeforeClass
+    public static void beforeClass() throws Exception {
+        PDConfig pdConfig = PDConfig.of("localhost:8686");
+        pdConfig.setEnableCache(true);
+        pdClient = PDClient.create(pdConfig);
+        pdClient.getLeader();
+    }
+
+    // @Test
+    public void listen() {
+
+        PDPulse pulse = new PDPulseImpl(pdClient.getLeaderIp());
+        CountDownLatch latch = new CountDownLatch(60);
+
+        PDPulse.Notifier<PartitionHeartbeatRequest.Builder> notifier1 =
+                pulse.connectPartition(new PulseListener(latch, "listener1"));
+        PDPulse.Notifier<PartitionHeartbeatRequest.Builder> notifier2 =
+                pulse.connectPartition(new PulseListener(latch, "listener2"));
+        PDPulse.Notifier<PartitionHeartbeatRequest.Builder> notifier3 =
+                pulse.connectPartition(new PulseListener(latch, "listener3"));
+
+        try {
+            latch.await(120, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        PartitionHeartbeatRequest.Builder builder = PartitionHeartbeatRequest.newBuilder();
+
+        notifier1.notifyServer(builder);
+
+
+        notifier2.notifyServer(builder);
+
+        notifier3.notifyServer(builder);
+
+        notifier1.close();
+        notifier2.close();
+        notifier3.close();
+    }
+
+
+    private class PulseListener<T> implements PDPulse.Listener<T> {
+        private final String listenerName;
+        CountDownLatch latch = new CountDownLatch(10);
+
+        private PulseListener(CountDownLatch latch, String listenerName) {
+            this.latch = latch;
+            this.listenerName = listenerName;
+        }
+
+        @Override
+        public void onNext(T response) {
+            // println(this.listenerName+" res: "+response);
+            // this.latch.countDown();
+        }
+
+        @Override
+        public void onNotice(PulseServerNotice<T> notice) {
+            HgPDTestUtil.println(this.listenerName + " ---> res: " + notice.getContent());
+
+            notice.ack();
+            this.latch.countDown();
+        }
+
+        @Override
+        public void onError(Throwable throwable) {
+            HgPDTestUtil.println(this.listenerName + " error: " + throwable.toString());
+        }
+
+        @Override
+        public void onCompleted() {
+            HgPDTestUtil.println(this.listenerName + " is completed");
+        }
+    }
+}

hugegraph-pd/hg-pd-client/src/test/java/org/apache/hugegraph/pd/client/PDWatchTest.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hugegraph.pd.client.test.HgPDTestUtil;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+@Deprecated
+public class PDWatchTest {
+    private static PDClient pdClient;
+
+    private final long storeId = 0;
+    private final String storeAddr = "localhost";
+    private final String graphName = "graph1";
+
+    @BeforeClass
+    public static void beforeClass() {
+        pdClient = PDClient.create(PDConfig.of("localhost:9000"));
+    }
+
+    @Test
+    public void watch() {
+        PDWatch watch = pdClient.getWatchClient();
+        CountDownLatch latch = new CountDownLatch(10);
+
+        PDWatch.Watcher watcher1 = watch.watchPartition(new WatchListener<>(latch, "watcher1"));
+        PDWatch.Watcher watcher2 = watch.watchPartition(new WatchListener<>(latch, "watcher2"));
+        PDWatch.Watcher watcher3 = watch.watchPartition(new WatchListener<>(latch, "watcher3"));
+
+        PDWatch.Watcher nodeWatcher1 = watch.watchNode(new WatchListener<>(latch, "nodeWatcher1"));
+
+        try {
+            latch.await(15, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        watcher1.close();
+        watcher2.close();
+        watcher3.close();
+    }
+
+    private class WatchListener<T> implements PDWatch.Listener<T> {
+        private final String watcherName;
+        CountDownLatch latch;
+
+        private WatchListener(CountDownLatch latch, String watcherName) {
+            this.latch = latch;
+            this.watcherName = watcherName;
+        }
+
+        @Override
+        public void onNext(T response) {
+            HgPDTestUtil.println(this.watcherName + " res: " + response);
+            this.latch.countDown();
+        }
+
+        @Override
+        public void onError(Throwable throwable) {
+            HgPDTestUtil.println(this.watcherName + " error: " + throwable.toString());
+        }
+
+        @Override
+        public void onCompleted() {
+            HgPDTestUtil.println(this.watcherName + " is completed");
+        }
+    }
+}

hugegraph-pd/hg-pd-client/src/test/java/org/apache/hugegraph/pd/client/test/HgPDTestUtil.java

@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.client.test;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Iterator;
+import java.util.List;
+
+public class HgPDTestUtil {
+
+    public static void println(Object str) {
+        System.out.println(str);
+    }
+
+    public static String toStr(byte[] b) {
+        if (b == null) return "";
+        if (b.length == 0) return "";
+        return new String(b, StandardCharsets.UTF_8);
+    }
+
+    public static byte[] toBytes(String str) {
+        if (str == null) return null;
+        return str.getBytes(StandardCharsets.UTF_8);
+    }
+
+    public static byte[] toBytes(long l) {
+        ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
+        buffer.putLong(l);
+        return buffer.array();
+    }
+
+    private static byte[] toBytes(final int i) {
+        ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
+        buffer.putInt(i);
+        return buffer.array();
+    }
+
+    public static long toLong(byte[] bytes) {
+        ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
+        buffer.put(bytes);
+        buffer.flip();//need flip
+        return buffer.getLong();
+    }
+
+    public static long toInt(byte[] bytes) {
+        ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
+        buffer.put(bytes);
+        buffer.flip();//need flip
+        return buffer.getInt();
+    }
+
+    public static String padLeftZeros(String str, int n) {
+        return String.format("%1$" + n + "s", str).replace(' ', '0');
+    }
+
+    public static String toSuffix(int num, int length) {
+        return "-" + padLeftZeros(String.valueOf(num), length);
+    }
+
+    public static int amountOf(List list) {
+        if (list == null) {
+            return 0;
+        }
+        return list.size();
+    }
+
+    public static int amountOf(Iterator iterator) {
+        if (iterator == null) return 0;
+        int count = 0;
+        while (iterator.hasNext()) {
+            iterator.next();
+            count++;
+        }
+        return count;
+    }
+}

hugegraph-pd/hg-pd-clitools/pom.xml

@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements. See the NOTICE file distributed with this
+  work for additional information regarding copyright ownership. The ASF
+  licenses this file to You under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  License for the specific language governing permissions and limitations
+  under the License.
+  -->
+
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>hugegraph-pd</artifactId>
+        <groupId>org.apache.hugegraph</groupId>
+        <version>${revision}</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>hg-pd-clitools</artifactId>
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.hugegraph</groupId>
+            <artifactId>hg-pd-client</artifactId>
+            <version>${revision}</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.13.2</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <configuration>
+                            <archive>
+                                <manifest>
+                                    <mainClass>
+                                        org.apache.hugegraph.pd.clitools.Main
+                                    </mainClass>
+                                </manifest>
+                            </archive>
+                            <descriptorRefs>
+                                <descriptorRef>jar-with-dependencies</descriptorRef>
+                            </descriptorRefs>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

hugegraph-pd/hg-pd-clitools/src/main/java/org/apache/hugegraph/pd/clitools/Main.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.clitools;
+
+import org.apache.hugegraph.pd.client.PDClient;
+import org.apache.hugegraph.pd.client.PDConfig;
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.grpc.Metapb;
+
+public class Main {
+
+    public static void main(String[] args) throws PDException {
+        if (args.length < 3) {
+            String error = " usage: pd-address config key[=value] \n key list: " +
+                           "\n\tenableBatchLoad";
+            System.out.println(error);
+            System.exit(0);
+        }
+        String pd = args[0];
+        String cmd = args[1];
+        String param = args[2];
+        System.out.println(pd + " " + cmd + " " + param);
+        System.out.println("Result: \n");
+        switch (cmd) {
+            case "config":
+                doConfig(pd, param);
+            case "change_raft":
+                doChangeRaft(pd, param);
+        }
+    }
+
+    private static void doChangeRaft(String pd, String param) throws PDException {
+        PDClient pdClient = PDClient.create(PDConfig.of(pd));
+        pdClient.updatePdRaft(param);
+    }
+
+    public static void doConfig(String pd, String param) throws PDException {
+        PDClient pdClient = PDClient.create(PDConfig.of(pd));
+        String[] pair = param.split("=");
+        String key = pair[0].trim();
+        Object value = null;
+        if (pair.length > 1) {
+            value = pair[1].trim();
+        }
+        if (value == null) {
+            Metapb.PDConfig pdConfig = pdClient.getPDConfig();
+            switch (key) {
+                case "enableBatchLoad":
+                    //    value = pdConfig.getEnableBatchLoad();
+                    break;
+                case "shardCount":
+                    value = pdConfig.getShardCount();
+                    break;
+            }
+
+            System.out.println("Get config " + key + "=" + value);
+        } else {
+            Metapb.PDConfig.Builder builder = Metapb.PDConfig.newBuilder();
+            switch (key) {
+                case "enableBatchLoad":
+                    //   builder.setEnableBatchLoad(Boolean.valueOf((String)value));
+                case "shardCount":
+                    builder.setShardCount(Integer.valueOf((String) value));
+            }
+            pdClient.setPDConfig(builder.build());
+            System.out.println("Set config " + key + "=" + value);
+        }
+    }
+
+}

hugegraph-pd/hg-pd-clitools/src/test/java/org/apache/hugegraph/pd/clitools/MainTest.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.clitools;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hugegraph.pd.common.PDException;
+
+public class MainTest {
+    public static boolean test2sup(List<Integer> arrays, int tail, int res) {
+        System.out.printf("%d %d%n", tail, res);
+        if (tail == 0) {
+            System.out.printf("a = %d %d%n", tail, res);
+            return false;
+        } else if (tail == 1) {
+            System.out.printf("b = %d %d%n", arrays.get(0), res);
+            return (arrays.get(0) == res);
+        } else if (tail == 2) {
+            System.out.printf("c = %d %d %d%n", arrays.get(0), arrays.get(1), res);
+            return (arrays.get(0) + arrays.get(1) == Math.abs(res)) ||
+                   (Math.abs(arrays.get(0) - arrays.get(1)) == Math.abs(res));
+        } else {
+            return test2sup(arrays, tail - 1, res + arrays.get(tail - 1)) ||
+                   test2sup(arrays, tail - 1, res - arrays.get(tail - 1));
+        }
+    }
+
+    // @Test
+    public void getConfig() throws PDException {
+        Main.main(new String[]{"127.0.0.1:8686", "config", "enableBatchLoad"});
+    }
+
+    // @Test
+    public void setBatchTrue() throws PDException {
+        Main.main(new String[]{"127.0.0.1:8686", "config", "enableBatchLoad= true "});
+    }
+
+    // @Test
+    public void setBatchFalse() throws PDException {
+        Main.main(new String[]{"127.0.0.1:8686", "config", "enableBatchLoad=false"});
+    }
+
+    // @Test
+    public void getConfig2() throws PDException {
+        Main.main(new String[]{"127.0.0.1:8686", "config", "shardCount"});
+    }
+
+    // @Test
+    public void setShardCount1() throws PDException {
+        Main.main(new String[]{"127.0.0.1:8686", "config", "shardCount=1"});
+    }
+
+    // @Test
+    public void setShardCount3() throws PDException {
+        Main.main(new String[]{"127.0.0.1:8686", "config", "shardCount=3"});
+    }
+
+    // @Test
+    public void test2() {
+        Integer[] a = new Integer[]{1, 0, 3, 2};
+        List<Integer> aa = Arrays.asList(a);
+        System.out.printf(test2sup(aa, aa.size(), 0) ? "TRUE" : "FALSE");
+    }
+}

hugegraph-pd/hg-pd-common/pom.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements. See the NOTICE file distributed with this
+  work for additional information regarding copyright ownership. The ASF
+  licenses this file to You under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  License for the specific language governing permissions and limitations
+  under the License.
+  -->
+
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.hugegraph</groupId>
+        <artifactId>hugegraph-pd</artifactId>
+        <version>${revision}</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <artifactId>hg-pd-common</artifactId>
+
+    <properties>
+        <maven.compiler.source>11</maven.compiler.source>
+        <maven.compiler.target>11</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.hugegraph</groupId>
+            <artifactId>hg-pd-grpc</artifactId>
+            <version>${revision}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.24</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-collections4</artifactId>
+            <version>4.4</version>
+        </dependency>
+    </dependencies>
+</project>

hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/GraphCache.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.common;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hugegraph.pd.grpc.Metapb.Graph;
+import org.apache.hugegraph.pd.grpc.Metapb.Partition;
+
+import com.google.common.collect.RangeMap;
+import com.google.common.collect.TreeRangeMap;
+
+import lombok.Data;
+
+@Data
+public class GraphCache {
+
+    private Graph graph;
+    private AtomicBoolean initialized = new AtomicBoolean(false);
+    private AtomicBoolean writing = new AtomicBoolean(false);
+    private ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+    private Map<Integer, AtomicBoolean> state = new ConcurrentHashMap<>();
+    private Map<Integer, Partition> partitions = new ConcurrentHashMap<>();
+    private RangeMap<Long, Integer> range = TreeRangeMap.create();
+
+    public GraphCache(Graph graph) {
+        this.graph = graph;
+    }
+
+    public GraphCache() {
+    }
+
+    public Partition getPartition(Integer id) {
+        return partitions.get(id);
+    }
+
+    public Partition addPartition(Integer id, Partition p) {
+        return partitions.put(id, p);
+    }
+
+    public Partition removePartition(Integer id) {
+        return partitions.remove(id);
+    }
+}

hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/HgAssert.java

@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.common;
+
+import java.util.Collection;
+import java.util.Map;
+
+public final class HgAssert {
+
+    public static void isTrue(boolean expression, String message) {
+        if (message == null) {
+            throw new IllegalArgumentException("message is null");
+        }
+
+        if (!expression) {
+            throw new IllegalArgumentException(message);
+        }
+    }
+
+    public static void isFalse(boolean expression, String message) {
+        isTrue(!expression, message);
+    }
+
+    public static void isArgumentValid(byte[] bytes, String parameter) {
+        isFalse(isInvalid(bytes), "The argument is invalid: " + parameter);
+    }
+
+    public static void isArgumentValid(String str, String parameter) {
+        isFalse(isInvalid(str), "The argument is invalid: " + parameter);
+    }
+
+    public static void isArgumentNotNull(Object obj, String parameter) {
+        isTrue(obj != null, "The argument is null: " + parameter);
+    }
+
+    public static void istValid(byte[] bytes, String msg) {
+        isFalse(isInvalid(bytes), msg);
+    }
+
+    public static void isValid(String str, String msg) {
+        isFalse(isInvalid(str), msg);
+    }
+
+    public static void isNotNull(Object obj, String msg) {
+        isTrue(obj != null, msg);
+    }
+
+    public static boolean isContains(Object[] objs, Object obj) {
+        if (objs == null || objs.length == 0 || obj == null) {
+            return false;
+        }
+        for (Object item : objs) {
+            if (obj.equals(item)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public static boolean isInvalid(String... strs) {
+        if (strs == null || strs.length == 0) {
+            return true;
+        }
+        for (String item : strs) {
+            if (item == null || "".equals(item.trim())) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public static boolean isInvalid(byte[] bytes) {
+        return bytes == null || bytes.length == 0;
+    }
+
+    public static boolean isInvalid(Map<?, ?> map) {
+        return map == null || map.isEmpty();
+    }
+
+    public static boolean isInvalid(Collection<?> list) {
+        return list == null || list.isEmpty();
+    }
+
+    public static <T> boolean isContains(Collection<T> list, T item) {
+        if (list == null || item == null) {
+            return false;
+        }
+        return list.contains(item);
+    }
+
+    public static boolean isNull(Object... objs) {
+        if (objs == null) {
+            return true;
+        }
+        for (Object item : objs) {
+            if (item == null) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/KVPair.java

@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.common;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class KVPair<K, V> implements Serializable {
+
+    /**
+     * Key of this <code>Pair</code>.
+     */
+    private K key;
+    /**
+     * Value of this this <code>Pair</code>.
+     */
+    private V value;
+
+    /**
+     * Creates a new pair
+     *
+     * @param key   The key for this pair
+     * @param value The value to use for this pair
+     */
+    public KVPair(K key, V value) {
+        this.key = key;
+        this.value = value;
+    }
+
+    /**
+     * Gets the key for this pair.
+     *
+     * @return key for this pair
+     */
+    public K getKey() {
+        return key;
+    }
+
+    public void setKey(K key) {
+        this.key = key;
+    }
+
+    /**
+     * Gets the value for this pair.
+     *
+     * @return value for this pair
+     */
+    public V getValue() {
+        return value;
+    }
+
+    public void setValue(V value) {
+        this.value = value;
+    }
+
+    /**
+     * <p><code>String</code> representation of this
+     * <code>Pair</code>.</p>
+     *
+     * <p>The default name/value delimiter '=' is always used.</p>
+     *
+     * @return <code>String</code> representation of this <code>Pair</code>
+     */
+    @Override
+    public String toString() {
+        return key + "=" + value;
+    }
+
+    /**
+     * <p>Generate a hash code for this <code>Pair</code>.</p>
+     *
+     * <p>The hash code is calculated using both the name and
+     * the value of the <code>Pair</code>.</p>
+     *
+     * @return hash code for this <code>Pair</code>
+     */
+    @Override
+    public int hashCode() {
+        // name's hashCode is multiplied by an arbitrary prime number (13)
+        // in order to make sure there is a difference in the hashCode between
+        // these two parameters:
+        //  name: a  value: aa
+        //  name: aa value: a
+        return key.hashCode() * 13 + (value == null ? 0 : value.hashCode());
+    }
+
+    /**
+     * <p>Test this <code>Pair</code> for equality with another
+     * <code>Object</code>.</p>
+     *
+     * <p>If the <code>Object</code> to be tested is not a
+     * <code>Pair</code> or is <code>null</code>, then this method
+     * returns <code>false</code>.</p>
+     *
+     * <p>Two <code>Pair</code>s are considered equal if and only if
+     * both the names and values are equal.</p>
+     *
+     * @param o the <code>Object</code> to test for
+     *          equality with this <code>Pair</code>
+     * @return <code>true</code> if the given <code>Object</code> is
+     * equal to this <code>Pair</code> else <code>false</code>
+     */
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o instanceof KVPair) {
+            KVPair pair = (KVPair) o;
+            if (!Objects.equals(key, pair.key)) {
+                return false;
+            }
+            return Objects.equals(value, pair.value);
+        }
+        return false;
+    }
+}

hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PDException.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.common;
+
+public class PDException extends Exception {
+
+    private final int errorCode;
+
+    public PDException(int error) {
+        super(String.format("Error code = %d", error));
+        this.errorCode = error;
+    }
+
+    public PDException(int error, String msg) {
+        super(msg);
+        this.errorCode = error;
+    }
+
+    public PDException(int error, Throwable e) {
+        super(e);
+        this.errorCode = error;
+    }
+
+    public PDException(int error, String msg, Throwable e) {
+        super(msg, e);
+        this.errorCode = error;
+    }
+
+    public int getErrorCode() {
+        return errorCode;
+    }
+}

hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PDRuntimeException.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.common;
+
+public class PDRuntimeException extends RuntimeException {
+
+    // public static final int LICENSE_ERROR = -11;
+
+    private int errorCode = 0;
+
+    public PDRuntimeException(int error) {
+        super(String.format("Error code = %d", error));
+        this.errorCode = error;
+    }
+
+    public PDRuntimeException(int error, String msg) {
+        super(msg);
+        this.errorCode = error;
+    }
+
+    public PDRuntimeException(int error, Throwable e) {
+        super(e);
+        this.errorCode = error;
+    }
+
+    public PDRuntimeException(int error, String msg, Throwable e) {
+        super(msg, e);
+        this.errorCode = error;
+    }
+
+    public int getErrorCode() {
+        return errorCode;
+    }
+}

hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionCache.java

@@ -0,0 +1,460 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.common;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hugegraph.pd.grpc.Metapb;
+
+import com.google.common.collect.Range;
+import com.google.common.collect.RangeMap;
+import com.google.common.collect.TreeRangeMap;
+
+/**
+ * 放弃 copy on write 的方式
+ * 1. 在 graph * partition 数量极多的时候，效率严重下降，不能用
+ */
+public class PartitionCache {
+
+    // 读写锁对象
+    private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+    Lock writeLock = readWriteLock.writeLock();
+    // 每张图一个缓存
+    private volatile Map<String, RangeMap<Long, Integer>> keyToPartIdCache;
+    // graphName + PartitionID 组成 key
+    private volatile Map<String, Map<Integer, Metapb.Partition>> partitionCache;
+
+    private volatile Map<Integer, Metapb.ShardGroup> shardGroupCache;
+    private volatile Map<Long, Metapb.Store> storeCache;
+    private volatile Map<String, Metapb.Graph> graphCache;
+
+    private final Map<String, AtomicBoolean> locks = new HashMap<>();
+
+    public PartitionCache() {
+        keyToPartIdCache = new HashMap<>();
+        partitionCache = new HashMap<>();
+        shardGroupCache = new ConcurrentHashMap<>();
+        storeCache = new ConcurrentHashMap<>();
+        graphCache = new ConcurrentHashMap<>();
+    }
+
+    private AtomicBoolean getOrCreateGraphLock(String graphName) {
+        var lock = this.locks.get(graphName);
+        if (lock == null) {
+            try {
+                writeLock.lock();
+                if ((lock = this.locks.get(graphName)) == null) {
+                    lock = new AtomicBoolean();
+                    locks.put(graphName, lock);
+                }
+            } finally {
+                writeLock.unlock();
+            }
+        }
+        return lock;
+    }
+
+    public void waitGraphLock(String graphName) {
+        var lock = getOrCreateGraphLock(graphName);
+        while (lock.get()) {
+            Thread.onSpinWait();
+        }
+    }
+
+    public void lockGraph(String graphName) {
+        var lock = getOrCreateGraphLock(graphName);
+        while (lock.compareAndSet(false, true)) {
+            Thread.onSpinWait();
+        }
+    }
+
+    public void unlockGraph(String graphName) {
+        var lock = getOrCreateGraphLock(graphName);
+        lock.set(false);
+    }
+
+    /**
+     * 根据 partitionId 返回分区信息
+     *
+     * @param graphName
+     * @param partId
+     * @return
+     */
+    public KVPair<Metapb.Partition, Metapb.Shard> getPartitionById(String graphName, int partId) {
+        waitGraphLock(graphName);
+        var graphs = partitionCache.get(graphName);
+        if (graphs != null) {
+            var partition = graphs.get(partId);
+            if (partition != null) {
+                return new KVPair<>(partition, getLeaderShard(partId));
+            }
+        }
+
+        return null;
+    }
+
+    /**
+     * 返回 key 所在的分区信息
+     *
+     * @param key
+     * @return
+     */
+    public KVPair<Metapb.Partition, Metapb.Shard> getPartitionByKey(String graphName, byte[] key) {
+        int code = PartitionUtils.calcHashcode(key);
+        return getPartitionByCode(graphName, code);
+    }
+
+    /**
+     * 根据 key 的 hashcode 返回分区信息
+     *
+     * @param graphName
+     * @param code
+     * @return
+     */
+    public KVPair<Metapb.Partition, Metapb.Shard> getPartitionByCode(String graphName, long code) {
+        waitGraphLock(graphName);
+        RangeMap<Long, Integer> rangeMap = keyToPartIdCache.get(graphName);
+        if (rangeMap != null) {
+            Integer partId = rangeMap.get(code);
+            if (partId != null) {
+                return getPartitionById(graphName, partId);
+            }
+        }
+        return null;
+    }
+
+    public List<Metapb.Partition> getPartitions(String graphName) {
+        waitGraphLock(graphName);
+
+        List<Metapb.Partition> partitions = new ArrayList<>();
+        if (!partitionCache.containsKey(graphName)) {
+            return partitions;
+        }
+        partitionCache.get(graphName).forEach((k, v) -> {
+            partitions.add(v);
+        });
+
+        return partitions;
+    }
+
+    public boolean addPartition(String graphName, int partId, Metapb.Partition partition) {
+        waitGraphLock(graphName);
+        Metapb.Partition old = null;
+
+        if (partitionCache.containsKey(graphName)) {
+            old = partitionCache.get(graphName).get(partId);
+        }
+
+        if (old != null && old.equals(partition)) {
+            return false;
+        }
+        try {
+
+            lockGraph(graphName);
+
+            partitionCache.computeIfAbsent(graphName, k -> new HashMap<>()).put(partId, partition);
+
+            if (old != null) {
+                // old [1-3) 被 [2-3) 覆盖了。当 [1-3) 变成 [1-2) 不应该删除原先的 [1-3)
+                // 当确认老的 start, end 都是自己的时候，才可以删除老的。(即还没覆盖）
+                var graphRange = keyToPartIdCache.get(graphName);
+                if (Objects.equals(partition.getId(), graphRange.get(partition.getStartKey())) &&
+                    Objects.equals(partition.getId(), graphRange.get(partition.getEndKey() - 1))) {
+                    graphRange.remove(graphRange.getEntry(partition.getStartKey()).getKey());
+                }
+            }
+
+            keyToPartIdCache.computeIfAbsent(graphName, k -> TreeRangeMap.create())
+                            .put(Range.closedOpen(partition.getStartKey(),
+                                                  partition.getEndKey()), partId);
+        } finally {
+            unlockGraph(graphName);
+        }
+        return true;
+    }
+
+    public void updatePartition(String graphName, int partId, Metapb.Partition partition) {
+        try {
+            lockGraph(graphName);
+            Metapb.Partition old = null;
+            var graphs = partitionCache.get(graphName);
+            if (graphs != null) {
+                old = graphs.get(partId);
+            }
+
+            if (old != null) {
+                var graphRange = keyToPartIdCache.get(graphName);
+                if (Objects.equals(partition.getId(), graphRange.get(partition.getStartKey())) &&
+                    Objects.equals(partition.getId(), graphRange.get(partition.getEndKey() - 1))) {
+                    graphRange.remove(graphRange.getEntry(partition.getStartKey()).getKey());
+                }
+            }
+
+            partitionCache.computeIfAbsent(graphName, k -> new HashMap<>()).put(partId, partition);
+            keyToPartIdCache.computeIfAbsent(graphName, k -> TreeRangeMap.create())
+                            .put(Range.closedOpen(partition.getStartKey(), partition.getEndKey()),
+                                 partId);
+        } finally {
+            unlockGraph(graphName);
+        }
+    }
+
+    public boolean updatePartition(Metapb.Partition partition) {
+
+        var graphName = partition.getGraphName();
+        var partitionId = partition.getId();
+
+        var old = getPartitionById(graphName, partitionId);
+        if (old != null && Objects.equals(partition, old.getKey())) {
+            return false;
+        }
+
+        updatePartition(graphName, partitionId, partition);
+        return true;
+    }
+
+    public void removePartition(String graphName, int partId) {
+        try {
+            lockGraph(graphName);
+            var partition = partitionCache.get(graphName).remove(partId);
+            if (partition != null) {
+                var graphRange = keyToPartIdCache.get(graphName);
+
+                if (Objects.equals(partition.getId(), graphRange.get(partition.getStartKey())) &&
+                    Objects.equals(partition.getId(), graphRange.get(partition.getEndKey() - 1))) {
+                    graphRange.remove(graphRange.getEntry(partition.getStartKey()).getKey());
+                }
+            }
+        } finally {
+            unlockGraph(graphName);
+        }
+    }
+
+    /**
+     * remove partition id of graph name
+     *
+     * @param graphName
+     * @param id
+     */
+    public void remove(String graphName, int id) {
+        removePartition(graphName, id);
+    }
+
+    /**
+     * remove all partitions
+     */
+    public void removePartitions() {
+        writeLock.lock();
+        try {
+            partitionCache = new HashMap<>();
+            keyToPartIdCache = new HashMap<>();
+            locks.clear();
+        } finally {
+            writeLock.unlock();
+        }
+    }
+
+    /**
+     * remove partition cache of graphName
+     *
+     * @param graphName
+     */
+    public void removeAll(String graphName) {
+        try {
+            lockGraph(graphName);
+            partitionCache.remove(graphName);
+            keyToPartIdCache.remove(graphName);
+            locks.remove(graphName);
+        } finally {
+            unlockGraph(graphName);
+        }
+    }
+
+    private String makePartitionKey(String graphName, int partId) {
+        return graphName + "/" + partId;
+    }
+
+    public boolean updateShardGroup(Metapb.ShardGroup shardGroup) {
+        Metapb.ShardGroup oldShardGroup = shardGroupCache.get(shardGroup.getId());
+        if (oldShardGroup != null && oldShardGroup.equals(shardGroup)) {
+            return false;
+        }
+        shardGroupCache.put(shardGroup.getId(), shardGroup);
+        return true;
+    }
+
+    public void deleteShardGroup(int shardGroupId) {
+        shardGroupCache.remove(shardGroupId);
+    }
+
+    public Metapb.ShardGroup getShardGroup(int groupId) {
+        return shardGroupCache.get(groupId);
+    }
+
+    public boolean addStore(Long storeId, Metapb.Store store) {
+        Metapb.Store oldStore = storeCache.get(storeId);
+        if (oldStore != null && oldStore.equals(store)) {
+            return false;
+        }
+        storeCache.put(storeId, store);
+        return true;
+    }
+
+    public Metapb.Store getStoreById(Long storeId) {
+        return storeCache.get(storeId);
+    }
+
+    public void removeStore(Long storeId) {
+        storeCache.remove(storeId);
+    }
+
+    public boolean hasGraph(String graphName) {
+        return getPartitions(graphName).size() > 0;
+    }
+
+    public void updateGraph(Metapb.Graph graph) {
+        if (Objects.equals(graph, getGraph(graph.getGraphName()))) {
+            return;
+        }
+        graphCache.put(graph.getGraphName(), graph);
+    }
+
+    public Metapb.Graph getGraph(String graphName) {
+        return graphCache.get(graphName);
+    }
+
+    public List<Metapb.Graph> getGraphs() {
+        List<Metapb.Graph> graphs = new ArrayList<>();
+        graphCache.forEach((k, v) -> {
+            graphs.add(v);
+        });
+        return graphs;
+    }
+
+    public void reset() {
+        writeLock.lock();
+        try {
+            partitionCache = new HashMap<>();
+            keyToPartIdCache = new HashMap<>();
+            shardGroupCache = new ConcurrentHashMap<>();
+            storeCache = new ConcurrentHashMap<>();
+            graphCache = new ConcurrentHashMap<>();
+            locks.clear();
+        } finally {
+            writeLock.unlock();
+        }
+    }
+
+    public void clear() {
+        reset();
+    }
+
+    public String debugCacheByGraphName(String graphName) {
+        StringBuilder builder = new StringBuilder();
+        builder.append("Graph:").append(graphName).append(", cache info: range info: {");
+        var rangeMap = keyToPartIdCache.get(graphName);
+        builder.append(rangeMap == null ? "" : rangeMap).append("}");
+
+        if (rangeMap != null) {
+            builder.append(", partition info : {");
+            rangeMap.asMapOfRanges().forEach((k, v) -> {
+                var partition = partitionCache.get(graphName).get(v);
+                builder.append("[part_id:").append(v);
+                if (partition != null) {
+                    builder.append(", start_key:").append(partition.getStartKey())
+                           .append(", end_key:").append(partition.getEndKey())
+                           .append(", state:").append(partition.getState().name());
+                }
+                builder.append("], ");
+            });
+            builder.append("}");
+        }
+
+        builder.append(", graph info:{");
+        var graph = graphCache.get(graphName);
+        if (graph != null) {
+            builder.append("partition_count:").append(graph.getPartitionCount())
+                   .append(", state:").append(graph.getState().name());
+        }
+        builder.append("}]");
+        return builder.toString();
+    }
+
+    public Metapb.Shard getLeaderShard(int partitionId) {
+        var shardGroup = shardGroupCache.get(partitionId);
+        if (shardGroup != null) {
+            for (Metapb.Shard shard : shardGroup.getShardsList()) {
+                if (shard.getRole() == Metapb.ShardRole.Leader) {
+                    return shard;
+                }
+            }
+        }
+
+        return null;
+    }
+
+    public void updateShardGroupLeader(int partitionId, Metapb.Shard leader) {
+        if (shardGroupCache.containsKey(partitionId) && leader != null) {
+            if (!Objects.equals(getLeaderShard(partitionId), leader)) {
+                var shardGroup = shardGroupCache.get(partitionId);
+                var builder = Metapb.ShardGroup.newBuilder(shardGroup).clearShards();
+                for (var shard : shardGroup.getShardsList()) {
+                    builder.addShards(
+                            Metapb.Shard.newBuilder()
+                                        .setStoreId(shard.getStoreId())
+                                        .setRole(shard.getStoreId() == leader.getStoreId() ?
+                                                 Metapb.ShardRole.Leader :
+                                                 Metapb.ShardRole.Follower)
+                                        .build()
+                    );
+                }
+                shardGroupCache.put(partitionId, builder.build());
+            }
+        }
+    }
+
+    public String debugShardGroup() {
+        StringBuilder builder = new StringBuilder();
+        builder.append("shard group cache:{");
+        shardGroupCache.forEach((partitionId, shardGroup) -> {
+            builder.append(partitionId).append("::{")
+                   .append("version:").append(shardGroup.getVersion())
+                   .append(", conf_version:").append(shardGroup.getConfVer())
+                   .append(", state:").append(shardGroup.getState().name())
+                   .append(", shards:[");
+
+            for (var shard : shardGroup.getShardsList()) {
+                builder.append("{store_id:").append(shard.getStoreId())
+                       .append(", role:").append(shard.getRole().name())
+                       .append("},");
+            }
+            builder.append("], ");
+        });
+        builder.append("}");
+        return builder.toString();
+    }
+}

hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionUtils.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.common;
+
+public class PartitionUtils {
+
+    public static final int MAX_VALUE = 0xffff;
+
+    /**
+     * 计算key的hashcode
+     *
+     * @param key
+     * @return hashcode
+     */
+    public static int calcHashcode(byte[] key) {
+        final int p = 16777619;
+        int hash = (int) 2166136261L;
+        for (byte element : key) {
+            hash = (hash ^ element) * p;
+        }
+        hash += hash << 13;
+        hash ^= hash >> 7;
+        hash += hash << 3;
+        hash ^= hash >> 17;
+        hash += hash << 5;
+        hash = hash & PartitionUtils.MAX_VALUE;
+        if (hash == PartitionUtils.MAX_VALUE) {
+            hash = PartitionUtils.MAX_VALUE - 1;
+        }
+        return hash;
+    }
+}

hugegraph-pd/hg-pd-core/pom.xml

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements. See the NOTICE file distributed with this
+  work for additional information regarding copyright ownership. The ASF
+  licenses this file to You under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  License for the specific language governing permissions and limitations
+  under the License.
+  -->
+
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.apache.hugegraph</groupId>
+        <artifactId>hugegraph-pd</artifactId>
+        <version>${revision}</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>hg-pd-core</artifactId>
+
+    <properties>
+        <jetcd-version>0.5.10</jetcd-version>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>com.alipay.sofa</groupId>
+            <artifactId>jraft-core</artifactId>
+           <!-- TODO: use open source version & adopt the code later -->
+            <version>1.3.13</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.rocksdb</groupId>
+                    <artifactId>rocksdbjni</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.rocksdb</groupId>
+            <artifactId>rocksdbjni</artifactId>
+            <version>6.29.5</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hugegraph</groupId>
+            <artifactId>hg-pd-grpc</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context</artifactId>
+            <version>5.3.20</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hugegraph</groupId>
+            <artifactId>hg-pd-common</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot</artifactId>
+            <version>2.5.14</version>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.24</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.12.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>2.8.9</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hugegraph</groupId>
+            <artifactId>hg-store-grpc</artifactId>
+            <version>${revision}</version>
+        </dependency>
+    </dependencies>
+</project>

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/ConfigService.java

@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import java.util.List;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.apache.hugegraph.pd.meta.ConfigMetaStore;
+import org.apache.hugegraph.pd.meta.MetadataFactory;
+import org.apache.hugegraph.pd.raft.RaftStateListener;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class ConfigService implements RaftStateListener {
+
+    private final ConfigMetaStore meta;
+    private PDConfig pdConfig;
+
+    public ConfigService(PDConfig config) {
+        this.pdConfig = config;
+        config.setConfigService(this);
+        meta = MetadataFactory.newConfigMeta(config);
+    }
+
+
+    public Metapb.PDConfig getPDConfig(long version) throws PDException {
+        return this.meta.getPdConfig(version);
+    }
+
+    public Metapb.PDConfig getPDConfig() throws PDException {
+        return this.meta.getPdConfig(0);
+    }
+
+    public Metapb.PDConfig setPDConfig(Metapb.PDConfig mConfig) throws PDException {
+        Metapb.PDConfig oldCfg = getPDConfig();
+        Metapb.PDConfig.Builder builder = oldCfg.toBuilder().mergeFrom(mConfig)
+                                                .setVersion(oldCfg.getVersion() + 1)
+                                                .setTimestamp(System.currentTimeMillis());
+        mConfig = this.meta.setPdConfig(builder.build());
+        log.info("PDConfig has been modified, new PDConfig is {}", mConfig);
+        updatePDConfig(mConfig);
+        return mConfig;
+    }
+
+    public List<Metapb.GraphSpace> getGraphSpace(String graphSpaceName) throws PDException {
+        return this.meta.getGraphSpace(graphSpaceName);
+    }
+
+    public Metapb.GraphSpace setGraphSpace(Metapb.GraphSpace graphSpace) throws PDException {
+        return this.meta.setGraphSpace(graphSpace.toBuilder()
+                                                 .setTimestamp(System.currentTimeMillis())
+                                                 .build());
+    }
+
+    /**
+     * 从存储中读取配置项，并覆盖全局的PDConfig对象
+     *
+     * @return
+     */
+    public PDConfig loadConfig() {
+        try {
+            Metapb.PDConfig mConfig = this.meta.getPdConfig(0);
+            if (mConfig == null) {
+                mConfig = Metapb.PDConfig.newBuilder()
+                                         .setPartitionCount(pdConfig.getInitialPartitionCount())
+                                         .setShardCount(pdConfig.getPartition().getShardCount())
+                                         .setVersion(1)
+                                         .setTimestamp(System.currentTimeMillis())
+                                         .setMaxShardsPerStore(
+                                                 pdConfig.getPartition().getMaxShardsPerStore())
+                                         .build();
+                this.meta.setPdConfig(mConfig);
+            }
+            pdConfig = updatePDConfig(mConfig);
+        } catch (Exception e) {
+            log.error("ConfigService loadConfig exception {}", e);
+        }
+        return pdConfig;
+    }
+
+    public synchronized PDConfig updatePDConfig(Metapb.PDConfig mConfig) {
+        log.info("update pd config: mConfig:{}", mConfig);
+        pdConfig.getPartition().setShardCount(mConfig.getShardCount());
+        pdConfig.getPartition().setTotalCount(mConfig.getPartitionCount());
+        pdConfig.getPartition().setMaxShardsPerStore(mConfig.getMaxShardsPerStore());
+        return pdConfig;
+    }
+
+    public synchronized PDConfig setPartitionCount(int count) {
+        Metapb.PDConfig mConfig = null;
+        try {
+            mConfig = getPDConfig();
+            mConfig = mConfig.toBuilder().setPartitionCount(count).build();
+            setPDConfig(mConfig);
+        } catch (PDException e) {
+            log.error("ConfigService exception {}", e);
+            e.printStackTrace();
+        }
+        return pdConfig;
+    }
+
+    /**
+     * meta store中的数量
+     * 由于可能会受分区分裂/合并的影响，原始的partition count不推荐使用
+     *
+     * @return partition count of cluster
+     * @throws PDException when io error
+     */
+    public int getPartitionCount() throws PDException {
+        return getPDConfig().getPartitionCount();
+    }
+
+    @Override
+    public void onRaftLeaderChanged() {
+        loadConfig();
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/IdService.java

@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.meta.IdMetaStore;
+import org.apache.hugegraph.pd.meta.MetadataFactory;
+
+public class IdService {
+
+    private final IdMetaStore meta;
+    private PDConfig pdConfig;
+
+    public IdService(PDConfig config) {
+        this.pdConfig = config;
+        meta = MetadataFactory.newHugeServerMeta(config);
+    }
+
+    public PDConfig getPdConfig() {
+        return pdConfig;
+    }
+
+    public void setPdConfig(PDConfig pdConfig) {
+        this.pdConfig = pdConfig;
+    }
+
+    public long getId(String key, int delta) throws PDException {
+        return meta.getId(key, delta);
+    }
+
+    public void resetId(String key) throws PDException {
+        meta.resetId(key);
+    }
+
+    /**
+     * 获取自增循环不重复id, 达到上限后从0开始自增.自动跳过正在使用的cid
+     *
+     * @param key
+     * @param max
+     * @return
+     * @throws PDException
+     */
+    public long getCId(String key, long max) throws PDException {
+        return meta.getCId(key, max);
+    }
+
+    public long getCId(String key, String name, long max) throws PDException {
+        return meta.getCId(key, name, max);
+    }
+
+    /**
+     * 删除一个自增循环id
+     *
+     * @param key
+     * @param cid
+     * @return
+     * @throws PDException
+     */
+    public long delCId(String key, long cid) throws PDException {
+        return meta.delCId(key, cid);
+    }
+
+    public long delCIdDelay(String key, String name, long cid) throws PDException {
+        return meta.delCIdDelay(key, name, cid);
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/KvService.java

@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.kv.Kv;
+import org.apache.hugegraph.pd.grpc.kv.V;
+import org.apache.hugegraph.pd.meta.MetadataKeyHelper;
+import org.apache.hugegraph.pd.meta.MetadataRocksDBStore;
+import org.apache.hugegraph.pd.store.KV;
+import org.springframework.stereotype.Service;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ *
+ **/
+@Slf4j
+@Service
+public class KvService {
+
+
+    public static final char KV_DELIMITER = '@';
+    // TODO 主前缀之后，增加类名做区分
+    private static final String TTL_PREFIX = "T";
+    private static final String KV_PREFIX = "K";
+    private static final String LOCK_PREFIX = "L";
+    private static final String KV_PREFIX_DELIMITER = KV_PREFIX + KV_DELIMITER;
+    private static final byte[] EMPTY_VALUE = new byte[0];
+    private final MetadataRocksDBStore meta;
+    private PDConfig pdConfig;
+
+    public KvService(PDConfig config) {
+        this.pdConfig = config;
+        meta = new MetadataRocksDBStore(config);
+    }
+
+    public static String getKey(Object... keys) {
+        StringBuilder builder = MetadataKeyHelper.getStringBuilderHelper();
+        builder.append(KV_PREFIX).append(KV_DELIMITER);
+        for (Object key : keys) {
+            builder.append(key == null ? "" : key).append(KV_DELIMITER);
+        }
+        return builder.substring(0, builder.length() - 1);
+    }
+
+    public static byte[] getKeyBytes(Object... keys) {
+        String key = getKey(keys);
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static String getKeyWithoutPrefix(Object... keys) {
+        StringBuilder builder = MetadataKeyHelper.getStringBuilderHelper();
+        for (Object key : keys) {
+            builder.append(key == null ? "" : key).append(KV_DELIMITER);
+        }
+        return builder.substring(0, builder.length() - 1);
+    }
+
+    public static String getDelimiter() {
+        return String.valueOf(KV_DELIMITER);
+    }
+
+    public PDConfig getPdConfig() {
+        return pdConfig;
+    }
+
+    public void setPdConfig(PDConfig pdConfig) {
+        this.pdConfig = pdConfig;
+    }
+
+    public void put(String key, String value) throws PDException {
+        V storeValue = V.newBuilder().setValue(value).setTtl(0).build();
+        meta.put(getStoreKey(key), storeValue.toByteArray());
+        // log.warn("add key with key-{}:value-{}", key, value);
+    }
+
+    public void put(String key, String value, long ttl) throws PDException {
+        long curTime = System.currentTimeMillis();
+        curTime += ttl;
+        V storeValue = V.newBuilder().setValue(value).setSt(ttl).setTtl(curTime).build();
+        meta.put(getStoreKey(key), storeValue.toByteArray());
+        meta.put(getTTLStoreKey(key, curTime), EMPTY_VALUE);
+        // log.warn("add key with key-{}:value-{}:ttl-{}", key, value, ttl);
+    }
+
+    public String get(String key) throws PDException {
+        byte[] storeKey = getStoreKey(key);
+        return get(storeKey);
+    }
+
+    public String get(byte[] keyBytes) throws PDException {
+        byte[] bytes = meta.getOne(keyBytes);
+        String v = getValue(keyBytes, bytes);
+        return v;
+    }
+
+    private String getValue(byte[] keyBytes, byte[] valueBytes) throws PDException {
+        if (valueBytes == null || valueBytes.length == 0) {
+            return "";
+        }
+        try {
+            V v = V.parseFrom(valueBytes);
+            if (v.getTtl() == 0 || v.getTtl() >= System.currentTimeMillis()) {
+                return v.getValue();
+            } else {
+                meta.remove(keyBytes);
+                meta.remove(getTTLStoreKey(new String(keyBytes), v.getTtl()));
+            }
+        } catch (Exception e) {
+            log.error("parse value with error:{}", e.getMessage());
+            throw new PDException(-1, e.getMessage());
+        }
+        return null;
+    }
+
+    public boolean keepAlive(String key) throws PDException {
+        byte[] bytes = meta.getOne(getStoreKey(key));
+        try {
+            if (bytes == null || bytes.length == 0) {
+                return false;
+            }
+            V v = V.parseFrom(bytes);
+            if (v != null) {
+                long ttl = v.getTtl();
+                long st = v.getSt();
+                meta.remove(getTTLStoreKey(key, ttl));
+                put(key, v.getValue(), st);
+                return true;
+            } else {
+                return false;
+            }
+        } catch (InvalidProtocolBufferException e) {
+            throw new PDException(-1, e.getMessage());
+        }
+    }
+
+    public Kv delete(String key) throws PDException {
+        byte[] storeKey = getStoreKey(key);
+        String value = this.get(storeKey);
+        meta.remove(storeKey);
+        Kv.Builder builder = Kv.newBuilder().setKey(key);
+        if (value != null) {
+            builder.setValue(value);
+        }
+        Kv kv = builder.build();
+        // log.warn("delete kv with key :{}", key);
+        return kv;
+    }
+
+    public List<Kv> deleteWithPrefix(String key) throws PDException {
+        byte[] storeKey = getStoreKey(key);
+        //TODO to many rows for scan
+        List<KV> kvList = meta.scanPrefix(storeKey);
+        LinkedList<Kv> kvs = new LinkedList<>();
+        for (KV kv : kvList) {
+            String kvKey = new String(kv.getKey()).replaceFirst(KV_PREFIX_DELIMITER, "");
+            String kvValue = getValue(kv.getKey(), kv.getValue());
+            if (kvValue != null) {
+                kvs.add(Kv.newBuilder().setKey(kvKey).setValue(kvValue).build());
+            }
+        }
+        meta.removeByPrefix(storeKey);
+        // log.warn("delete kv with key prefix :{}", key);
+        return kvs;
+    }
+
+    /**
+     * scan result ranged from key start and key end
+     *
+     * @param keyStart
+     * @param keyEnd
+     * @return Records
+     * @throws PDException
+     */
+    public Map<String, String> scanRange(String keyStart, String keyEnd) throws PDException {
+        List<KV> list = meta.scanRange(getStoreKey(keyStart), getStoreKey(keyEnd));
+        Map<String, String> map = new HashMap<>();
+        for (KV kv : list) {
+            String kvKey = new String(kv.getKey()).replaceFirst(KV_PREFIX_DELIMITER, "");
+            String kvValue = getValue(kv.getKey(), kv.getValue());
+            if (kvValue != null) {
+                map.put(kvKey, kvValue);
+            }
+        }
+        return map;
+    }
+
+    public Map<String, String> scanWithPrefix(String key) throws PDException {
+        List<KV> kvList = meta.scanPrefix(getStoreKey(key));
+        HashMap<String, String> map = new HashMap<>();
+        for (KV kv : kvList) {
+            String kvKey = new String(kv.getKey()).replaceFirst(KV_PREFIX_DELIMITER, "");
+            String kvValue = getValue(kv.getKey(), kv.getValue());
+            if (kvValue != null) {
+                map.put(kvKey, kvValue);
+            }
+        }
+        return map;
+    }
+
+    public boolean locked(String key) throws PDException {
+        String lockKey = KvService.getKeyWithoutPrefix(KvService.LOCK_PREFIX, key);
+        Map<String, String> allLock = scanWithPrefix(lockKey);
+        return allLock != null && allLock.size() != 0;
+    }
+
+    private boolean owned(String key, long clientId) throws PDException {
+        String lockKey = KvService.getKeyWithoutPrefix(KvService.LOCK_PREFIX, key);
+        Map<String, String> allLock = scanWithPrefix(lockKey);
+        if (allLock.size() == 0) {
+            return true;
+        }
+        for (Map.Entry<String, String> entry : allLock.entrySet()) {
+            String entryKey = entry.getKey();
+            String[] split = entryKey.split(String.valueOf(KV_DELIMITER));
+            if (Long.valueOf(split[split.length - 1]).equals(clientId)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public boolean lock(String key, long ttl, long clientId) throws PDException {
+        //TODO lock improvement
+        synchronized (KvService.class) {
+            if (!owned(key, clientId)) {
+                return false;
+            }
+            put(getLockKey(key, clientId), " ", ttl);
+            return true;
+        }
+    }
+
+    public boolean lockWithoutReentrant(String key, long ttl,
+                                        long clientId) throws PDException {
+        synchronized (KvService.class) {
+            if (locked(key)) {
+                return false;
+            }
+            put(getLockKey(key, clientId), " ", ttl);
+            return true;
+        }
+    }
+
+    public boolean unlock(String key, long clientId) throws PDException {
+        synchronized (KvService.class) {
+            if (!owned(key, clientId)) {
+                return false;
+            }
+            delete(getLockKey(key, clientId));
+            return true;
+        }
+    }
+
+    public boolean keepAlive(String key, long clientId) throws PDException {
+        String lockKey = getLockKey(key, clientId);
+        return keepAlive(lockKey);
+    }
+
+    public String getLockKey(String key, long clientId) {
+        return getKeyWithoutPrefix(LOCK_PREFIX, key, clientId);
+    }
+
+    public byte[] getStoreKey(String key) {
+        return getKeyBytes(key);
+    }
+
+    public byte[] getTTLStoreKey(String key, long time) {
+        return getKeyBytes(TTL_PREFIX, time, key);
+    }
+
+    public void clearTTLData() {
+        try {
+            byte[] ttlStartKey = getTTLStoreKey("", 0);
+            byte[] ttlEndKey = getTTLStoreKey("", System.currentTimeMillis());
+            List<KV> kvList = meta.scanRange(ttlStartKey, ttlEndKey);
+            for (KV kv : kvList) {
+                String key = new String(kv.getKey());
+                int index = key.indexOf(KV_DELIMITER, 2);
+                String delKey = key.substring(index + 1);
+                delete(delKey);
+                meta.remove(kv.getKey());
+            }
+        } catch (Exception e) {
+            log.error("clear ttl data with error :", e);
+        }
+    }
+
+    public MetadataRocksDBStore getMeta() {
+        return meta;
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/LogService.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import java.util.List;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.apache.hugegraph.pd.meta.LogMeta;
+import org.apache.hugegraph.pd.meta.MetadataFactory;
+import org.springframework.stereotype.Service;
+
+import com.google.protobuf.Any;
+import com.google.protobuf.GeneratedMessageV3;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class LogService {
+
+    public static final String GRPC = "GRPC";
+    public static final String REST = "REST";
+    public static final String TASK = "TASK";
+    public static final String NODE_CHANGE = "NODE_CHANGE";
+    public static final String PARTITION_CHANGE = "PARTITION_CHANGE";
+    private final LogMeta logMeta;
+
+    public LogService(PDConfig pdConfig) {
+        logMeta = MetadataFactory.newLogMeta(pdConfig);
+    }
+
+    public List<Metapb.LogRecord> getLog(String action, Long start, Long end) throws PDException {
+        return logMeta.getLog(action, start, end);
+    }
+
+    public void insertLog(String action, String message, GeneratedMessageV3 target) {
+        try {
+            Metapb.LogRecord logRecord = Metapb.LogRecord.newBuilder()
+                                                         .setAction(action)
+                                                         .setMessage(message)
+                                                         .setTimestamp(System.currentTimeMillis())
+                                                         .setObject(Any.pack(target))
+                                                         .build();
+            logMeta.insertLog(logRecord);
+        } catch (PDException e) {
+            log.debug("Insert log with error:{}", e);
+        }
+
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/PartitionInstructionListener.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.apache.hugegraph.pd.grpc.pulse.ChangeShard;
+import org.apache.hugegraph.pd.grpc.pulse.CleanPartition;
+import org.apache.hugegraph.pd.grpc.pulse.DbCompaction;
+import org.apache.hugegraph.pd.grpc.pulse.MovePartition;
+import org.apache.hugegraph.pd.grpc.pulse.PartitionKeyRange;
+import org.apache.hugegraph.pd.grpc.pulse.SplitPartition;
+import org.apache.hugegraph.pd.grpc.pulse.TransferLeader;
+
+/**
+ * 分区命令监听
+ */
+public interface PartitionInstructionListener {
+    void changeShard(Metapb.Partition partition, ChangeShard changeShard) throws PDException;
+
+    void transferLeader(Metapb.Partition partition, TransferLeader transferLeader) throws
+                                                                                   PDException;
+
+    void splitPartition(Metapb.Partition partition, SplitPartition splitPartition) throws
+                                                                                   PDException;
+
+    void dbCompaction(Metapb.Partition partition, DbCompaction dbCompaction) throws PDException;
+
+    void movePartition(Metapb.Partition partition, MovePartition movePartition) throws PDException;
+
+    void cleanPartition(Metapb.Partition partition, CleanPartition cleanPartition) throws
+                                                                                   PDException;
+
+    void changePartitionKeyRange(Metapb.Partition partition,
+                                 PartitionKeyRange partitionKeyRange) throws PDException;
+
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/PartitionStatusListener.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import org.apache.hugegraph.pd.grpc.Metapb;
+
+/**
+ * 分区状态监听
+ */
+public interface PartitionStatusListener {
+    void onPartitionChanged(Metapb.Partition partition, Metapb.Partition newPartition);
+
+    void onPartitionRemoved(Metapb.Partition partition);
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/RegistryService.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.discovery.NodeInfo;
+import org.apache.hugegraph.pd.grpc.discovery.NodeInfos;
+import org.apache.hugegraph.pd.grpc.discovery.Query;
+import org.apache.hugegraph.pd.meta.DiscoveryMetaStore;
+import org.apache.hugegraph.pd.meta.MetadataFactory;
+
+public class RegistryService {
+    private final PDConfig pdConfig;
+    private final DiscoveryMetaStore meta;
+
+    public RegistryService(PDConfig config) {
+        this.pdConfig = config;
+        meta = MetadataFactory.newDiscoveryMeta(config);
+    }
+
+    public void register(NodeInfo nodeInfo, int outTimes) throws PDException {
+        meta.register(nodeInfo, outTimes);
+    }
+
+    public NodeInfos getNodes(Query query) {
+        return meta.getNodes(query);
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/ShardGroupStatusListener.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import org.apache.hugegraph.pd.grpc.Metapb;
+
+public interface ShardGroupStatusListener {
+    void onShardListChanged(Metapb.ShardGroup shardGroup, Metapb.ShardGroup newShardGroup);
+
+    void onShardListOp(Metapb.ShardGroup shardGroup);
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreMonitorDataService.java

@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.apache.hugegraph.pd.meta.MetadataKeyHelper;
+import org.springframework.stereotype.Service;
+
+import lombok.extern.slf4j.Slf4j;
+
+
+@Slf4j
+@Service
+public class StoreMonitorDataService {
+    private static final String MONITOR_DATA_PREFIX = "SMD";
+    private final PDConfig pdConfig;
+    private final KvService kvService;
+    /**
+     * the last timestamp of the store monitor data,
+     * used for determine the gap of store's heartbeat.
+     */
+    private final Map<Long, Long> lastStoreStateTimestamp;
+
+
+    public StoreMonitorDataService(PDConfig pdConfig) {
+        this.pdConfig = pdConfig;
+        this.kvService = new KvService(pdConfig);
+        this.lastStoreStateTimestamp = new HashMap<>();
+    }
+
+    /**
+     * save the store stats
+     *
+     * @param storeStats
+     */
+    public void saveMonitorData(Metapb.StoreStats storeStats) throws PDException {
+        long storeId = storeStats.getStoreId();
+        /**
+         * load the latest store timestamp when start up or alter leader
+         */
+        if (!lastStoreStateTimestamp.containsKey(storeId)) {
+            long lastTimestamp = getLatestStoreMonitorDataTimeStamp(storeId);
+            log.debug("store id : {}, last timestamp :{}", storeId, lastTimestamp);
+            lastStoreStateTimestamp.put(storeId, lastTimestamp);
+        }
+
+        long current = System.currentTimeMillis() / 1000;
+        long interval = this.pdConfig.getStore().getMonitorInterval();
+
+        // exceed the interval
+        if (current - lastStoreStateTimestamp.getOrDefault(storeId, 0L) >= interval) {
+            saveMonitorDataToDb(storeStats, current);
+            log.debug("store id: {}, system info:{}", storeId,
+                      debugMonitorInfo(storeStats.getSystemMetricsList()));
+            lastStoreStateTimestamp.put(storeId, current);
+        }
+    }
+
+    /**
+     * save the snapshot of store status
+     *
+     * @param storeStats store status
+     * @param ts,        timestamp
+     * @return store status
+     * @throws PDException
+     */
+    private void saveMonitorDataToDb(Metapb.StoreStats storeStats, long ts) throws PDException {
+        String key = getMonitorDataKey(storeStats.getStoreId(), ts);
+        log.debug("store id: {}, save monitor data info, ts:{}, my key:{}", storeStats.getStoreId(),
+                  ts, key);
+        kvService.put(key, extractMetricsFromStoreStatus(storeStats));
+    }
+
+    public String debugMonitorInfo(List<Metapb.RecordPair> systemInfo) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("[");
+        for (Metapb.RecordPair pair : systemInfo) {
+            sb.append(pair.getKey());
+            sb.append(":");
+            sb.append(pair.getValue());
+            sb.append(",");
+        }
+        sb.append("]");
+        return sb.toString();
+    }
+
+    /**
+     * get the historical monitor data by store id, by range(start, end)
+     *
+     * @param storeId store id
+     * @param start   range start
+     * @param end     range end
+     * @return list of store stats
+     */
+    public Map<String, String> getStoreMonitorData(long storeId, long start, long end) throws
+                                                                                       PDException {
+        log.debug("get monitor data, store id:{}, start{}, end:{}",
+                  storeId,
+                  getMonitorDataKey(storeId, start),
+                  getMonitorDataKey(storeId, end));
+        return kvService.scanRange(getMonitorDataKey(storeId, start),
+                                   getMonitorDataKey(storeId, end));
+    }
+
+    /**
+     * for api service
+     *
+     * @param storeId
+     * @return
+     * @throws PDException
+     */
+    public List<Map<String, Long>> getStoreMonitorData(long storeId) throws PDException {
+        List<Map<String, Long>> result = new LinkedList<>();
+        long current = System.currentTimeMillis() / 1000;
+        long start = current - this.pdConfig.getStore().getRetentionPeriod();
+
+        try {
+            for (Map.Entry<String, String> entry : getStoreMonitorData(storeId, start,
+                                                                       current).entrySet()) {
+                String[] arr =
+                        entry.getKey().split(String.valueOf(MetadataKeyHelper.getDelimiter()));
+                Map<String, Long> map = new HashMap();
+                long timestamp = Long.parseLong(arr[arr.length - 1]);
+                map.put("ts", timestamp);
+                for (String pair : entry.getValue().split(",")) {
+                    String[] p = pair.split(":");
+                    if (p.length == 2) {
+                        map.put(p[0], Long.parseLong(p[1]));
+                    }
+                }
+                result.add(map);
+            }
+            result.sort((o1, o2) -> o1.get("ts").compareTo(o2.get("ts")));
+        } catch (PDException e) {
+            log.error(e.getMessage());
+        }
+        return result;
+    }
+
+    /**
+     * for api service, export txt
+     *
+     * @param storeId
+     * @return
+     * @throws PDException
+     */
+    public String getStoreMonitorDataText(long storeId) throws PDException {
+
+        List<Map<String, Long>> result = getStoreMonitorData(storeId);
+        StringBuilder sb = new StringBuilder();
+        if (result.size() > 0) {
+            DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+            Map<String, Long> lastRow = result.get(result.size() - 1);
+            List<String> columns = new ArrayList<>();
+            // construct columns, ts + sorted keys
+            columns.add("ts");
+            columns.addAll(lastRow.keySet().stream()
+                                  .filter(x -> !"ts".equals(x))
+                                  .sorted()
+                                  .collect(Collectors.toList()));
+            sb.append(String.join(",", columns).replace("\"", "")).append("\r\n");
+            for (Map<String, Long> row : result) {
+                for (String key : columns) {
+                    // ts + , + ...
+                    if ("ts".equals(key)) {
+                        // format ts
+                        sb.append(dtf.format(
+                                LocalDateTime.ofInstant(Instant.ofEpochSecond(row.get(key)),
+                                                        ZoneId.systemDefault())));
+                        continue;
+                    } else {
+                        sb.append(",").append(row.getOrDefault(key, 0L));
+                    }
+                }
+                sb.append("\r\n");
+            }
+        }
+        return sb.toString();
+    }
+
+    /**
+     * remove the monitor data of the store that before till(not include)
+     *
+     * @param storeId store id
+     * @param till    expire time
+     * @return affect rows
+     */
+    public int removeExpiredMonitorData(long storeId, long till) throws PDException {
+        String keyStart = getMonitorDataKey(storeId, 1);
+        String keyEnd = getMonitorDataKey(storeId, till);
+        int records = 0;
+        for (String key : kvService.scanRange(keyStart, keyEnd).keySet()) {
+            kvService.delete(key);
+            log.debug("remove monitor data, key: {}", key);
+            records += 1;
+        }
+        return records;
+    }
+
+    /**
+     * get the latest timestamp of the store monitor data
+     *
+     * @param storeId
+     * @return timestamp(by seconds)
+     */
+    public long getLatestStoreMonitorDataTimeStamp(long storeId) {
+        long maxId = 0L;
+        long current = System.currentTimeMillis() / 1000;
+        long start = current - this.pdConfig.getStore().getMonitorInterval();
+        String keyStart = getMonitorDataKey(storeId, start);
+        String keyEnd = getMonitorDataKey(storeId, current);
+        try {
+            for (String key : kvService.scanRange(keyStart, keyEnd).keySet()) {
+                String[] arr = key.split(String.valueOf(MetadataKeyHelper.getDelimiter()));
+                maxId = Math.max(maxId, Long.parseLong(arr[arr.length - 1]));
+            }
+        } catch (PDException e) {
+        }
+        return maxId;
+    }
+
+    private String getMonitorDataKey(long storeId, long ts) {
+        String builder = MONITOR_DATA_PREFIX +
+                         MetadataKeyHelper.getDelimiter() +
+                         storeId +
+                         MetadataKeyHelper.getDelimiter() +
+                         ts;
+        return builder;
+    }
+
+    private String extractMetricsFromStoreStatus(Metapb.StoreStats storeStats) {
+        List<String> list = new ArrayList<>();
+        for (Metapb.RecordPair pair : storeStats.getSystemMetricsList()) {
+            list.add("\"" + pair.getKey() + "\":" + pair.getValue());
+        }
+        return String.join(",", list);
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreStatusListener.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import org.apache.hugegraph.pd.grpc.Metapb;
+
+public interface StoreStatusListener {
+
+    void onStoreStatusChanged(Metapb.Store store, Metapb.StoreState old,
+                              Metapb.StoreState status);
+
+    void onGraphChange(Metapb.Graph graph, Metapb.GraphState stateOld,
+                       Metapb.GraphState stateNew);
+
+    void onStoreRaftChanged(Metapb.Store store);
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/TaskScheduleService.java

@@ -0,0 +1,853 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.PriorityQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import org.apache.hugegraph.pd.common.KVPair;
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.MetaTask;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.apache.hugegraph.pd.grpc.Pdpb;
+import org.apache.hugegraph.pd.meta.TaskInfoMeta;
+import org.apache.hugegraph.pd.raft.RaftEngine;
+
+import lombok.extern.slf4j.Slf4j;
+
+
+/**
+ * 任务调度服务，定时检查Store、资源、分区的状态，及时迁移数据，错误节点
+ * 1、监测Store是否离线
+ * 2、监测Partition的副本是否正确
+ * 3、监测Partition的工作模式是否正确
+ * 4、监测Partition是否需要分裂,监测分裂是否完成
+ */
+@Slf4j
+public class TaskScheduleService {
+    private static final String BALANCE_SHARD_KEY = "BALANCE_SHARD_KEY";
+    private final long TurnOffAndBalanceInterval = 30 * 60 * 1000; //机器下线30后才能进行动态平衡
+    private final long BalanceLeaderInterval = 30 * 1000;   // leader平衡时间间隔
+    private final PDConfig pdConfig;
+    private final long clusterStartTime;    //
+    private final StoreNodeService storeService;
+    private final PartitionService partitionService;
+    private final ScheduledExecutorService executor;
+    private final TaskInfoMeta taskInfoMeta;
+    private final StoreMonitorDataService storeMonitorDataService;
+    private final KvService kvService;
+    private final LogService logService;
+    // 先按照value排序，再按照key排序
+    private final Comparator<KVPair<Long, Integer>> kvPairComparatorAsc = (o1, o2) -> {
+        if (o1.getValue() == o2.getValue()) {
+            return o1.getKey().compareTo(o2.getKey());
+        }
+        return o1.getValue().compareTo(o2.getValue());
+    };
+    // 先按照value排序(倒序)，再按照key排序(升序）
+    private final Comparator<KVPair<Long, Integer>> kvPairComparatorDesc = (o1, o2) -> {
+        if (o1.getValue() == o2.getValue()) {
+            return o2.getKey().compareTo(o1.getKey());
+        }
+        return o2.getValue().compareTo(o1.getValue());
+    };
+    private long lastStoreTurnoffTime = 0;
+    private long lastBalanceLeaderTime = 0;
+
+
+    public TaskScheduleService(PDConfig config, StoreNodeService storeService,
+                               PartitionService partitionService) {
+        this.pdConfig = config;
+        this.storeService = storeService;
+        this.partitionService = partitionService;
+        this.taskInfoMeta = new TaskInfoMeta(config);
+        this.logService = new LogService(pdConfig);
+        this.storeMonitorDataService = new StoreMonitorDataService(pdConfig);
+        this.clusterStartTime = System.currentTimeMillis();
+        this.kvService = new KvService(pdConfig);
+        this.executor = new ScheduledThreadPoolExecutor(16);
+    }
+
+    public void init() {
+        executor.scheduleWithFixedDelay(() -> {
+            try {
+                patrolStores();
+            } catch (Throwable e) {
+                log.error("patrolStores exception: ", e);
+            }
+
+        }, 60, 60, TimeUnit.SECONDS);
+        executor.scheduleWithFixedDelay(() -> {
+            try {
+                patrolPartitions();
+                balancePartitionLeader(false);
+                balancePartitionShard();
+            } catch (Throwable e) {
+                log.error("patrolPartitions exception: ", e);
+            }
+        }, pdConfig.getPatrolInterval(), pdConfig.getPatrolInterval(), TimeUnit.SECONDS);
+        executor.scheduleWithFixedDelay(() -> {
+            if (isLeader()) {
+                kvService.clearTTLData();
+            }
+        }, 1000, 1000, TimeUnit.MILLISECONDS);
+        executor.scheduleWithFixedDelay(
+                () -> {
+                    if (isLeader()) {
+                        storeService.getQuotaChecker();
+                    }
+                }, 2, 30,
+                TimeUnit.SECONDS);
+        // clean expired monitor data each 10 minutes, delay 3min.
+        if (isLeader() && this.pdConfig.getStore().isMonitorDataEnabled()) {
+            executor.scheduleAtFixedRate(() -> {
+                Long expTill = System.currentTimeMillis() / 1000 -
+                               this.pdConfig.getStore().getRetentionPeriod();
+                log.debug("monitor data keys before " + expTill + " will be deleted");
+                int records = 0;
+                try {
+                    for (Metapb.Store store : storeService.getStores()) {
+                        int cnt =
+                                this.storeMonitorDataService.removeExpiredMonitorData(store.getId(),
+                                                                                      expTill);
+                        log.debug("store id :{}, records:{}", store.getId(), cnt);
+                        records += cnt;
+                    }
+                } catch (PDException e) {
+                    throw new RuntimeException(e);
+                }
+                log.debug(String.format("%d records has been deleted", records));
+            }, 180, 600, TimeUnit.SECONDS);
+        }
+
+        storeService.addStatusListener(new StoreStatusListener() {
+            @Override
+            public void onStoreStatusChanged(Metapb.Store store, Metapb.StoreState old,
+                                             Metapb.StoreState status) {
+                if (status == Metapb.StoreState.Tombstone) {
+                    lastStoreTurnoffTime = System.currentTimeMillis();
+                }
+
+                if (status == Metapb.StoreState.Up) {
+                    executor.schedule(() -> {
+                        try {  //store 上线后延时1分钟进行leader平衡
+                            balancePartitionLeader(false);
+                        } catch (PDException e) {
+                            log.error("exception {}", e);
+                        }
+                    }, BalanceLeaderInterval, TimeUnit.MILLISECONDS);
+
+                }
+            }
+
+            @Override
+            public void onGraphChange(Metapb.Graph graph,
+                                      Metapb.GraphState stateOld,
+                                      Metapb.GraphState stateNew) {
+
+            }
+
+            @Override
+            public void onStoreRaftChanged(Metapb.Store store) {
+
+            }
+        });
+    }
+
+    public void shutDown() {
+        executor.shutdownNow();
+    }
+
+    private boolean isLeader() {
+        return RaftEngine.getInstance().isLeader();
+    }
+
+    /**
+     * 巡查所有的store，检查是否在线，存储空间是否充足
+     */
+    public List<Metapb.Store> patrolStores() throws PDException {
+        if (!isLeader()) {
+            return null;
+        }
+
+        List<Metapb.Store> changedStores = new ArrayList<>();
+        // 检查store在线状态
+        List<Metapb.Store> stores = storeService.getStores("");
+        Map<Long, Metapb.Store> activeStores = storeService.getActiveStores("")
+                                                           .stream().collect(
+                        Collectors.toMap(Metapb.Store::getId, t -> t));
+        for (Metapb.Store store : stores) {
+            Metapb.Store changeStore = null;
+            if ((store.getState() == Metapb.StoreState.Up
+                 || store.getState() == Metapb.StoreState.Unknown)
+                && !activeStores.containsKey(store.getId())) {
+                // 不在线，修改状态为离线
+                changeStore = Metapb.Store.newBuilder(store)
+                                          .setState(Metapb.StoreState.Offline)
+                                          .build();
+
+            } else if ((store.getState() == Metapb.StoreState.Exiting &&
+                        !activeStores.containsKey(store.getId())) ||
+                       (store.getState() == Metapb.StoreState.Offline &&
+                        (System.currentTimeMillis() - store.getLastHeartbeat() >
+                         pdConfig.getStore().getMaxDownTime() * 1000) &&
+                        (System.currentTimeMillis() - clusterStartTime >
+                         pdConfig.getStore().getMaxDownTime() * 1000))) {
+                //手工修改为下线或者离线达到时长
+                // 修改状态为关机, 增加 checkStoreCanOffline 检测
+                if (storeService.checkStoreCanOffline(store)) {
+                    changeStore = Metapb.Store.newBuilder(store)
+                                              .setState(Metapb.StoreState.Tombstone).build();
+                    this.logService.insertLog(LogService.NODE_CHANGE,
+                                              LogService.TASK, changeStore);
+                    log.info("patrolStores store {} Offline", changeStore.getId());
+                }
+            }
+            if (changeStore != null) {
+                storeService.updateStore(changeStore);
+                changedStores.add(changeStore);
+            }
+        }
+        return changedStores;
+    }
+
+
+    /**
+     * 巡查所有的分区，检查副本数是否正确
+     */
+    public List<Metapb.Partition> patrolPartitions() throws PDException {
+        if (!isLeader()) {
+            return null;
+        }
+
+        // 副本数不一致，重新分配副本
+        for (Metapb.ShardGroup group : storeService.getShardGroups()) {
+            if (group.getShardsCount() != pdConfig.getPartition().getShardCount()) {
+                storeService.reallocShards(group);
+                // 避免后面的 balance partition shard 马上执行.
+                kvService.put(BALANCE_SHARD_KEY, "DOING", 180 * 1000);
+            }
+        }
+        //检查shard是否在线。
+        Map<Long, Metapb.Store> tombStores = storeService.getTombStores().stream().collect(
+                Collectors.toMap(Metapb.Store::getId, t -> t));
+
+        var partIds = new HashSet<Integer>();
+
+        for (var pair : tombStores.entrySet()) {
+            for (var partition : partitionService.getPartitionByStore(pair.getValue())) {
+                if (partIds.contains(partition.getId())) {
+                    continue;
+                }
+                partIds.add(partition.getId());
+
+                storeService.storeTurnoff(pair.getValue());
+                partitionService.shardOffline(partition, pair.getValue().getId());
+            }
+
+        }
+
+        return null;
+    }
+
+
+    /**
+     * 在Store之间平衡分区的数量
+     * 机器转为UP半小时后才能进行动态平衡
+     */
+    public synchronized Map<Integer, KVPair<Long, Long>> balancePartitionShard() throws
+                                                                                 PDException {
+        log.info("balancePartitions starting, isleader:{}", isLeader());
+
+        if (!isLeader()) {
+            return null;
+        }
+
+        if (System.currentTimeMillis() - lastStoreTurnoffTime < TurnOffAndBalanceInterval) {
+            return null;//机器下线半小时后才能进行动态平衡
+        }
+
+
+        int activeStores = storeService.getActiveStores().size();
+        if (activeStores == 0) {
+            log.warn("balancePartitionShard non active stores, skip to balancePartitionShard");
+            return null;
+        }
+
+        // 避免频繁调用. (当改变副本数，需要调整shard list，此时又需要平衡分区）会发送重复的指令。造成结果不可预料。
+        // 严重会删除掉分区.
+        if (Objects.equals(kvService.get(BALANCE_SHARD_KEY), "DOING")) {
+            return null;
+        }
+
+        int totalShards = pdConfig.getConfigService().getPartitionCount() *
+                          pdConfig.getPartition().getShardCount();
+        int averageCount = totalShards / activeStores;
+        int remainder = totalShards % activeStores;
+
+        // 统计每个store上分区, StoreId ->PartitionID, ShardRole
+        Map<Long, Map<Integer, Metapb.ShardRole>> partitionMap = new HashMap<>();
+        storeService.getActiveStores().forEach(store -> {
+            partitionMap.put(store.getId(), new HashMap<>());
+        });
+
+        // 如果是leaner 说明迁移正在进行，不要重复提交任务
+        AtomicReference<Boolean> isLeaner = new AtomicReference<>(false);
+        partitionService.getPartitions().forEach(partition -> {
+
+            try {
+                storeService.getShardList(partition.getId()).forEach(shard -> {
+                    Long storeId = shard.getStoreId();
+                    // 判断每个shard为leaner或者状态非正常状态
+                    if (shard.getRole() == Metapb.ShardRole.Learner
+                        || partition.getState() != Metapb.PartitionState.PState_Normal) {
+                        isLeaner.set(true);
+                    }
+                    if (partitionMap.containsKey(storeId)) {
+                        partitionMap.get(storeId).put(partition.getId(), shard.getRole());
+                    }
+                });
+            } catch (PDException e) {
+                log.error("get partition {} shard list error:{}.", partition.getId(),
+                          e.getMessage());
+            }
+        });
+
+        if (isLeaner.get()) {
+            log.warn("balancePartitionShard is doing, skip this balancePartitionShard task");
+            return null;
+        }
+
+        // 按照shard数量由高到低排序store
+        List<KVPair<Long, Integer>> sortedList = new ArrayList<>();
+        partitionMap.forEach((storeId, shards) -> {
+            sortedList.add(new KVPair(storeId, shards.size()));
+        });
+        // 由大到小排序的list
+        sortedList.sort(((o1, o2) -> o2.getValue().compareTo(o1.getValue())));
+        // 最大堆
+        PriorityQueue<KVPair<Long, Integer>> maxHeap = new PriorityQueue<>(sortedList.size(),
+                                                                           (o1, o2) -> o2.getValue()
+                                                                                         .compareTo(
+                                                                                                 o1.getValue()));
+
+        // 各个副本的 committedIndex
+        Map<Integer, Map<Long, Long>> committedIndexMap = partitionService.getCommittedIndexStats();
+        // 分区ID --> 源StoreID,目标StoreID
+        Map<Integer, KVPair<Long, Long>> movedPartitions = new HashMap<>();
+        // 移除多余的shard, 按照shards由多到少的顺序遍历store，余数remainder优先给shards多的store分配，减少迁移的概率
+        for (int index = 0; index < sortedList.size(); index++) {
+            long storeId = sortedList.get(index).getKey();
+            if (!partitionMap.containsKey(storeId)) {
+                log.error("cannot found storeId {} in partitionMap", storeId);
+                return null;
+            }
+            Map<Integer, Metapb.ShardRole> shards = partitionMap.get(storeId);
+            int targetCount = index < remainder ? averageCount + 1 : averageCount;
+            //  移除多余的shard, 添加源StoreID. 非Leader，并且该分区唯一
+            if (shards.size() > targetCount) {
+                int movedCount = shards.size() - targetCount;
+                log.info(
+                        "balancePartitionShard storeId {}, shardsSize {}, targetCount {}, " +
+                        "moveCount {}",
+                        storeId, shards.size(), targetCount, movedCount);
+                for (Iterator<Integer> iterator = shards.keySet().iterator();
+                     movedCount > 0 && iterator.hasNext(); ) {
+                    Integer id = iterator.next();
+
+                    if (!movedPartitions.containsKey(id)) {
+                        log.info("store {}, shard of partition {} can be moved", storeId, id);
+                        movedPartitions.put(id, new KVPair<>(storeId, 0L));
+                        movedCount--;
+                    }
+                }
+            } else if (shards.size() < targetCount) {
+                int addCount = targetCount - shards.size();
+                log.info(
+                        "balancePartitionShard storeId {}, shardsSize {}, targetCount {}, " +
+                        "addCount {}",
+                        storeId, shards.size(), targetCount, addCount);
+                maxHeap.add(new KVPair<>(storeId, addCount));
+            }
+        }
+
+        if (movedPartitions.size() == 0) {
+            log.warn(
+                    "movedPartitions is empty, totalShards:{} averageCount:{} remainder:{} " +
+                    "sortedList:{}",
+                    totalShards, averageCount, remainder, sortedList);
+        }
+        Iterator<Map.Entry<Integer, KVPair<Long, Long>>> moveIterator =
+                movedPartitions.entrySet().iterator();
+
+        while (moveIterator.hasNext()) {
+            if (maxHeap.size() == 0) {
+                break;
+            }
+            Map.Entry<Integer, KVPair<Long, Long>> moveEntry = moveIterator.next();
+            int partitionId = moveEntry.getKey();
+            long sourceStoreId = moveEntry.getValue().getKey();
+
+            List<KVPair<Long, Integer>> tmpList = new ArrayList<>(maxHeap.size());
+            while (maxHeap.size() > 0) {
+                KVPair<Long, Integer> pair = maxHeap.poll();
+                long destStoreId = pair.getKey();
+                boolean destContains = false;
+                if (partitionMap.containsKey(destStoreId)) {
+                    destContains = partitionMap.get(destStoreId).containsKey(partitionId);
+                }
+                // 如果目的store已经包含了该partition，则取一下store
+                if (!destContains) {
+                    moveEntry.getValue().setValue(pair.getKey());
+                    log.info(
+                            "balancePartitionShard will move partition {} from store {} to store " +
+                            "{}",
+                            moveEntry.getKey(),
+                            moveEntry.getValue().getKey(),
+                            moveEntry.getValue().getValue());
+                    if (pair.getValue() > 1) {
+                        pair.setValue(pair.getValue() - 1);
+                        tmpList.add(pair);
+                    }
+                    break;
+                }
+                tmpList.add(pair);
+            }
+            maxHeap.addAll(tmpList);
+        }
+
+        kvService.put(BALANCE_SHARD_KEY, "DOING", 180 * 1000);
+
+        // 开始迁移
+        movedPartitions.forEach((partId, storePair) -> {
+            // 源和目标storeID都不为0
+            if (storePair.getKey() > 0 && storePair.getValue() > 0) {
+                partitionService.movePartitionsShard(partId, storePair.getKey(),
+                                                     storePair.getValue());
+            } else {
+                log.warn("balancePartitionShard key or value is zero, partId:{} storePair:{}",
+                         partId, storePair);
+            }
+        });
+        return movedPartitions;
+    }
+
+    /**
+     * 在Store之间平衡分区的Leader的数量
+     */
+    public synchronized Map<Integer, Long> balancePartitionLeader(boolean immediately) throws
+                                                                                       PDException {
+        Map<Integer, Long> results = new HashMap<>();
+
+        if (!isLeader()) {
+            return results;
+        }
+
+        if (!immediately &&
+            System.currentTimeMillis() - lastBalanceLeaderTime < BalanceLeaderInterval) {
+            return results;
+        }
+        lastBalanceLeaderTime = System.currentTimeMillis();
+
+        List<Metapb.ShardGroup> shardGroups = storeService.getShardGroups();
+
+        // 分裂或者缩容任务的时候，退出
+        var taskMeta = storeService.getTaskInfoMeta();
+        if (taskMeta.hasSplitTaskDoing() || taskMeta.hasMoveTaskDoing()) {
+            throw new PDException(1001, "split or combine task is processing, please try later!");
+        }
+
+        // 数据迁移的时候，退出
+        if (Objects.equals(kvService.get(BALANCE_SHARD_KEY), "DOING")) {
+            throw new PDException(1001, "balance shard is processing, please try later!");
+        }
+
+        if (shardGroups.size() == 0) {
+            return results;
+        }
+
+        Map<Long, Integer> storeShardCount = new HashMap<>();
+
+        shardGroups.forEach(group -> {
+            group.getShardsList().forEach(shard -> {
+                storeShardCount.put(shard.getStoreId(),
+                                    storeShardCount.getOrDefault(shard.getStoreId(), 0) + 1);
+            });
+        });
+
+        log.info("balancePartitionLeader, shard group size: {}, by store: {}", shardGroups.size(),
+                 storeShardCount);
+
+        // 按照 target count， store id稳定排序
+        PriorityQueue<KVPair<Long, Integer>> targetCount =
+                new PriorityQueue<>(kvPairComparatorDesc);
+
+        var sortedGroups = storeShardCount.entrySet().stream()
+                                          .map(entry -> new KVPair<>(entry.getKey(),
+                                                                     entry.getValue()))
+                                          .sorted(kvPairComparatorAsc)
+                                          .collect(Collectors.toList());
+        int sum = 0;
+
+        for (int i = 0; i < sortedGroups.size() - 1; i++) {
+            // at least one
+            int v = Math.max(
+                    sortedGroups.get(i).getValue() / pdConfig.getPartition().getShardCount(), 1);
+            targetCount.add(new KVPair<>(sortedGroups.get(i).getKey(), v));
+            sum += v;
+        }
+        // 最后一个, 除不尽的情况，保证总数正确
+        targetCount.add(new KVPair<>(sortedGroups.get(sortedGroups.size() - 1).getKey(),
+                                     shardGroups.size() - sum));
+        log.info("target count: {}", targetCount);
+
+        for (var group : shardGroups) {
+            var map = group.getShardsList().stream()
+                           .collect(Collectors.toMap(Metapb.Shard::getStoreId, shard -> shard));
+            var tmpList = new ArrayList<KVPair<Long, Integer>>();
+            // store比较多的情况，可能不包含对应的store id. 则先将不符合的store保存到临时列表，直到找到一个合适的store
+            while (!targetCount.isEmpty()) {
+                var pair = targetCount.poll();
+                var storeId = pair.getKey();
+                if (map.containsKey(storeId)) {
+                    if (map.get(storeId).getRole() != Metapb.ShardRole.Leader) {
+                        log.info("shard group{}, store id:{}, set to leader", group.getId(),
+                                 storeId);
+                        partitionService.transferLeader(group.getId(), map.get(storeId));
+                        results.put(group.getId(), storeId);
+                    } else {
+                        log.info("shard group {}, store id :{}, is leader, no need change",
+                                 group.getId(), storeId);
+                    }
+
+                    if (pair.getValue() > 1) {
+                        // count -1
+                        pair.setValue(pair.getValue() - 1);
+                        tmpList.add(pair);
+                    }
+                    // 找到了，则处理完成
+                    break;
+                } else {
+                    tmpList.add(pair);
+                }
+            }
+            targetCount.addAll(tmpList);
+        }
+
+        return results;
+    }
+
+
+    private long getMaxIndexGap(Map<Integer, Map<Long, Long>> committedIndexMap, int partitionId) {
+        long maxGap = Long.MAX_VALUE;
+        if (committedIndexMap == null || !committedIndexMap.containsKey(partitionId)) {
+            return maxGap;
+        }
+        Map<Long, Long> shardMap = committedIndexMap.get(partitionId);
+        if (shardMap == null || shardMap.size() == 0) {
+            return maxGap;
+        }
+        List<Long> sortedList = new ArrayList<>();
+        shardMap.forEach((storeId, committedIndex) -> {
+            sortedList.add(committedIndex);
+        });
+        // 由大到小排序的list
+        sortedList.sort(Comparator.reverseOrder());
+        maxGap = sortedList.get(0) - sortedList.get(sortedList.size() - 1);
+        return maxGap;
+    }
+
+
+    /**
+     * 执行分区分裂，分为自动分裂和手工分裂
+     *
+     * @return
+     * @throws PDException
+     */
+    public List<Metapb.Partition> splitPartition(
+            Pdpb.OperationMode mode, List<Pdpb.SplitDataParam> params) throws PDException {
+
+        if (mode == Pdpb.OperationMode.Auto) {
+            return autoSplitPartition();
+        }
+
+        var list = params.stream()
+                         .map(param -> new KVPair<>(param.getPartitionId(), param.getCount()))
+                         .collect(Collectors.toList());
+
+        storeService.splitShardGroups(list);
+        return null;
+    }
+
+    /**
+     * 自动进行分区分裂，每个store达到最大分区数量
+     * 执行条件
+     * 分裂后每台机器分区数量少于partition.max-partitions-per-store
+     *
+     * @throws PDException
+     */
+    public List<Metapb.Partition> autoSplitPartition() throws PDException {
+        if (!isLeader()) {
+            return null;
+        }
+
+        if (Metapb.ClusterState.Cluster_OK != storeService.getClusterStats().getState()) {
+            if (Metapb.ClusterState.Cluster_Offline == storeService.getClusterStats().getState()) {
+                throw new PDException(Pdpb.ErrorType.Split_Partition_Doing_VALUE,
+                                      "The data is splitting");
+            } else {
+                throw new PDException(Pdpb.ErrorType.Cluster_State_Forbid_Splitting_VALUE,
+                                      "The current state of the cluster prohibits splitting data");
+            }
+        }
+
+        //For TEST
+        //   pdConfig.getPartition().setMaxShardsPerStore(pdConfig.getPartition()
+        //   .getMaxShardsPerStore()*2);
+
+        // 计算集群能能支持的最大split count
+        int splitCount = pdConfig.getPartition().getMaxShardsPerStore() *
+                         storeService.getActiveStores().size() /
+                         (storeService.getShardGroups().size() *
+                          pdConfig.getPartition().getShardCount());
+
+        if (splitCount < 2) {
+            throw new PDException(Pdpb.ErrorType.Too_Many_Partitions_Per_Store_VALUE,
+                                  "Too many partitions per store, partition.store-max-shard-count" +
+                                  " = "
+                                  + pdConfig.getPartition().getMaxShardsPerStore());
+        }
+
+        // 每store未达最大分区数，进行分裂
+        log.info("Start to split partitions..., split count = {}", splitCount);
+
+        // 设置集群状态为下线
+        storeService.updateClusterStatus(Metapb.ClusterState.Cluster_Offline);
+        // 修改默认分区数量
+        // pdConfig.getConfigService().setPartitionCount(storeService.getShardGroups().size() *
+        // splitCount);
+
+        var list = storeService.getShardGroups().stream()
+                               .map(shardGroup -> new KVPair<>(shardGroup.getId(), splitCount))
+                               .collect(Collectors.toList());
+        storeService.splitShardGroups(list);
+
+        return null;
+    }
+
+
+    /**
+     * Store汇报任务状态
+     * 分区状态发生改变，重新计算分区所在的ShardGroup、图和整个集群的状态
+     *
+     * @param task
+     */
+    public void reportTask(MetaTask.Task task) {
+        try {
+            switch (task.getType()) {
+                case Split_Partition:
+                    partitionService.handleSplitTask(task);
+                    break;
+                case Move_Partition:
+                    partitionService.handleMoveTask(task);
+                    break;
+                case Clean_Partition:
+                    partitionService.handleCleanPartitionTask(task);
+                    break;
+                default:
+                    break;
+            }
+        } catch (Exception e) {
+            log.error("Report task exception {}, {}", e, task);
+        }
+    }
+
+    /**
+     * 对rocksdb进行compaction
+     *
+     * @throws PDException
+     */
+    public Boolean dbCompaction(String tableName) throws PDException {
+        if (!isLeader()) {
+            return false;
+        }
+
+        for (Metapb.ShardGroup shardGroup : storeService.getShardGroups()) {
+            storeService.shardGroupsDbCompaction(shardGroup.getId(), tableName);
+        }
+
+        //
+        return true;
+    }
+
+    /**
+     * 判断是否能把一个store的分区全部迁出，给出判断结果和迁移方案
+     */
+    public Map<String, Object> canAllPartitionsMovedOut(Metapb.Store sourceStore) throws
+                                                                                  PDException {
+        if (!isLeader()) {
+            return null;
+        }
+        // 分析一个store上面的分区是否可以完全迁出
+        Map<String, Object> resultMap = new HashMap<>();
+        // 定义对象用于保存源store上面的分区 StoreId ->PartitionID, ShardRole
+        Map<Long, Map<Integer, Metapb.ShardRole>> sourcePartitionMap = new HashMap<>();
+        sourcePartitionMap.put(sourceStore.getId(), new HashMap<>());
+        // 定义对象用于保存其他活跃store上面的分区 StoreId ->PartitionID, ShardRole
+        Map<Long, Map<Integer, Metapb.ShardRole>> otherPartitionMap = new HashMap<>();
+        Map<Long, Long> availableDiskSpace = new HashMap<>(); // 每个store剩余的磁盘空间
+        Map<Integer, Long> partitionDataSize = new HashMap<>(); // 记录待迁移的分区的数据量
+
+        storeService.getActiveStores().forEach(store -> {
+            if (store.getId() != sourceStore.getId()) {
+                otherPartitionMap.put(store.getId(), new HashMap<>());
+                // 记录其他store的剩余的磁盘空间, 单位为Byte
+                availableDiskSpace.put(store.getId(), store.getStats().getAvailable());
+            } else {
+                resultMap.put("current_store_is_online", true);
+            }
+        });
+        // 统计待迁移的分区的数据大小 (从storeStats中统计，单位为KB)
+        for (Metapb.GraphStats graphStats : sourceStore.getStats().getGraphStatsList()) {
+            partitionDataSize.put(graphStats.getPartitionId(),
+                                  partitionDataSize.getOrDefault(graphStats.getPartitionId(), 0L)
+                                  + graphStats.getApproximateSize());
+        }
+        // 给sourcePartitionMap 和 otherPartitionMap赋值
+        partitionService.getPartitions().forEach(partition -> {
+            try {
+                storeService.getShardList(partition.getId()).forEach(shard -> {
+                    long storeId = shard.getStoreId();
+                    if (storeId == sourceStore.getId()) {
+                        sourcePartitionMap.get(storeId).put(partition.getId(), shard.getRole());
+                    } else {
+                        if (otherPartitionMap.containsKey(storeId)) {
+                            otherPartitionMap.get(storeId).put(partition.getId(), shard.getRole());
+                        }
+                    }
+
+                });
+            } catch (PDException e) {
+                throw new RuntimeException(e);
+            }
+        });
+        // 统计待移除的分区：即源store上面的所有分区
+        Map<Integer, KVPair<Long, Long>> movedPartitions = new HashMap<>();
+        for (Map.Entry<Integer, Metapb.ShardRole> entry : sourcePartitionMap.get(
+                sourceStore.getId()).entrySet()) {
+            movedPartitions.put(entry.getKey(), new KVPair<>(sourceStore.getId(), 0L));
+        }
+        // 统计其他store的分区数量, 用小顶堆保存，以便始终把分区数量较少的store优先考虑
+        PriorityQueue<KVPair<Long, Integer>> minHeap = new PriorityQueue<>(otherPartitionMap.size(),
+                                                                           (o1, o2) -> o1.getValue()
+                                                                                         .compareTo(
+                                                                                                 o2.getValue()));
+        otherPartitionMap.forEach((storeId, shards) -> {
+            minHeap.add(new KVPair(storeId, shards.size()));
+        });
+        // 遍历待迁移的分区,优先迁移到分区比较少的store
+        Iterator<Map.Entry<Integer, KVPair<Long, Long>>> moveIterator =
+                movedPartitions.entrySet().iterator();
+        while (moveIterator.hasNext()) {
+            Map.Entry<Integer, KVPair<Long, Long>> moveEntry = moveIterator.next();
+            int partitionId = moveEntry.getKey();
+            List<KVPair<Long, Integer>> tmpList = new ArrayList<>(); // 记录已经弹出优先队列的元素
+            while (minHeap.size() > 0) {
+                KVPair<Long, Integer> pair = minHeap.poll(); //弹出首个元素
+                long storeId = pair.getKey();
+                int partitionCount = pair.getValue();
+                Map<Integer, Metapb.ShardRole> shards = otherPartitionMap.get(storeId);
+                final int unitRate = 1024; // 平衡不同存储单位的进率
+                if ((!shards.containsKey(partitionId)) && (
+                        availableDiskSpace.getOrDefault(storeId, 0L) / unitRate >=
+                        partitionDataSize.getOrDefault(partitionId, 0L))) {
+                    // 如果目标store上面不包含该分区，且目标store剩余空间能容纳该分区，则进行迁移
+                    moveEntry.getValue().setValue(storeId); //设置移动的目标store
+                    log.info("plan to move partition {} to store {}, " +
+                             "available disk space {}, current partitionSize:{}",
+                             partitionId,
+                             storeId,
+                             availableDiskSpace.getOrDefault(storeId, 0L) / unitRate,
+                             partitionDataSize.getOrDefault(partitionId, 0L)
+                    );
+                    // 更新该store预期的剩余空间
+                    availableDiskSpace.put(storeId, availableDiskSpace.getOrDefault(storeId, 0L)
+                                                    - partitionDataSize.getOrDefault(partitionId,
+                                                                                     0L) *
+                                                      unitRate);
+                    // 更新统计变量中该store的分区数量
+                    partitionCount += 1;
+                    pair.setValue(partitionCount);
+                    tmpList.add(pair);
+                    break;
+                } else {
+                    tmpList.add(pair);
+                }
+            }
+            minHeap.addAll(tmpList);
+        }
+        //检查是否未存在未分配目标store的分区
+        List<Integer> remainPartitions = new ArrayList<>();
+        movedPartitions.forEach((partId, storePair) -> {
+            if (storePair.getValue() == 0L) {
+                remainPartitions.add(partId);
+            }
+        });
+        if (remainPartitions.size() > 0) {
+            resultMap.put("flag", false);
+            resultMap.put("movedPartitions", null);
+        } else {
+            resultMap.put("flag", true);
+            resultMap.put("movedPartitions", movedPartitions);
+        }
+        return resultMap;
+
+    }
+
+    public Map<Integer, KVPair<Long, Long>> movePartitions(
+            Map<Integer, KVPair<Long, Long>> movedPartitions) {
+        if (!isLeader()) {
+            return null;
+        }
+        // 开始迁移
+        log.info("begin move partitions:");
+        movedPartitions.forEach((partId, storePair) -> {
+            // 源和目标storeID都不为0
+            if (storePair.getKey() > 0 && storePair.getValue() > 0) {
+                partitionService.movePartitionsShard(partId, storePair.getKey(),
+                                                     storePair.getValue());
+            }
+        });
+        return movedPartitions;
+    }
+
+
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java

@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.config;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hugegraph.pd.ConfigService;
+import org.apache.hugegraph.pd.IdService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.stereotype.Component;
+
+import lombok.Data;
+
+
+/**
+ * PD配置文件
+ */
+@Data
+@Component
+public class PDConfig {
+
+    @Value("${pd.cluster_id:1}")
+    private long clusterId;   // 集群ID
+
+    @Value("${pd.patrol-interval:300}")
+    private long patrolInterval = 300;  //巡查任务时间间隔
+    @Value("${pd.data-path}")
+    private String dataPath;
+    @Value("${pd.initial-store-count:3}")
+    private int minStoreCount;
+
+    // 初始store列表，该列表内的store自动激活
+    @Value("${pd.initial-store-list: ''}")
+    private String initialStoreList;
+    @Value("${grpc.host}")
+    private String host;
+
+    @Value("${license.verify-path}")
+    private String verifyPath;
+    @Value("${license.license-path}")
+    private String licensePath;
+    @Autowired
+    private ThreadPoolGrpc threadPoolGrpc;
+    @Autowired
+    private Raft raft;
+    @Autowired
+    private Store store;
+    @Autowired
+    private Partition partition;
+    @Autowired
+    private Discovery discovery;
+    private Map<String, String> initialStoreMap = null;
+    private ConfigService configService;
+    private IdService idService;
+
+    public Map<String, String> getInitialStoreMap() {
+        if (initialStoreMap == null) {
+            initialStoreMap = new HashMap<>();
+            Arrays.asList(initialStoreList.split(",")).forEach(s -> {
+                initialStoreMap.put(s, s);
+            });
+        }
+        return initialStoreMap;
+    }
+
+    /**
+     * 初始分区数量
+     * Store数量 * 每Store最大副本数 /每分区副本数
+     *
+     * @return
+     */
+    public int getInitialPartitionCount() {
+        return getInitialStoreMap().size() * partition.getMaxShardsPerStore()
+               / partition.getShardCount();
+    }
+
+    public ConfigService getConfigService() {
+        return configService;
+    }
+
+    public void setConfigService(ConfigService configService) {
+        this.configService = configService;
+    }
+
+    public IdService getIdService() {
+        return idService;
+    }
+
+    public void setIdService(IdService idService) {
+        this.idService = idService;
+    }
+
+    @Data
+    @Configuration
+    public class ThreadPoolGrpc {
+        @Value("${thread.pool.grpc.core:600}")
+        private int core;
+        @Value("${thread.pool.grpc.max:1000}")
+        private int max;
+        @Value("${thread.pool.grpc.queue:" + Integer.MAX_VALUE + "}")
+        private int queue;
+    }
+
+    @Data
+    @Configuration
+    public class Raft {
+        @Value("${raft.enable:true }")
+        private boolean enable;
+        @Value("${raft.address}")
+        private String address;
+        @Value("${pd.data-path}")
+        private String dataPath;
+        @Value("${raft.peers-list}")
+        private String peersList;
+        @Value("${raft.snapshotInterval: 300}")
+        private int snapshotInterval;
+        @Value("${raft.rpc-timeout:10000}")
+        private int rpcTimeout;
+        @Value("${grpc.host}")
+        private String host;
+        @Value("${server.port}")
+        private int port;
+
+        @Value("${pd.cluster_id:1}")
+        private long clusterId;   // 集群ID
+        @Value("${grpc.port}")
+        private int grpcPort;
+
+        public String getGrpcAddress() {
+            return host + ":" + grpcPort;
+        }
+    }
+
+    @Data
+    @Configuration
+    public class Store {
+        // store 心跳超时时间
+        @Value("${store.keepAlive-timeout:300}")
+        private long keepAliveTimeout = 300;
+        @Value("${store.max-down-time:1800}")
+        private long maxDownTime = 1800;
+
+        @Value("${store.monitor_data_enabled:true}")
+        private boolean monitorDataEnabled = true;
+
+        @Value("${store.monitor_data_interval: 1 minute}")
+        private String monitorDataInterval = "1 minute";
+
+        @Value("${store.monitor_data_retention: 1 day}")
+        private String monitorDataRetention = "1 day";
+
+        /**
+         * interval -> seconds.
+         * minimum value is 1 seconds.
+         *
+         * @return the seconds of the interval
+         */
+        public Long getMonitorInterval() {
+            return parseTimeExpression(this.monitorDataInterval);
+        }
+
+        /**
+         * the monitor data that saved in rocksdb, will be deleted
+         * out of period
+         *
+         * @return the period of the monitor data should keep
+         */
+        public Long getRetentionPeriod() {
+            return parseTimeExpression(this.monitorDataRetention);
+        }
+
+        /**
+         * parse time expression , support pattern:
+         * [1-9][ ](second, minute, hour, day, month, year)
+         * unit could not be null, the number part is 1 by default.
+         *
+         * @param exp
+         * @return seconds value of the expression. 1 will return by illegal expression
+         */
+        private Long parseTimeExpression(String exp) {
+            if (exp != null) {
+                Pattern pattern = Pattern.compile(
+                    "(?<n>(\\d+)*)(\\s)*(?<unit>(second|minute|hour|day|month|year)$)");
+                Matcher matcher = pattern.matcher(exp.trim());
+                if (matcher.find()) {
+                    String n = matcher.group("n");
+                    String unit = matcher.group("unit");
+
+                    if (null == n || n.length() == 0) {
+                        n = "1";
+                    }
+
+                    Long interval;
+                    switch (unit) {
+                        case "minute":
+                            interval = 60L;
+                            break;
+                        case "hour":
+                            interval = 3600L;
+                            break;
+                        case "day":
+                            interval = 86400L;
+                            break;
+                        case "month":
+                            interval = 86400L * 30;
+                            break;
+                        case "year":
+                            interval = 86400L * 365;
+                            break;
+                        case "second":
+                        default:
+                            interval = 1L;
+                    }
+                    // avoid n == '0'
+                    return Math.max(1L, interval * Integer.parseInt(n));
+                }
+            }
+            return 1L;
+        }
+
+    }
+
+    @Data
+    @Configuration
+    public class Partition {
+        private int totalCount = 0;
+
+        // 每个Store最大副本数
+        @Value("${partition.store-max-shard-count:24}")
+        private int maxShardsPerStore = 24;
+
+        // 默认分副本数量
+        @Value("${partition.default-shard-count:3}")
+        private int shardCount = 3;
+
+        public int getTotalCount() {
+            if (totalCount == 0) {
+                totalCount = getInitialPartitionCount();
+            }
+            return totalCount;
+        }
+
+        public void setTotalCount(int totalCount) {
+            this.totalCount = totalCount;
+        }
+    }
+
+    @Data
+    @Configuration
+    public class Discovery {
+        // 客户端注册后，无心跳最长次数，超过后，之前的注册信息会被删除
+        @Value("${discovery.heartbeat-try-count:3}")
+        private int heartbeatOutTimes = 3;
+    }
+
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/ConfigMetaStore.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.meta;
+
+import java.util.List;
+import java.util.Optional;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.Metapb;
+
+public class ConfigMetaStore extends MetadataRocksDBStore {
+
+
+    private final long clusterId;
+
+    public ConfigMetaStore(PDConfig pdConfig) {
+        super(pdConfig);
+        this.clusterId = pdConfig.getClusterId();
+    }
+
+    /**
+     * 更新图空间存储状态信息
+     *
+     * @param
+     */
+    public Metapb.GraphSpace setGraphSpace(Metapb.GraphSpace graphSpace) throws PDException {
+        byte[] graphSpaceKey = MetadataKeyHelper.getGraphSpaceKey(graphSpace.getName());
+        graphSpace = graphSpace.toBuilder().setTimestamp(System.currentTimeMillis()).build();
+        put(graphSpaceKey, graphSpace.toByteArray());
+        return graphSpace;
+    }
+
+    public List<Metapb.GraphSpace> getGraphSpace(String graphSpace) throws PDException {
+        byte[] graphSpaceKey = MetadataKeyHelper.getGraphSpaceKey(graphSpace);
+        return scanPrefix(Metapb.GraphSpace.parser(), graphSpaceKey);
+    }
+
+    public Metapb.PDConfig setPdConfig(Metapb.PDConfig pdConfig) throws PDException {
+        byte[] graphSpaceKey =
+                MetadataKeyHelper.getPdConfigKey(String.valueOf(pdConfig.getVersion()));
+        Metapb.PDConfig config = Metapb.PDConfig.newBuilder(
+                pdConfig).setTimestamp(System.currentTimeMillis()).build();
+        put(graphSpaceKey, config.toByteArray());
+        return config;
+    }
+
+    public Metapb.PDConfig getPdConfig(long version) throws PDException {
+        byte[] graphSpaceKey = MetadataKeyHelper.getPdConfigKey(version <= 0 ? null :
+                                                                String.valueOf(version));
+        Optional<Metapb.PDConfig> max = scanPrefix(
+                Metapb.PDConfig.parser(), graphSpaceKey).stream().max(
+                (o1, o2) -> (o1.getVersion() > o2.getVersion()) ? 1 : -1);
+        return max.isPresent() ? max.get() : null;
+    }
+
+
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/DiscoveryMetaStore.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.meta;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.discovery.NodeInfo;
+import org.apache.hugegraph.pd.grpc.discovery.NodeInfos;
+import org.apache.hugegraph.pd.grpc.discovery.Query;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class DiscoveryMetaStore extends MetadataRocksDBStore {
+
+    /**
+     * appName --> address --> registryInfo
+     */
+    private static final String PREFIX = "REGIS-";
+    private static final String SPLITTER = "-";
+
+    public DiscoveryMetaStore(PDConfig pdConfig) {
+        super(pdConfig);
+    }
+
+    public void register(NodeInfo nodeInfo, int outTimes) throws PDException {
+        putWithTTL(toKey(nodeInfo.getAppName(), nodeInfo.getVersion(), nodeInfo.getAddress()),
+                   nodeInfo.toByteArray(), (nodeInfo.getInterval() / 1000) * outTimes);
+    }
+
+    byte[] toKey(String appName, String version, String address) {
+        StringBuilder builder = getPrefixBuilder(appName, version);
+        builder.append(SPLITTER);
+        builder.append(address);
+        return builder.toString().getBytes();
+    }
+
+    private StringBuilder getPrefixBuilder(String appName, String version) {
+        StringBuilder builder = new StringBuilder();
+        builder.append(PREFIX);
+        if (!StringUtils.isEmpty(appName)) {
+            builder.append(appName);
+            builder.append(SPLITTER);
+        }
+        if (!StringUtils.isEmpty(version)) {
+            builder.append(version);
+        }
+        return builder;
+    }
+
+    public NodeInfos getNodes(Query query) {
+        List<NodeInfo> nodeInfos = null;
+        try {
+            StringBuilder builder = getPrefixBuilder(query.getAppName(),
+                                                     query.getVersion());
+            nodeInfos = getInstanceListWithTTL(
+                    NodeInfo.parser(),
+                    builder.toString().getBytes());
+            builder.setLength(0);
+        } catch (PDException e) {
+            log.error("An error occurred getting data from the store,{}", e);
+        }
+        if (query.getLabelsMap() != null && !query.getLabelsMap().isEmpty()) {
+            List result = new LinkedList<NodeInfo>();
+            for (NodeInfo node : nodeInfos) {
+                if (labelMatch(node, query)) {
+                    result.add(node);
+                }
+            }
+            return NodeInfos.newBuilder().addAllInfo(result).build();
+        }
+        return NodeInfos.newBuilder().addAllInfo(nodeInfos).build();
+
+    }
+
+    private boolean labelMatch(NodeInfo node, Query query) {
+        Map<String, String> labelsMap = node.getLabelsMap();
+        for (Map.Entry<String, String> entry : query.getLabelsMap().entrySet()) {
+            if (!entry.getValue().equals(labelsMap.get(entry.getKey()))) {
+                return false;
+            }
+        }
+        return true;
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/IdMetaStore.java

@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.meta;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.store.KV;
+
+import com.caucho.hessian.io.Hessian2Input;
+import com.caucho.hessian.io.Hessian2Output;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * 自增id的实现类
+ */
+@Slf4j
+public class IdMetaStore extends MetadataRocksDBStore {
+
+
+    private static final String ID_PREFIX = "@ID@";
+    private static final String CID_PREFIX = "@CID@";
+    private static final String CID_SLOT_PREFIX = "@CID_SLOT@";
+    private static final String CID_DEL_SLOT_PREFIX = "@CID_DEL_SLOT@";
+    private static final String SEPARATOR = "@";
+    private static final ConcurrentHashMap<String, Object> SEQUENCES = new ConcurrentHashMap<>();
+    public static long CID_DEL_TIMEOUT = 24 * 3600 * 1000;
+    private final long clusterId;
+
+    public IdMetaStore(PDConfig pdConfig) {
+        super(pdConfig);
+        this.clusterId = pdConfig.getClusterId();
+    }
+
+    public static long bytesToLong(byte[] b) {
+        ByteBuffer buf = ByteBuffer.wrap(b);
+        return buf.getLong();
+    }
+
+    public static byte[] longToBytes(long l) {
+        ByteBuffer buf = ByteBuffer.wrap(new byte[Long.SIZE]);
+        buf.putLong(l);
+        buf.flip();
+        return buf.array();
+    }
+
+    /**
+     * 获取自增id
+     *
+     * @param key
+     * @param delta
+     * @return
+     * @throws PDException
+     */
+    public long getId(String key, int delta) throws PDException {
+        Object probableLock = getLock(key);
+        byte[] keyBs = (ID_PREFIX + key).getBytes(Charset.defaultCharset());
+        synchronized (probableLock) {
+            byte[] bs = getOne(keyBs);
+            long current = bs != null ? bytesToLong(bs) : 0L;
+            long next = current + delta;
+            put(keyBs, longToBytes(next));
+            return current;
+        }
+    }
+
+    private Object getLock(String key) {
+        Object probableLock = new Object();
+        Object currentLock = SEQUENCES.putIfAbsent(key, probableLock);
+        if (currentLock != null) {
+            probableLock = currentLock;
+        }
+        return probableLock;
+    }
+
+    public void resetId(String key) throws PDException {
+        Object probableLock = new Object();
+        Object currentLock = SEQUENCES.putIfAbsent(key, probableLock);
+        if (currentLock != null) {
+            probableLock = currentLock;
+        }
+        byte[] keyBs = (ID_PREFIX + key).getBytes(Charset.defaultCharset());
+        synchronized (probableLock) {
+            removeByPrefix(keyBs);
+        }
+    }
+
+    /**
+     * 在删除name标识的cid的24小时内重复申请同一个name的cid保持同一值
+     * 如此设计为了防止缓存的不一致，造成数据错误
+     *
+     * @param key
+     * @param name cid 标识
+     * @param max
+     * @return
+     * @throws PDException
+     */
+    public long getCId(String key, String name, long max) throws PDException {
+        // 检测是否有过期的cid，删除图的频率比较低，此处对性能影响不大
+        byte[] delKeyPrefix = (CID_DEL_SLOT_PREFIX +
+                               key + SEPARATOR).getBytes(Charset.defaultCharset());
+        synchronized (this) {
+            scanPrefix(delKeyPrefix).forEach(kv -> {
+                long[] value = (long[]) deserialize(kv.getValue());
+                if (value.length >= 2) {
+                    if (System.currentTimeMillis() - value[1] > CID_DEL_TIMEOUT) {
+                        try {
+                            delCId(key, value[0]);
+                            remove(kv.getKey());
+                        } catch (Exception e) {
+                            log.error("Exception ", e);
+                        }
+                    }
+                }
+            });
+
+            // 从延时删除队列恢复Key
+            byte[] cidDelayKey = getCIDDelayKey(key, name);
+            byte[] value = getOne(cidDelayKey);
+            if (value != null) {
+                // 从延迟删除队列删除
+                remove(cidDelayKey);
+                return ((long[]) deserialize(value))[0];
+            } else {
+                return getCId(key, max);
+            }
+        }
+    }
+
+    /**
+     * 添加到删除队列，延后删除
+     */
+    public long delCIdDelay(String key, String name, long cid) throws PDException {
+        byte[] delKey = getCIDDelayKey(key, name);
+        put(delKey, serialize(new long[]{cid, System.currentTimeMillis()}));
+        return cid;
+    }
+
+    /**
+     * 获取自增循环不重复id, 达到上限后从0开始自增
+     *
+     * @param key
+     * @param max id上限，达到该值后，重新从0开始自增
+     * @return
+     * @throws PDException
+     */
+    public long getCId(String key, long max) throws PDException {
+        Object probableLock = getLock(key);
+        byte[] keyBs = (CID_PREFIX + key).getBytes(Charset.defaultCharset());
+        synchronized (probableLock) {
+            byte[] bs = getOne(keyBs);
+            long current = bs != null ? bytesToLong(bs) : 0L;
+            long last = current == 0 ? max - 1 : current - 1;
+            {   // 查找一个未使用的cid
+                List<KV> kvs = scanRange(genCIDSlotKey(key, current), genCIDSlotKey(key, max));
+                for (KV kv : kvs) {
+                    if (current == bytesToLong(kv.getValue())) {
+                        current++;
+                    } else {
+                        break;
+                    }
+                }
+            }
+            if (current == max) {
+                current = 0;
+                List<KV> kvs = scanRange(genCIDSlotKey(key, current), genCIDSlotKey(key, last));
+                for (KV kv : kvs) {
+                    if (current == bytesToLong(kv.getValue())) {
+                        current++;
+                    } else {
+                        break;
+                    }
+                }
+            }
+            if (current == last) {
+                return -1;
+            }
+            put(genCIDSlotKey(key, current), longToBytes(current));
+            put(keyBs, longToBytes(current + 1));
+            return current;
+        }
+    }
+
+    private byte[] genCIDSlotKey(String key, long value) {
+        byte[] keySlot = (CID_SLOT_PREFIX + key + SEPARATOR).getBytes(Charset.defaultCharset());
+        ByteBuffer buf = ByteBuffer.allocate(keySlot.length + Long.SIZE);
+        buf.put(keySlot);
+        buf.put(longToBytes(value));
+        return buf.array();
+    }
+
+    private byte[] getCIDDelayKey(String key, String name) {
+        byte[] bsKey = (CID_DEL_SLOT_PREFIX +
+                        key + SEPARATOR +
+                        name).getBytes(Charset.defaultCharset());
+        return bsKey;
+    }
+
+    /**
+     * 删除一个循环id，释放id值
+     *
+     * @param key
+     * @param cid
+     * @return
+     * @throws PDException
+     */
+    public long delCId(String key, long cid) throws PDException {
+        return remove(genCIDSlotKey(key, cid));
+    }
+
+    private byte[] serialize(Object obj) {
+        try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
+            Hessian2Output output = new Hessian2Output(bos);
+            output.writeObject(obj);
+            output.flush();
+            return bos.toByteArray();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private Object deserialize(byte[] bytes) {
+        try (ByteArrayInputStream bis = new ByteArrayInputStream(bytes)) {
+            Hessian2Input input = new Hessian2Input(bis);
+            Object obj = input.readObject();
+            input.close();
+            return obj;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/LogMeta.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.meta;
+
+import java.util.List;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.Metapb;
+
+public class LogMeta extends MetadataRocksDBStore {
+
+    private final PDConfig pdConfig;
+
+    public LogMeta(PDConfig pdConfig) {
+        super(pdConfig);
+        this.pdConfig = pdConfig;
+    }
+
+    public void insertLog(Metapb.LogRecord record) throws PDException {
+        byte[] storeLogKey = MetadataKeyHelper.getLogKey(record);
+        put(storeLogKey, record.toByteArray());
+
+    }
+
+    public List<Metapb.LogRecord> getLog(String action, Long start, Long end) throws PDException {
+        byte[] keyStart = MetadataKeyHelper.getLogKeyPrefix(action, start);
+        byte[] keyEnd = MetadataKeyHelper.getLogKeyPrefix(action, end);
+        List<Metapb.LogRecord> stores = this.scanRange(Metapb.LogRecord.parser(),
+                                                       keyStart, keyEnd);
+        return stores;
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataFactory.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.meta;
+
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.raft.RaftEngine;
+import org.apache.hugegraph.pd.store.HgKVStore;
+import org.apache.hugegraph.pd.store.HgKVStoreImpl;
+import org.apache.hugegraph.pd.store.RaftKVStore;
+
+/**
+ * 存储工厂类，创建相关对象的存储类
+ */
+public class MetadataFactory {
+
+    private static HgKVStore store = null;
+
+    public static HgKVStore getStore(PDConfig pdConfig) {
+        if (store == null) {
+            synchronized (MetadataFactory.class) {
+                if (store == null) {
+                    HgKVStore proto = new HgKVStoreImpl();
+                    //proto.init(pdConfig);
+                    store = pdConfig.getRaft().isEnable() ?
+                            new RaftKVStore(RaftEngine.getInstance(), proto) :
+                            proto;
+                    store.init(pdConfig);
+                }
+            }
+        }
+        return store;
+    }
+
+    public static void closeStore() {
+        if (store != null) {
+            store.close();
+        }
+    }
+
+    public static StoreInfoMeta newStoreInfoMeta(PDConfig pdConfig) {
+        return new StoreInfoMeta(pdConfig);
+    }
+
+    public static PartitionMeta newPartitionMeta(PDConfig pdConfig) {
+        return new PartitionMeta(pdConfig);
+    }
+
+    public static IdMetaStore newHugeServerMeta(PDConfig pdConfig) {
+        return new IdMetaStore(pdConfig);
+    }
+
+    public static DiscoveryMetaStore newDiscoveryMeta(PDConfig pdConfig) {
+        return new DiscoveryMetaStore(pdConfig);
+    }
+
+    public static ConfigMetaStore newConfigMeta(PDConfig pdConfig) {
+        return new ConfigMetaStore(pdConfig);
+    }
+
+    public static TaskInfoMeta newTaskInfoMeta(PDConfig pdConfig) {
+        return new TaskInfoMeta(pdConfig);
+    }
+
+
+    public static QueueStore newQueueStore(PDConfig pdConfig) {
+        return new QueueStore(pdConfig);
+    }
+
+    public static LogMeta newLogMeta(PDConfig pdConfig) {
+        return new LogMeta(pdConfig);
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataKeyHelper.java

@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.meta;
+
+import java.nio.charset.Charset;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hugegraph.pd.grpc.Metapb;
+
+public class MetadataKeyHelper {
+
+    public static final char DELIMITER = '/';
+
+    private static final String STORE = "STORE";
+    private static final String ACTIVESTORE = "ACTIVESTORE";
+    private static final String STORESTATUS = "STORESTATUS";
+    private static final String PARTITION = "PARTITION";
+    private static final String PARTITION_V36 = "PARTITION_V36";
+    private static final String SHARDGROUP = "SHARDGROUP";
+
+    private static final String PARTITION_STATUS = "PARTITION_STATUS";
+    private static final String GRAPH = "GRAPH";
+    private static final String GRAPHMETA = "GRAPHMETA";
+    private static final String GRAPH_SPACE = "GRAPH_SPACE";
+    private static final String PD_CONFIG = "PD_CONFIG";
+    private static final String TASK_SPLIT = "TASK_SPLIT";
+    private static final String TASK_MOVE = "TASK_MOVE";
+    private static final String LOG_RECORD = "LOG_RECORD";
+
+    private static final String QUEUE = "QUEUE";
+
+    public static byte[] getStoreInfoKey(final long storeId) {
+        //STORE/{storeId}
+        String key = StringBuilderHelper.get()
+                                        .append(STORE).append(DELIMITER)
+                                        .append(storeId)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getActiveStoreKey(final long storeId) {
+        //ACTIVESTORE/{storeId}
+        String key = StringBuilderHelper.get()
+                                        .append(ACTIVESTORE).append(DELIMITER)
+                                        .append(storeId)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getActiveStorePrefix() {
+        //ACTIVESTORE
+        String key = StringBuilderHelper.get()
+                                        .append(ACTIVESTORE).append(DELIMITER)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getStorePrefix() {
+        //STORE
+        String key = StringBuilderHelper.get()
+                                        .append(STORE).append(DELIMITER)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getStoreStatusKey(final long storeId) {
+        //STORESTATUS/{storeId}
+        String key = StringBuilderHelper.get()
+                                        .append(STORESTATUS).append(DELIMITER)
+                                        .append(storeId)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getShardGroupKey(final long groupId) {
+        //SHARDGROUP/{storeId}
+        String key = StringBuilderHelper.get()
+                                        .append(SHARDGROUP).append(DELIMITER)
+                                        .append(groupId)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getShardGroupPrefix() {
+        //SHARDGROUP
+        String key = StringBuilderHelper.get()
+                                        .append(SHARDGROUP).append(DELIMITER)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getPartitionKey(final String graphName, final int partId) {
+        //GRAPH/{graphName}/Partition/{partId}
+        String key = StringBuilderHelper.get()
+                                        .append(GRAPH).append(DELIMITER)
+                                        .append(graphName).append(DELIMITER)
+                                        .append(PARTITION).append(DELIMITER)
+                                        .append(partId)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getPartitionV36Key(final String graphName, final int partId) {
+        // GRAPH/{graphName}/PartitionV36/{partId}
+        String key = StringBuilderHelper.get()
+                                        .append(GRAPH).append(DELIMITER)
+                                        .append(graphName).append(DELIMITER)
+                                        .append(PARTITION_V36).append(DELIMITER)
+                                        .append(partId)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getPartitionPrefix(final String graphName) {
+        //GRAPH/{graph}/Partition
+        String key = StringBuilderHelper.get()
+                                        .append(GRAPH).append(DELIMITER)
+                                        .append(graphName).append(DELIMITER)
+                                        .append(PARTITION).append(DELIMITER)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getShardKey(final long storeId, final int partId) {
+        //SHARD/{graphName}/{type}
+        String key = StringBuilderHelper.get()
+                                        .append(SHARDGROUP).append(DELIMITER)
+                                        .append(storeId).append(DELIMITER)
+                                        .append(partId)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getShardPrefix(final long storeId) {
+        //SHARD/{graphName}/{type}
+        String key = StringBuilderHelper.get()
+                                        .append(SHARDGROUP).append(DELIMITER)
+                                        .append(storeId).append(DELIMITER)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getGraphKey(final String graphName) {
+        //GRAPHMETA/{graphName}
+        String key = StringBuilderHelper.get()
+                                        .append(GRAPHMETA).append(DELIMITER)
+                                        .append(graphName).append(DELIMITER)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getGraphPrefix() {
+        //GRAPHMETA/{
+        String key = StringBuilderHelper.get()
+                                        .append(GRAPHMETA).append(DELIMITER)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getPartitionStatusKey(String graphName, int id) {
+        //PARTITION_STATUS/{
+        String key = StringBuilderHelper.get()
+                                        .append(PARTITION_STATUS)
+                                        .append(DELIMITER)
+                                        .append(graphName).append(DELIMITER)
+                                        .append(id).append(DELIMITER)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getPartitionStatusPrefixKey(String graphName) {
+        //PARTITION_STATUS/{
+        StringBuilder builder = StringBuilderHelper.get().append(PARTITION_STATUS)
+                                                   .append(DELIMITER);
+        if (!StringUtils.isEmpty(graphName)) {
+            builder.append(graphName).append(DELIMITER);
+        }
+        return builder.toString().getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getGraphSpaceKey(String graphSpace) {
+        //GRAPH_SPACE/{
+        StringBuilder builder = StringBuilderHelper.get().append(
+                GRAPH_SPACE).append(DELIMITER);
+        if (!StringUtils.isEmpty(graphSpace)) {
+            builder.append(graphSpace).append(DELIMITER);
+        }
+        return builder.toString().getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getPdConfigKey(String configKey) {
+        //PD_CONFIG/{
+        StringBuilder builder = StringBuilderHelper.get().append(
+                PD_CONFIG).append(DELIMITER);
+        if (!StringUtils.isEmpty(configKey)) {
+            builder.append(configKey).append(DELIMITER);
+        }
+        return builder.toString().getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getQueueItemPrefix() {
+        //QUEUE
+        String key = StringBuilderHelper.get()
+                                        .append(QUEUE).append(DELIMITER)
+                                        .toString();
+        return key.getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getQueueItemKey(String itemId) {
+        //QUEUE
+        StringBuilder builder = StringBuilderHelper.get()
+                                                   .append(QUEUE).append(DELIMITER);
+        if (!StringUtils.isEmpty(itemId)) {
+            builder.append(itemId).append(DELIMITER);
+        }
+        return builder.toString().getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getSplitTaskKey(String graphName, int groupId) {
+        // TASK_SPLIT/{GraphName}/{partitionID}
+        StringBuilder builder = StringBuilderHelper.get()
+                                                   .append(TASK_SPLIT).append(DELIMITER)
+                                                   .append(graphName).append(DELIMITER)
+                                                   .append(groupId);
+        return builder.toString().getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getSplitTaskPrefix(String graphName) {
+        // TASK_SPLIT/{GraphName}/
+        StringBuilder builder = StringBuilderHelper.get()
+                                                   .append(TASK_SPLIT).append(DELIMITER)
+                                                   .append(graphName);
+        return builder.toString().getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getAllSplitTaskPrefix() {
+        // TASK_SPLIT/{GraphName}/
+        StringBuilder builder = StringBuilderHelper.get()
+                                                   .append(TASK_SPLIT).append(DELIMITER);
+        return builder.toString().getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getMoveTaskKey(String graphName, int targetGroupId, int groupId) {
+        // TASK_MOVE/{GraphName}/to PartitionID/{source partitionID}
+        StringBuilder builder = StringBuilderHelper.get()
+                                                   .append(TASK_MOVE).append(DELIMITER)
+                                                   .append(graphName).append(DELIMITER)
+                                                   .append(targetGroupId).append(DELIMITER)
+                                                   .append(groupId);
+        return builder.toString().getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getMoveTaskPrefix(String graphName) {
+        // TASK_MOVE/{graphName}/toPartitionId/
+        StringBuilder builder = StringBuilderHelper.get()
+                                                   .append(TASK_MOVE).append(DELIMITER)
+                                                   .append(graphName);
+        return builder.toString().getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getAllMoveTaskPrefix() {
+        // TASK_MOVE/{graphName}/toPartitionId/
+        StringBuilder builder = StringBuilderHelper.get()
+                                                   .append(TASK_MOVE).append(DELIMITER);
+        return builder.toString().getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getLogKey(Metapb.LogRecord record) {
+        //LOG_RECORD/{action}/{time}/
+        StringBuilder builder = StringBuilderHelper.get()
+                                                   .append(LOG_RECORD)
+                                                   .append(DELIMITER)
+                                                   .append(record.getAction())
+                                                   .append(DELIMITER)
+                                                   .append(record.getTimestamp());
+        return builder.toString().getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getLogKeyPrefix(String action, long time) {
+        //LOG_DATA_SPLIT/{time}/{GraphName}
+        StringBuilder builder = StringBuilderHelper.get()
+                                                   .append(LOG_RECORD)
+                                                   .append(DELIMITER)
+                                                   .append(action)
+                                                   .append(DELIMITER)
+                                                   .append(time);
+        return builder.toString().getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getKVPrefix(String prefix, String key) {
+        //K@/{key}
+        StringBuilder builder = StringBuilderHelper.get()
+                                                   .append(prefix).append(DELIMITER);
+        if (!StringUtils.isEmpty(key)) {
+            builder.append(key).append(DELIMITER);
+        }
+        return builder.toString().getBytes(Charset.defaultCharset());
+    }
+
+    public static byte[] getKVTTLPrefix(String ttlPrefix, String prefix, String key) {
+        StringBuilder builder = StringBuilderHelper.get().append(ttlPrefix)
+                                                   .append(prefix).append(DELIMITER);
+        if (!StringUtils.isEmpty(key)) {
+            builder.append(key).append(DELIMITER);
+        }
+        return builder.toString().getBytes(Charset.defaultCharset());
+    }
+
+    public static String getKVWatchKeyPrefix(String key, String watchDelimiter, long clientId) {
+        StringBuilder builder = StringBuilderHelper.get();
+        builder.append(watchDelimiter).append(DELIMITER);
+        builder.append(key == null ? "" : key).append(DELIMITER);
+        builder.append(clientId);
+        return builder.toString();
+    }
+
+    public static String getKVWatchKeyPrefix(String key, String watchDelimiter) {
+        StringBuilder builder = StringBuilderHelper.get();
+        builder.append(watchDelimiter).append(DELIMITER);
+        builder.append(key == null ? "" : key).append(DELIMITER);
+        return builder.toString();
+    }
+
+    public static char getDelimiter() {
+        return DELIMITER;
+    }
+
+    public static StringBuilder getStringBuilderHelper() {
+        return StringBuilderHelper.get();
+    }
+
+    static class StringBuilderHelper {
+        private static final int DISCARD_LIMIT = 1024 << 3;     // 8k
+
+        private static final ThreadLocal<StringBuilderHolder> holderThreadLocal = ThreadLocal
+                .withInitial(StringBuilderHolder::new);
+
+        public static StringBuilder get() {
+            final StringBuilderHolder holder = holderThreadLocal.get();
+            return holder.getStringBuilder();
+        }
+
+        public static void truncate() {
+            final StringBuilderHolder holder = holderThreadLocal.get();
+            holder.truncate();
+        }
+
+        private static class StringBuilderHolder {
+
+            private final StringBuilder buf = new StringBuilder();
+
+            private StringBuilder getStringBuilder() {
+                truncate();
+                return buf;
+            }
+
+            private void truncate() {
+                buf.setLength(0);
+            }
+        }
+    }
+
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataRocksDBStore.java

@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.meta;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.Pdpb;
+import org.apache.hugegraph.pd.store.HgKVStore;
+import org.apache.hugegraph.pd.store.KV;
+
+import com.google.protobuf.Parser;
+
+public class MetadataRocksDBStore extends MetadataStoreBase {
+
+    HgKVStore store;
+
+    PDConfig pdConfig;
+
+    public MetadataRocksDBStore(PDConfig pdConfig) {
+        store = MetadataFactory.getStore(pdConfig);
+        this.pdConfig = pdConfig;
+    }
+
+    public HgKVStore getStore() {
+        if (store == null) {
+            store = MetadataFactory.getStore(pdConfig);
+        }
+        return store;
+    }
+
+    @Override
+    public byte[] getOne(byte[] key) throws PDException {
+        try {
+            byte[] bytes = store.get(key);
+            return bytes;
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_READ_ERROR_VALUE, e);
+        }
+    }
+
+    @Override
+    public <E> E getOne(Parser<E> parser, byte[] key) throws PDException {
+        try {
+            byte[] bytes = store.get(key);
+            if (ArrayUtils.isEmpty(bytes)) {
+                return null;
+            }
+            return parser.parseFrom(bytes);
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_READ_ERROR_VALUE, e);
+        }
+    }
+
+    @Override
+    public void put(byte[] key, byte[] value) throws PDException {
+        try {
+            getStore().put(key, value);
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_WRITE_ERROR_VALUE, e);
+        }
+    }
+
+    @Override
+    public void putWithTTL(byte[] key, byte[] value, long ttl) throws PDException {
+        this.store.putWithTTL(key, value, ttl);
+    }
+
+    @Override
+    public void putWithTTL(byte[] key, byte[] value, long ttl, TimeUnit timeUnit) throws
+                                                                                  PDException {
+        this.store.putWithTTL(key, value, ttl, timeUnit);
+    }
+
+    @Override
+    public byte[] getWithTTL(byte[] key) throws PDException {
+        return this.store.getWithTTL(key);
+    }
+
+    @Override
+    public List getListWithTTL(byte[] key) throws PDException {
+        return this.store.getListWithTTL(key);
+    }
+
+    @Override
+    public void removeWithTTL(byte[] key) throws PDException {
+        this.store.removeWithTTL(key);
+    }
+
+    @Override
+    public List<KV> scanPrefix(byte[] prefix) throws PDException {
+        //TODO 使用rocksdb 前缀查询
+        try {
+            return this.store.scanPrefix(prefix);
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_READ_ERROR_VALUE, e);
+        }
+    }
+
+    @Override
+    public List<KV> scanRange(byte[] start, byte[] end) throws PDException {
+        return this.store.scanRange(start, end);
+    }
+
+    @Override
+    public <E> List<E> scanRange(Parser<E> parser, byte[] start, byte[] end) throws PDException {
+        List<E> stores = new LinkedList<>();
+        try {
+            List<KV> kvs = this.scanRange(start, end);
+            for (KV keyValue : kvs) {
+                stores.add(parser.parseFrom(keyValue.getValue()));
+            }
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_READ_ERROR_VALUE, e);
+        }
+        return stores;
+    }
+
+    @Override
+    public <E> List<E> scanPrefix(Parser<E> parser, byte[] prefix) throws PDException {
+        List<E> stores = new LinkedList<>();
+        try {
+            List<KV> kvs = this.scanPrefix(prefix);
+            for (KV keyValue : kvs) {
+                stores.add(parser.parseFrom(keyValue.getValue()));
+            }
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_READ_ERROR_VALUE, e);
+        }
+        return stores;
+    }
+
+    @Override
+    public boolean containsKey(byte[] key) throws PDException {
+        return !ArrayUtils.isEmpty(store.get(key));
+    }
+
+    @Override
+    public long remove(byte[] key) throws PDException {
+        try {
+            return this.store.remove(key);
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_WRITE_ERROR_VALUE, e);
+        }
+    }
+
+    @Override
+    public long removeByPrefix(byte[] prefix) throws PDException {
+        try {
+            return this.store.removeByPrefix(prefix);
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_WRITE_ERROR_VALUE, e);
+        }
+    }
+
+    @Override
+    public void clearAllCache() throws PDException {
+        this.store.clear();
+    }
+
+    @Override
+    public void close() {
+
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/MetadataStoreBase.java

@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.meta;
+
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.grpc.Pdpb;
+import org.apache.hugegraph.pd.store.KV;
+
+import com.google.protobuf.Parser;
+
+public abstract class MetadataStoreBase {
+
+    //  public long timeout = 3; // 请求超时时间，默认三秒
+
+    public abstract byte[] getOne(byte[] key) throws PDException;
+
+    public abstract <E> E getOne(Parser<E> parser, byte[] key) throws PDException;
+
+    public abstract void put(byte[] key, byte[] value) throws PDException;
+
+    /**
+     * 带有过期时间的put
+     */
+
+    public abstract void putWithTTL(byte[] key,
+                                    byte[] value,
+                                    long ttl) throws PDException;
+
+    public abstract void putWithTTL(byte[] key,
+                                    byte[] value,
+                                    long ttl, TimeUnit timeUnit) throws PDException;
+
+    public abstract byte[] getWithTTL(byte[] key) throws PDException;
+
+    public abstract List getListWithTTL(byte[] key) throws PDException;
+
+    public abstract void removeWithTTL(byte[] key) throws PDException;
+
+    /**
+     * 前缀查询
+     *
+     * @param prefix
+     * @return
+     * @throws PDException
+     */
+    public abstract List<KV> scanPrefix(byte[] prefix) throws PDException;
+
+    /**
+     * 前缀查询
+     *
+     * @param prefix
+     * @return
+     * @throws PDException
+     */
+
+    public abstract <E> List<E> scanPrefix(Parser<E> parser, byte[] prefix) throws PDException;
+
+    public abstract List<KV> scanRange(byte[] start, byte[] end) throws PDException;
+
+    public abstract <E> List<E> scanRange(Parser<E> parser, byte[] start, byte[] end) throws
+                                                                                      PDException;
+
+    /**
+     * 检查Key是否存在
+     *
+     * @param key
+     * @return
+     * @throws PDException
+     */
+
+    public abstract boolean containsKey(byte[] key) throws PDException;
+
+    public abstract long remove(byte[] key) throws PDException;
+
+    public abstract long removeByPrefix(byte[] prefix) throws PDException;
+
+    public abstract void clearAllCache() throws PDException;
+
+    public abstract void close() throws IOException;
+
+    public <T> T getInstanceWithTTL(Parser<T> parser, byte[] key) throws PDException {
+        try {
+            byte[] withTTL = this.getWithTTL(key);
+            return parser.parseFrom(withTTL);
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_READ_ERROR_VALUE, e);
+        }
+    }
+
+    public <T> List<T> getInstanceListWithTTL(Parser<T> parser, byte[] key)
+            throws PDException {
+        try {
+            List withTTL = this.getListWithTTL(key);
+            LinkedList<T> ts = new LinkedList<>();
+            for (int i = 0; i < withTTL.size(); i++) {
+                ts.add(parser.parseFrom((byte[]) withTTL.get(i)));
+            }
+            return ts;
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_READ_ERROR_VALUE, e);
+        }
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/PartitionMeta.java

@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.meta;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.common.PartitionCache;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.Metapb;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * 分区信息管理
+ */
+@Slf4j
+public class PartitionMeta extends MetadataRocksDBStore {
+    static String CID_GRAPH_ID_KEY = "GraphID";
+    static int CID_GRAPH_ID_MAX = 0xFFFE;
+    private final PDConfig pdConfig;
+    private final PartitionCache cache;
+
+    public PartitionMeta(PDConfig pdConfig) {
+        super(pdConfig);
+        this.pdConfig = pdConfig;
+        //this.timeout = pdConfig.getEtcd().getTimeout();
+        this.cache = new PartitionCache();
+    }
+
+    /**
+     * 初始化，加载所有的分区
+     */
+    public void init() throws PDException {
+        loadShardGroups();
+        loadGraphs();
+    }
+
+    public void reload() throws PDException {
+        cache.clear();
+        loadShardGroups();
+        loadGraphs();
+    }
+
+    private void loadGraphs() throws PDException {
+        byte[] key = MetadataKeyHelper.getGraphPrefix();
+        List<Metapb.Graph> graphs = scanPrefix(Metapb.Graph.parser(), key);
+        for (Metapb.Graph graph : graphs) {
+            cache.updateGraph(graph);
+            loadPartitions(graph);
+        }
+    }
+
+    /**
+     * partition 和 shard group分开存储，再init的时候，需要加载进来
+     *
+     * @throws PDException
+     */
+    private void loadShardGroups() throws PDException {
+        byte[] shardGroupPrefix = MetadataKeyHelper.getShardGroupPrefix();
+        for (var shardGroup : scanPrefix(Metapb.ShardGroup.parser(), shardGroupPrefix)) {
+            cache.updateShardGroup(shardGroup);
+        }
+    }
+
+    private void loadPartitions(Metapb.Graph graph) throws PDException {
+        byte[] prefix = MetadataKeyHelper.getPartitionPrefix(graph.getGraphName());
+        List<Metapb.Partition> partitions = scanPrefix(Metapb.Partition.parser(), prefix);
+        partitions.forEach(p -> {
+            cache.updatePartition(p);
+        });
+    }
+
+    /**
+     * 根据id查找分区 (先从缓存找，再到数据库中找）
+     *
+     * @param graphName
+     * @param partId
+     * @return
+     * @throws PDException
+     */
+    public Metapb.Partition getPartitionById(String graphName, int partId) throws PDException {
+        var pair = cache.getPartitionById(graphName, partId);
+        Metapb.Partition partition;
+        if (pair == null) {
+            byte[] key = MetadataKeyHelper.getPartitionKey(graphName, partId);
+            partition = getOne(Metapb.Partition.parser(), key);
+            if (partition != null) {
+                cache.updatePartition(partition);
+            }
+        } else {
+            partition = pair.getKey();
+        }
+        return partition;
+    }
+
+    public List<Metapb.Partition> getPartitionById(int partId) throws PDException {
+        List<Metapb.Partition> partitions = new ArrayList<>();
+        cache.getGraphs().forEach(graph -> {
+            cache.getPartitions(graph.getGraphName()).forEach(partition -> {
+                if (partition.getId() == partId) {
+                    partitions.add(partition);
+                }
+            });
+        });
+        return partitions;
+    }
+
+    /**
+     * 根据code查找分区
+     */
+    public Metapb.Partition getPartitionByCode(String graphName, long code) throws PDException {
+        var pair = cache.getPartitionByCode(graphName, code);
+        if (pair != null) {
+            return pair.getKey();
+        }
+        return null;
+    }
+
+    public Metapb.Graph getAndCreateGraph(String graphName) throws PDException {
+        return getAndCreateGraph(graphName, pdConfig.getPartition().getTotalCount());
+    }
+
+    public Metapb.Graph getAndCreateGraph(String graphName, int partitionCount) throws PDException {
+
+        if (partitionCount > pdConfig.getPartition().getTotalCount()) {
+            partitionCount = pdConfig.getPartition().getTotalCount();
+        }
+
+        // 管理图，只有一个分区
+        if (graphName.endsWith("/s") || graphName.endsWith("/m")) {
+            partitionCount = 1;
+        }
+
+        Metapb.Graph graph = cache.getGraph(graphName);
+        if (graph == null) {
+            // 保存图信息
+            graph = Metapb.Graph.newBuilder()
+                                .setGraphName(graphName)
+                                .setPartitionCount(partitionCount)
+                                .setState(Metapb.PartitionState.PState_Normal)
+                                .build();
+            updateGraph(graph);
+        }
+        return graph;
+    }
+
+    /**
+     * 保存分区信息
+     *
+     * @param partition
+     * @return
+     * @throws PDException
+     */
+    public Metapb.Partition updatePartition(Metapb.Partition partition) throws PDException {
+        if (!cache.hasGraph(partition.getGraphName())) {
+            getAndCreateGraph(partition.getGraphName());
+        }
+        byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId());
+        put(key, partition.toByteString().toByteArray());
+        cache.updatePartition(partition);
+        return partition;
+    }
+
+    /**
+     * 检查数据库，是否存在对应的图，不存在，则创建。
+     * 更新partition的 version, conf version 和 shard list
+     *
+     * @param partition
+     * @return
+     * @throws PDException
+     */
+    public Metapb.Partition updateShardList(Metapb.Partition partition) throws PDException {
+        if (!cache.hasGraph(partition.getGraphName())) {
+            getAndCreateGraph(partition.getGraphName());
+        }
+
+        Metapb.Partition pt = getPartitionById(partition.getGraphName(), partition.getId());
+        //  pt = pt.toBuilder().setVersion(partition.getVersion())
+        //        .setConfVer(partition.getConfVer())
+        //        .clearShards()
+        //        .addAllShards(partition.getShardsList()).build();
+
+        byte[] key = MetadataKeyHelper.getPartitionKey(pt.getGraphName(), pt.getId());
+        put(key, pt.toByteString().toByteArray());
+        cache.updatePartition(pt);
+        return partition;
+    }
+
+    /**
+     * 删除所有分区
+     */
+    public long removeAllPartitions(String graphName) throws PDException {
+        cache.removeAll(graphName);
+        byte[] prefix = MetadataKeyHelper.getPartitionPrefix(graphName);
+        return removeByPrefix(prefix);
+    }
+
+    public long removePartition(String graphName, int id) throws PDException {
+        cache.remove(graphName, id);
+        byte[] key = MetadataKeyHelper.getPartitionKey(graphName, id);
+        return remove(key);
+    }
+
+    public void updatePartitionStats(Metapb.PartitionStats stats) throws PDException {
+        for (String graphName : stats.getGraphNameList()) {
+            byte[] prefix = MetadataKeyHelper.getPartitionStatusKey(graphName, stats.getId());
+            put(prefix, stats.toByteArray());
+        }
+    }
+
+    /**
+     * 获取分区状态
+     */
+    public Metapb.PartitionStats getPartitionStats(String graphName, int id) throws PDException {
+        byte[] prefix = MetadataKeyHelper.getPartitionStatusKey(graphName, id);
+        return getOne(Metapb.PartitionStats.parser(), prefix);
+    }
+
+
+    /**
+     * 获取分区状态
+     */
+    public List<Metapb.PartitionStats> getPartitionStats(String graphName) throws PDException {
+        byte[] prefix = MetadataKeyHelper.getPartitionStatusPrefixKey(graphName);
+        return scanPrefix(Metapb.PartitionStats.parser(), prefix);
+    }
+
+    /**
+     * 更新图信息
+     *
+     * @param graph
+     * @return
+     */
+    public Metapb.Graph updateGraph(Metapb.Graph graph) throws PDException {
+        log.info("updateGraph {}", graph);
+        byte[] key = MetadataKeyHelper.getGraphKey(graph.getGraphName());
+        // 保存图信息
+        put(key, graph.toByteString().toByteArray());
+        cache.updateGraph(graph);
+        return graph;
+    }
+
+    public List<Metapb.Partition> getPartitions() {
+        List<Metapb.Partition> partitions = new ArrayList<>();
+        List<Metapb.Graph> graphs = cache.getGraphs();
+        graphs.forEach(e -> {
+            partitions.addAll(cache.getPartitions(e.getGraphName()));
+        });
+        return partitions;
+    }
+
+    public List<Metapb.Partition> getPartitions(String graphName) {
+        return cache.getPartitions(graphName);
+    }
+
+    public List<Metapb.Graph> getGraphs() throws PDException {
+        byte[] key = MetadataKeyHelper.getGraphPrefix();
+        return scanPrefix(Metapb.Graph.parser(), key);
+    }
+
+    public Metapb.Graph getGraph(String graphName) throws PDException {
+        byte[] key = MetadataKeyHelper.getGraphKey(graphName);
+        return getOne(Metapb.Graph.parser(), key);
+    }
+
+    /**
+     * 删除图，并删除图id
+     */
+    public long removeGraph(String graphName) throws PDException {
+        byte[] key = MetadataKeyHelper.getGraphKey(graphName);
+        long l = remove(key);
+        return l;
+    }
+
+    public PartitionCache getPartitionCache() {
+        return cache;
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/QueueStore.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.meta;
+
+import java.util.List;
+
+import org.apache.hugegraph.pd.common.HgAssert;
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.apache.hugegraph.pd.raft.RaftEngine;
+import org.apache.hugegraph.pd.store.RaftKVStore;
+
+public class QueueStore extends MetadataRocksDBStore {
+    QueueStore(PDConfig pdConfig) {
+        super(pdConfig);
+    }
+
+    public void addItem(Metapb.QueueItem queueItem) throws PDException {
+        HgAssert.isArgumentNotNull(queueItem, "queueItem");
+        byte[] key = MetadataKeyHelper.getQueueItemKey(queueItem.getItemId());
+        put(key, queueItem.toByteString().toByteArray());
+    }
+
+    public void removeItem(String itemId) throws PDException {
+        if (RaftEngine.getInstance().isLeader()) {
+            remove(MetadataKeyHelper.getQueueItemKey(itemId));
+        } else {
+            var store = getStore();
+            // todo: delete record via client
+            if (store instanceof RaftKVStore) {
+                ((RaftKVStore) store).doRemove(MetadataKeyHelper.getQueueItemKey(itemId));
+            }
+        }
+    }
+
+    public List<Metapb.QueueItem> getQueue() throws PDException {
+        byte[] prefix = MetadataKeyHelper.getQueueItemPrefix();
+        return scanPrefix(Metapb.QueueItem.parser(), prefix);
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/StoreInfoMeta.java

@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.meta;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.Metapb;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Store信息存储
+ */
+@Slf4j
+public class StoreInfoMeta extends MetadataRocksDBStore {
+    private final PDConfig pdConfig;
+
+    public StoreInfoMeta(PDConfig pdConfig) {
+        super(pdConfig);
+        this.pdConfig = pdConfig;
+        //   this.timeout = pdConfig.getDiscovery().getHeartbeatOutTimes();
+    }
+
+    public static boolean shardGroupEquals(List<Metapb.Shard> g1, List<Metapb.Shard> g2) {
+        ListIterator<Metapb.Shard> e1 = g1.listIterator();
+        ListIterator<Metapb.Shard> e2 = g2.listIterator();
+        while (e1.hasNext() && e2.hasNext()) {
+            Metapb.Shard o1 = e1.next();
+            Metapb.Shard o2 = e2.next();
+            if (!(o1 == null ? o2 == null : o1.getStoreId() == o2.getStoreId())) {
+                return false;
+            }
+        }
+        return !(e1.hasNext() || e2.hasNext());
+    }
+
+    /**
+     * 更新Store信息
+     *
+     * @param store
+     * @throws PDException
+     */
+    public void updateStore(Metapb.Store store) throws PDException {
+        byte[] storeInfoKey = MetadataKeyHelper.getStoreInfoKey(store.getId());
+        put(storeInfoKey, store.toByteArray());
+    }
+
+    /**
+     * 更新Store的存活状态
+     *
+     * @param store
+     */
+    public void keepStoreAlive(Metapb.Store store) throws PDException {
+        byte[] activeStoreKey = MetadataKeyHelper.getActiveStoreKey(store.getId());
+        putWithTTL(activeStoreKey, store.toByteArray(), pdConfig.getStore().getKeepAliveTimeout());
+    }
+
+    public void removeActiveStore(Metapb.Store store) throws PDException {
+        byte[] activeStoreKey = MetadataKeyHelper.getActiveStoreKey(store.getId());
+        removeWithTTL(activeStoreKey);
+    }
+
+    public Metapb.Store getStore(Long storeId) throws PDException {
+        byte[] storeInfoKey = MetadataKeyHelper.getStoreInfoKey(storeId);
+        Metapb.Store store = getOne(Metapb.Store.parser(), storeInfoKey);
+        return store;
+    }
+
+    /**
+     * 获取所有的store
+     *
+     * @param graphName
+     * @return
+     * @throws PDException
+     */
+    public List<Metapb.Store> getStores(String graphName) throws PDException {
+        byte[] storePrefix = MetadataKeyHelper.getStorePrefix();
+        return scanPrefix(Metapb.Store.parser(), storePrefix);
+    }
+
+    /**
+     * 获取活跃的Store
+     *
+     * @param graphName
+     * @return
+     * @throws PDException
+     */
+    public List<Metapb.Store> getActiveStores(String graphName) throws PDException {
+        byte[] activePrefix = MetadataKeyHelper.getActiveStorePrefix();
+        List listWithTTL = getInstanceListWithTTL(Metapb.Store.parser(),
+                                                  activePrefix);
+        return listWithTTL;
+    }
+
+    public List<Metapb.Store> getActiveStores() throws PDException {
+        byte[] activePrefix = MetadataKeyHelper.getActiveStorePrefix();
+        List listWithTTL = getInstanceListWithTTL(Metapb.Store.parser(),
+                                                  activePrefix);
+        return listWithTTL;
+    }
+
+    /**
+     * 检查storeid是否存在
+     *
+     * @param storeId
+     * @return
+     */
+    public boolean storeExists(Long storeId) throws PDException {
+        byte[] storeInfoKey = MetadataKeyHelper.getStoreInfoKey(storeId);
+        return containsKey(storeInfoKey);
+    }
+
+    /**
+     * 更新存储状态信息
+     *
+     * @param storeStats
+     */
+    public Metapb.StoreStats updateStoreStats(Metapb.StoreStats storeStats) throws PDException {
+        byte[] storeStatusKey = MetadataKeyHelper.getStoreStatusKey(storeStats.getStoreId());
+
+        put(storeStatusKey, storeStats.toByteArray());
+        return storeStats;
+    }
+
+    public long removeStore(long storeId) throws PDException {
+        byte[] storeInfoKey = MetadataKeyHelper.getStoreInfoKey(storeId);
+        return remove(storeInfoKey);
+    }
+
+    public long removeAll() throws PDException {
+        byte[] storePrefix = MetadataKeyHelper.getStorePrefix();
+        return this.removeByPrefix(storePrefix);
+    }
+
+    public void updateShardGroup(Metapb.ShardGroup group) throws PDException {
+        byte[] shardGroupKey = MetadataKeyHelper.getShardGroupKey(group.getId());
+        put(shardGroupKey, group.toByteArray());
+    }
+
+    public void deleteShardGroup(int groupId) throws PDException {
+        byte[] shardGroupKey = MetadataKeyHelper.getShardGroupKey(groupId);
+        remove(shardGroupKey);
+    }
+
+    public Metapb.ShardGroup getShardGroup(int groupId) throws PDException {
+        byte[] shardGroupKey = MetadataKeyHelper.getShardGroupKey(groupId);
+        return getOne(Metapb.ShardGroup.parser(), shardGroupKey);
+    }
+
+    public int getShardGroupCount() throws PDException {
+        byte[] shardGroupPrefix = MetadataKeyHelper.getShardGroupPrefix();
+        return scanPrefix(Metapb.ShardGroup.parser(), shardGroupPrefix).size();
+    }
+
+    public List<Metapb.ShardGroup> getShardGroups() throws PDException {
+        byte[] shardGroupPrefix = MetadataKeyHelper.getShardGroupPrefix();
+        return scanPrefix(Metapb.ShardGroup.parser(), shardGroupPrefix);
+    }
+
+    public Metapb.StoreStats getStoreStats(long storeId) throws PDException {
+        byte[] storeStatusKey = MetadataKeyHelper.getStoreStatusKey(storeId);
+        Metapb.StoreStats stats = getOne(Metapb.StoreStats.parser(),
+                                         storeStatusKey);
+        return stats;
+    }
+
+    /**
+     * @return store及状态信息
+     * @throws PDException
+     */
+    public List<Metapb.Store> getStoreStatus(boolean isActive) throws PDException {
+        byte[] storePrefix = MetadataKeyHelper.getStorePrefix();
+        List<Metapb.Store> stores = isActive ? getActiveStores() :
+                                    scanPrefix(Metapb.Store.parser(), storePrefix);
+        LinkedList<Metapb.Store> list = new LinkedList<>();
+        for (int i = 0; i < stores.size(); i++) {
+            Metapb.Store store = stores.get(i);
+            Metapb.StoreStats stats = getStoreStats(store.getId());
+            if (stats != null) {
+                store = Metapb.Store.newBuilder(store).setStats(getStoreStats(store.getId()))
+                                    .build();
+            }
+            list.add(store);
+        }
+        return list;
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/TaskInfoMeta.java

@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.meta;
+
+import java.util.List;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.MetaTask;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.apache.hugegraph.pd.grpc.pulse.MovePartition;
+import org.apache.hugegraph.pd.grpc.pulse.SplitPartition;
+
+/**
+ * 任务管理
+ */
+public class TaskInfoMeta extends MetadataRocksDBStore {
+    public TaskInfoMeta(PDConfig pdConfig) {
+        super(pdConfig);
+    }
+
+    /**
+     * 添加分区分裂任务
+     */
+    public void addSplitTask(int groupID, Metapb.Partition partition, SplitPartition splitPartition)
+            throws PDException {
+        byte[] key = MetadataKeyHelper.getSplitTaskKey(partition.getGraphName(), groupID);
+        MetaTask.Task task = MetaTask.Task.newBuilder()
+                                          .setType(MetaTask.TaskType.Split_Partition)
+                                          .setState(MetaTask.TaskState.Task_Doing)
+                                          .setStartTimestamp(System.currentTimeMillis())
+                                          .setPartition(partition)
+                                          .setSplitPartition(splitPartition)
+                                          .build();
+        put(key, task.toByteString().toByteArray());
+    }
+
+    public void updateSplitTask(MetaTask.Task task) throws PDException {
+        var partition = task.getPartition();
+        byte[] key = MetadataKeyHelper.getSplitTaskKey(partition.getGraphName(), partition.getId());
+        put(key, task.toByteString().toByteArray());
+    }
+
+    public MetaTask.Task getSplitTask(String graphName, int groupID) throws PDException {
+        byte[] key = MetadataKeyHelper.getSplitTaskKey(graphName, groupID);
+        return getOne(MetaTask.Task.parser(), key);
+    }
+
+    public List<MetaTask.Task> scanSplitTask(String graphName) throws PDException {
+        byte[] prefix = MetadataKeyHelper.getSplitTaskPrefix(graphName);
+        return scanPrefix(MetaTask.Task.parser(), prefix);
+    }
+
+    public void removeSplitTaskPrefix(String graphName) throws PDException {
+        byte[] key = MetadataKeyHelper.getSplitTaskPrefix(graphName);
+        removeByPrefix(key);
+    }
+
+    public boolean hasSplitTaskDoing() throws PDException {
+        byte[] key = MetadataKeyHelper.getAllSplitTaskPrefix();
+        return scanPrefix(key).size() > 0;
+    }
+
+    public void addMovePartitionTask(Metapb.Partition partition, MovePartition movePartition)
+            throws PDException {
+        byte[] key = MetadataKeyHelper.getMoveTaskKey(partition.getGraphName(),
+                                                      movePartition.getTargetPartition().getId(),
+                                                      partition.getId());
+
+        MetaTask.Task task = MetaTask.Task.newBuilder()
+                                          .setType(MetaTask.TaskType.Move_Partition)
+                                          .setState(MetaTask.TaskState.Task_Doing)
+                                          .setStartTimestamp(System.currentTimeMillis())
+                                          .setPartition(partition)
+                                          .setMovePartition(movePartition)
+                                          .build();
+        put(key, task.toByteArray());
+    }
+
+    public void updateMovePartitionTask(MetaTask.Task task)
+            throws PDException {
+
+        byte[] key = MetadataKeyHelper.getMoveTaskKey(task.getPartition().getGraphName(),
+                                                      task.getMovePartition().getTargetPartition()
+                                                          .getId(),
+                                                      task.getPartition().getId());
+        put(key, task.toByteArray());
+    }
+
+    public MetaTask.Task getMovePartitionTask(String graphName, int targetId, int partId) throws
+                                                                                          PDException {
+        byte[] key = MetadataKeyHelper.getMoveTaskKey(graphName, targetId, partId);
+        return getOne(MetaTask.Task.parser(), key);
+    }
+
+    public List<MetaTask.Task> scanMoveTask(String graphName) throws PDException {
+        byte[] prefix = MetadataKeyHelper.getMoveTaskPrefix(graphName);
+        return scanPrefix(MetaTask.Task.parser(), prefix);
+    }
+
+    /**
+     * 按照prefix删除迁移任务，一次分组的
+     *
+     * @param graphName 图名称
+     * @throws PDException io error
+     */
+    public void removeMoveTaskPrefix(String graphName) throws PDException {
+        byte[] key = MetadataKeyHelper.getMoveTaskPrefix(graphName);
+        removeByPrefix(key);
+    }
+
+    public boolean hasMoveTaskDoing() throws PDException {
+        byte[] key = MetadataKeyHelper.getAllMoveTaskPrefix();
+        return scanPrefix(key).size() > 0;
+    }
+
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/FutureClosureAdapter.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.raft;
+
+import java.util.concurrent.CompletableFuture;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Status;
+
+public class FutureClosureAdapter<T> implements Closure {
+    public final CompletableFuture<T> future = new CompletableFuture<>();
+    private T resp;
+
+    public T getResponse() {
+        return this.resp;
+    }
+
+    public void setResponse(T resp) {
+        this.resp = resp;
+        future.complete(resp);
+        run(Status.OK());
+    }
+
+    public void failure(Throwable t) {
+        future.completeExceptionally(t);
+        run(new Status(-1, t.getMessage()));
+    }
+
+    @Override
+    public void run(Status status) {
+
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/KVOperation.java

@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.raft;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.Requires;
+import com.caucho.hessian.io.Hessian2Input;
+import com.caucho.hessian.io.Hessian2Output;
+
+import lombok.Data;
+
+@Data
+public class KVOperation {
+
+    /**
+     * Put operation
+     */
+    public static final byte PUT = 0x01;
+    /**
+     * Get operation
+     */
+    public static final byte GET = 0x02;
+    public static final byte DEL = 0x03;
+    public static final byte REMOVE_BY_PREFIX = 0x04;
+    public static final byte REMOVE = 0x05;
+    public static final byte PUT_WITH_TTL = 0x06;
+    public static final byte CLEAR = 0x07;
+    public static final byte PUT_WITH_TTL_UNIT = 0x08;
+    public static final byte REMOVE_WITH_TTL = 0x09;
+    /**
+     * Snapshot operation
+     */
+    public static final byte SAVE_SNAPSHOT = 0x10;
+    public static final byte LOAD_SNAPSHOT = 0x11;
+
+    private byte[] key;
+    private byte[] value;
+    private Object attach; // 原始对象，用于本机处理，减少一次反序列化操作
+    private Object arg;
+    private byte op;
+
+    public KVOperation() {
+
+    }
+
+    public KVOperation(byte[] key, byte[] value, Object attach, byte op) {
+        this.key = key;
+        this.value = value;
+        this.attach = attach;
+        this.op = op;
+    }
+
+    public KVOperation(byte[] key, byte[] value, Object attach, byte op, Object arg) {
+        this.key = key;
+        this.value = value;
+        this.attach = attach;
+        this.op = op;
+        this.arg = arg;
+    }
+
+    public static KVOperation fromByteArray(byte[] value) throws IOException {
+
+        try (ByteArrayInputStream bis = new ByteArrayInputStream(value, 1, value.length - 1)) {
+            Hessian2Input input = new Hessian2Input(bis);
+            KVOperation op = new KVOperation();
+            op.op = value[0];
+            op.key = input.readBytes();
+            op.value = input.readBytes();
+            op.arg = input.readObject();
+            input.close();
+            return op;
+        }
+    }
+
+    public static KVOperation createPut(final byte[] key, final byte[] value) {
+        Requires.requireNonNull(key, "key");
+        Requires.requireNonNull(value, "value");
+        return new KVOperation(key, value, null, PUT);
+    }
+
+    public static KVOperation createGet(final byte[] key) {
+        Requires.requireNonNull(key, "key");
+        return new KVOperation(key, BytesUtil.EMPTY_BYTES, null, GET);
+    }
+
+    public static KVOperation createPutWithTTL(byte[] key, byte[] value, long ttl) {
+        Requires.requireNonNull(key, "key");
+        Requires.requireNonNull(value, "value");
+        return new KVOperation(key, value, value, PUT_WITH_TTL,
+                               ttl);
+    }
+
+    public static KVOperation createPutWithTTL(byte[] key, byte[] value, long ttl,
+                                               TimeUnit timeUnit) {
+        Requires.requireNonNull(key, "key");
+        Requires.requireNonNull(value, "value");
+        return new KVOperation(key, value, value, PUT_WITH_TTL_UNIT,
+                               new Object[]{ttl, timeUnit});
+    }
+
+    public static KVOperation createRemoveWithTTL(byte[] key) {
+        Requires.requireNonNull(key, "key");
+        return new KVOperation(key, key, null, REMOVE_WITH_TTL);
+    }
+
+    public static KVOperation createRemoveByPrefix(byte[] key) {
+        Requires.requireNonNull(key, "key");
+        return new KVOperation(key, key, null, REMOVE_BY_PREFIX);
+    }
+
+    public static KVOperation createRemove(byte[] key) {
+        Requires.requireNonNull(key, "key");
+        return new KVOperation(key, key, null, REMOVE);
+    }
+
+    public static KVOperation createClear() {
+        return new KVOperation(null, null, null, CLEAR);
+    }
+
+    public static KVOperation createSaveSnapshot(String snapshotPath) {
+        Requires.requireNonNull(snapshotPath, "snapshotPath");
+        return new KVOperation(null, null, snapshotPath, SAVE_SNAPSHOT);
+    }
+
+    public static KVOperation createLoadSnapshot(String snapshotPath) {
+        Requires.requireNonNull(snapshotPath, "snapshotPath");
+        return new KVOperation(null, null, snapshotPath, LOAD_SNAPSHOT);
+    }
+
+    public byte[] toByteArray() throws IOException {
+        try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
+            bos.write(op);
+            Hessian2Output output = new Hessian2Output(bos);
+            output.writeObject(key);
+            output.writeObject(value);
+            output.writeObject(arg);
+            output.flush();
+            return bos.toByteArray();
+        }
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/KVStoreClosure.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.raft;
+
+import org.apache.hugegraph.pd.grpc.Pdpb;
+
+import com.alipay.sofa.jraft.Closure;
+
+public interface KVStoreClosure extends Closure {
+
+    Pdpb.Error getError();
+
+    void setError(final Pdpb.Error error);
+
+    Object getData();
+
+    void setData(final Object data);
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftEngine.java

@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.raft;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.apache.hugegraph.pd.grpc.Pdpb;
+
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.RaftGroupService;
+import com.alipay.sofa.jraft.ReplicatorGroup;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.core.Replicator;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.Task;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.option.RpcOptions;
+import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.ThreadId;
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class RaftEngine {
+    private static final RaftEngine INSTANCE = new RaftEngine();
+    private final RaftStateMachine stateMachine;
+    private PDConfig.Raft config;
+    private RaftGroupService raftGroupService;
+    private RpcServer rpcServer;
+    private Node raftNode;
+    private RaftRpcClient raftRpcClient;
+
+    public RaftEngine() {
+        this.stateMachine = new RaftStateMachine();
+    }
+
+    public static RaftEngine getInstance() {
+        return INSTANCE;
+    }
+
+    public boolean init(PDConfig.Raft config) {
+        if (this.raftNode != null) {
+            return false;
+        }
+        this.config = config;
+
+        raftRpcClient = new RaftRpcClient();
+        raftRpcClient.init(new RpcOptions());
+
+        String groupId = "pd_raft";
+        String raftPath = config.getDataPath() + "/" + groupId;
+        new File(raftPath).mkdirs();
+
+        new File(config.getDataPath()).mkdirs();
+        Configuration initConf = new Configuration();
+        initConf.parse(config.getPeersList());
+        if (config.isEnable() && config.getPeersList().length() < 3) {
+            log.error("The RaftEngine parameter is incorrect." +
+                      " When RAFT is enabled, the number of peers " + "cannot be less than 3");
+        }
+        // 设置 Node 参数，包括日志存储路径和状态机实例
+        NodeOptions nodeOptions = new NodeOptions();
+        nodeOptions.setFsm(stateMachine);
+        nodeOptions.setEnableMetrics(true);
+        // 日志路径
+        nodeOptions.setLogUri(raftPath + "/log");
+        // raft 元数据路径
+        nodeOptions.setRaftMetaUri(raftPath + "/meta");
+        // 快照路径
+        nodeOptions.setSnapshotUri(raftPath + "/snapshot");
+        // 初始集群
+        nodeOptions.setInitialConf(initConf);
+        // 快照时间间隔
+        nodeOptions.setSnapshotIntervalSecs(config.getSnapshotInterval());
+
+        nodeOptions.setRpcConnectTimeoutMs(config.getRpcTimeout());
+        nodeOptions.setRpcDefaultTimeout(config.getRpcTimeout());
+        nodeOptions.setRpcInstallSnapshotTimeout(config.getRpcTimeout());
+        // 设置 raft 配置
+        RaftOptions raftOptions = nodeOptions.getRaftOptions();
+
+        nodeOptions.setEnableMetrics(true);
+
+        final PeerId serverId = JRaftUtils.getPeerId(config.getAddress());
+
+        rpcServer = createRaftRpcServer(config.getAddress());
+        // 构建 raft 组并启动 raft
+        this.raftGroupService =
+                new RaftGroupService(groupId, serverId, nodeOptions, rpcServer, true);
+        this.raftNode = raftGroupService.start(false);
+        log.info("RaftEngine start successfully: id = {}, peers list = {}", groupId,
+                 nodeOptions.getInitialConf().getPeers());
+        return this.raftNode != null;
+    }
+
+    /**
+     * 创建 raft rpc server，用于 pd 之间通讯
+     */
+    private RpcServer createRaftRpcServer(String raftAddr) {
+        Endpoint endpoint = JRaftUtils.getEndPoint(raftAddr);
+        RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(endpoint);
+        RaftRpcProcessor.registerProcessor(rpcServer, this);
+        rpcServer.init(null);
+        return rpcServer;
+    }
+
+    public void shutDown() {
+        if (this.raftGroupService != null) {
+            this.raftGroupService.shutdown();
+            try {
+                this.raftGroupService.join();
+            } catch (final InterruptedException e) {
+                this.raftNode = null;
+                ThrowUtil.throwException(e);
+            }
+            this.raftGroupService = null;
+        }
+        if (this.rpcServer != null) {
+            this.rpcServer.shutdown();
+            this.rpcServer = null;
+        }
+        if (this.raftNode != null) {
+            this.raftNode.shutdown();
+        }
+        this.raftNode = null;
+    }
+
+    public boolean isLeader() {
+        return this.raftNode.isLeader(true);
+    }
+
+    /**
+     * 添加 Raft 任务，grpc 通过该接口给 raft 发送数据
+     */
+    public void addTask(Task task) {
+        if (!isLeader()) {
+            KVStoreClosure closure = (KVStoreClosure) task.getDone();
+            closure.setError(Pdpb.Error.newBuilder().setType(Pdpb.ErrorType.NOT_LEADER).build());
+            closure.run(new Status(RaftError.EPERM, "Not leader"));
+            return;
+        }
+        this.raftNode.apply(task);
+    }
+
+    public void addStateListener(RaftStateListener listener) {
+        this.stateMachine.addStateListener(listener);
+    }
+
+    public void addTaskHandler(RaftTaskHandler handler) {
+        this.stateMachine.addTaskHandler(handler);
+    }
+
+    public PDConfig.Raft getConfig() {
+        return this.config;
+    }
+
+    public PeerId getLeader() {
+        return raftNode.getLeaderId();
+    }
+
+    /**
+     * 向 leader 发消息，获取 grpc 地址；
+     */
+    public String getLeaderGrpcAddress() throws ExecutionException, InterruptedException {
+        if (isLeader()) {
+            return config.getGrpcAddress();
+        }
+
+        if (raftNode.getLeaderId() == null) {
+            waitingForLeader(10000);
+        }
+
+        return raftRpcClient.getGrpcAddress(raftNode.getLeaderId().getEndpoint().toString()).get()
+                            .getGrpcAddress();
+    }
+
+    public Metapb.Member getLocalMember() {
+        Metapb.Member.Builder builder = Metapb.Member.newBuilder();
+        builder.setClusterId(config.getClusterId());
+        builder.setRaftUrl(config.getAddress());
+        builder.setDataPath(config.getDataPath());
+        builder.setGrpcUrl(config.getGrpcAddress());
+        builder.setState(Metapb.StoreState.Up);
+        return builder.build();
+    }
+
+    public List<Metapb.Member> getMembers() {
+        List<Metapb.Member> members = new ArrayList<>();
+
+        List<PeerId> peers = raftNode.listPeers();
+        peers.addAll(raftNode.listLearners());
+        var learners = new HashSet<>(raftNode.listLearners());
+
+        for (PeerId peerId : peers) {
+            Metapb.Member.Builder builder = Metapb.Member.newBuilder();
+            builder.setClusterId(config.getClusterId());
+            CompletableFuture<RaftRpcProcessor.GetMemberResponse> future =
+                    raftRpcClient.getGrpcAddress(peerId.getEndpoint().toString());
+
+            Metapb.ShardRole role = Metapb.ShardRole.Follower;
+            if (peerEquals(peerId, raftNode.getLeaderId())) {
+                role = Metapb.ShardRole.Leader;
+            } else if (learners.contains(peerId)) {
+                role = Metapb.ShardRole.Learner;
+                var state = getReplicatorState(peerId);
+                if (state != null) {
+                    builder.setReplicatorState(state.name());
+                }
+            }
+
+            builder.setRole(role);
+
+            try {
+                if (future.isCompletedExceptionally()) {
+                    log.error("failed to getGrpcAddress of {}", peerId.getEndpoint().toString());
+                    builder.setState(Metapb.StoreState.Offline);
+                    builder.setRaftUrl(peerId.getEndpoint().toString());
+                    members.add(builder.build());
+                } else {
+                    RaftRpcProcessor.GetMemberResponse response = future.get();
+                    builder.setState(Metapb.StoreState.Up);
+                    builder.setRaftUrl(response.getRaftAddress());
+                    builder.setDataPath(response.getDatePath());
+                    builder.setGrpcUrl(response.getGrpcAddress());
+                    builder.setRestUrl(response.getRestAddress());
+                    members.add(builder.build());
+                }
+            } catch (Exception e) {
+                log.error("failed to getGrpcAddress of {}.", peerId.getEndpoint().toString(), e);
+                builder.setState(Metapb.StoreState.Offline);
+                builder.setRaftUrl(peerId.getEndpoint().toString());
+                members.add(builder.build());
+            }
+
+        }
+        return members;
+    }
+
+    public Status changePeerList(String peerList) {
+        AtomicReference<Status> result = new AtomicReference<>();
+        try {
+            String[] peers = peerList.split(",", -1);
+            if ((peers.length & 1) != 1) {
+                throw new PDException(-1, "the number of peer list must be odd.");
+            }
+            Configuration newPeers = new Configuration();
+            newPeers.parse(peerList);
+            CountDownLatch latch = new CountDownLatch(1);
+            this.raftNode.changePeers(newPeers, status -> {
+                result.set(status);
+                latch.countDown();
+            });
+            latch.await();
+        } catch (Exception e) {
+            log.error("failed to changePeerList to {}", peerList, e);
+            result.set(new Status(-1, e.getMessage()));
+        }
+        return result.get();
+    }
+
+    public PeerId waitingForLeader(long timeOut) {
+        PeerId leader = getLeader();
+        if (leader != null) {
+            return leader;
+        }
+
+        synchronized (this) {
+            leader = getLeader();
+            long start = System.currentTimeMillis();
+            while ((System.currentTimeMillis() - start < timeOut) && (leader == null)) {
+                try {
+                    this.wait(1000);
+                } catch (InterruptedException e) {
+                    log.error("Raft wait for leader exception", e);
+                }
+                leader = getLeader();
+            }
+            return leader;
+        }
+
+    }
+
+    public Node getRaftNode() {
+        return raftNode;
+    }
+
+    private boolean peerEquals(PeerId p1, PeerId p2) {
+        if (p1 == null && p2 == null) {
+            return true;
+        }
+        if (p1 == null || p2 == null) {
+            return false;
+        }
+        return Objects.equals(p1.getIp(), p2.getIp()) && Objects.equals(p1.getPort(), p2.getPort());
+    }
+
+    private Replicator.State getReplicatorState(PeerId peerId) {
+        var replicateGroup = getReplicatorGroup();
+        if (replicateGroup == null) {
+            return null;
+        }
+
+        ThreadId threadId = replicateGroup.getReplicator(peerId);
+        if (threadId == null) {
+            return null;
+        } else {
+            Replicator r = (Replicator) threadId.lock();
+            if (r == null) {
+                return Replicator.State.Probe;
+            }
+            Replicator.State result = getState(r);
+            threadId.unlock();
+            return result;
+        }
+    }
+
+    private ReplicatorGroup getReplicatorGroup() {
+        var clz = this.raftNode.getClass();
+        try {
+            var f = clz.getDeclaredField("replicatorGroup");
+            f.setAccessible(true);
+            var group = (ReplicatorGroup) f.get(this.raftNode);
+            f.setAccessible(false);
+            return group;
+        } catch (NoSuchFieldException | IllegalAccessException e) {
+            log.info("getReplicatorGroup: error {}", e.getMessage());
+            return null;
+        }
+    }
+
+    private Replicator.State getState(Replicator r) {
+        var clz = r.getClass();
+        try {
+            var f = clz.getDeclaredField("state");
+            f.setAccessible(true);
+            var state = (Replicator.State) f.get(this.raftNode);
+            f.setAccessible(false);
+            return state;
+        } catch (NoSuchFieldException | IllegalAccessException e) {
+            log.info("getReplicatorGroup: error {}", e.getMessage());
+            return null;
+        }
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftRpcClient.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.raft;
+
+import java.util.concurrent.CompletableFuture;
+
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.option.RpcOptions;
+import com.alipay.sofa.jraft.rpc.InvokeCallback;
+import com.alipay.sofa.jraft.rpc.InvokeContext;
+import com.alipay.sofa.jraft.rpc.RaftRpcFactory;
+import com.alipay.sofa.jraft.rpc.RpcClient;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.RpcFactoryHelper;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class RaftRpcClient {
+    protected volatile RpcClient rpcClient;
+    private RpcOptions rpcOptions;
+
+    public synchronized boolean init(final RpcOptions rpcOptions) {
+        this.rpcOptions = rpcOptions;
+        final RaftRpcFactory factory = RpcFactoryHelper.rpcFactory();
+        this.rpcClient =
+                factory.createRpcClient(factory.defaultJRaftClientConfigHelper(this.rpcOptions));
+        return this.rpcClient.init(null);
+    }
+
+    /**
+     * 请求快照
+     */
+    public CompletableFuture<RaftRpcProcessor.GetMemberResponse>
+    getGrpcAddress(final String address) {
+        RaftRpcProcessor.GetMemberRequest request = new RaftRpcProcessor.GetMemberRequest();
+        FutureClosureAdapter<RaftRpcProcessor.GetMemberResponse> response =
+                new FutureClosureAdapter<>();
+        internalCallAsyncWithRpc(JRaftUtils.getEndPoint(address), request, response);
+        return response.future;
+    }
+
+    private <V> void internalCallAsyncWithRpc(final Endpoint endpoint,
+                                              final RaftRpcProcessor.BaseRequest request,
+                                              final FutureClosureAdapter<V> closure) {
+        final InvokeContext invokeCtx = null;
+        final InvokeCallback invokeCallback = new InvokeCallback() {
+
+            @Override
+            public void complete(final Object result, final Throwable err) {
+                if (err == null) {
+                    final RaftRpcProcessor.BaseResponse response =
+                            (RaftRpcProcessor.BaseResponse) result;
+                    closure.setResponse((V) response);
+                } else {
+                    closure.failure(err);
+                    closure.run(new Status(-1, err.getMessage()));
+                }
+            }
+        };
+
+        try {
+            this.rpcClient.invokeAsync(endpoint, request, invokeCtx, invokeCallback,
+                                       this.rpcOptions.getRpcDefaultTimeout());
+        } catch (final Throwable t) {
+            log.error("failed to call rpc to {}. {}", endpoint, t.getMessage());
+            closure.failure(t);
+            closure.run(new Status(-1, t.getMessage()));
+        }
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftRpcProcessor.java

@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.raft;
+
+import java.io.Serializable;
+
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.alipay.sofa.jraft.rpc.RpcProcessor;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+
+import lombok.Data;
+
+public class RaftRpcProcessor<T extends RaftRpcProcessor.BaseRequest> implements RpcProcessor<T> {
+
+
+    private final Class<?> requestClass;
+    private final RaftEngine raftEngine;
+
+    public RaftRpcProcessor(Class<?> requestClass, RaftEngine raftEngine) {
+        this.requestClass = requestClass;
+        this.raftEngine = raftEngine;
+    }
+
+    public static void registerProcessor(final RpcServer rpcServer, RaftEngine raftEngine) {
+        rpcServer.registerProcessor(new RaftRpcProcessor<>(GetMemberRequest.class, raftEngine));
+    }
+
+    @Override
+    public void handleRequest(RpcContext rpcCtx, T request) {
+        if (request.magic() == BaseRequest.GET_GRPC_ADDRESS) {
+            rpcCtx.sendResponse(getGrpcAddress());
+        }
+    }
+
+    @Override
+    public String interest() {
+        return this.requestClass.getName();
+    }
+
+    private GetMemberResponse getGrpcAddress() {
+        GetMemberResponse rep = new GetMemberResponse();
+        rep.setGrpcAddress(raftEngine.getConfig().getGrpcAddress());
+        rep.setClusterId(raftEngine.getConfig().getClusterId());
+        rep.setDatePath(raftEngine.getConfig().getDataPath());
+        rep.setRaftAddress(raftEngine.getConfig().getAddress());
+        rep.setRestAddress(
+                raftEngine.getConfig().getHost() + ":" + raftEngine.getConfig().getPort());
+        rep.setStatus(Status.OK);
+        return rep;
+    }
+
+    public enum Status implements Serializable {
+        UNKNOWN(-1, "unknown"),
+        OK(0, "ok"),
+        COMPLETE(0, "Transmission completed"),
+        INCOMPLETE(1, "Incomplete transmission"),
+        NO_PARTITION(10, "Partition not found"),
+        IO_ERROR(11, "io error"),
+        EXCEPTION(12, "exception"),
+        ABORT(100, "Transmission aborted");
+
+        private final int code;
+        private String msg;
+
+        Status(int code, String msg) {
+            this.code = code;
+            this.msg = msg;
+        }
+
+        public int getCode() {
+            return this.code;
+        }
+
+        public Status setMsg(String msg) {
+            this.msg = msg;
+            return this;
+        }
+
+        public boolean isOK() {
+            return this.code == 0;
+        }
+    }
+
+    public abstract static class BaseRequest implements Serializable {
+        public static final byte GET_GRPC_ADDRESS = 0x01;
+
+        public abstract byte magic();
+    }
+
+    @Data
+    public abstract static class BaseResponse implements Serializable {
+        private Status status;
+
+    }
+
+    @Data
+    public static class GetMemberRequest extends BaseRequest {
+        @Override
+        public byte magic() {
+            return GET_GRPC_ADDRESS;
+        }
+    }
+
+    @Data
+    public static class GetMemberResponse extends BaseResponse {
+        private long clusterId;
+        private String raftAddress;
+        private String grpcAddress;
+        private String datePath;
+        private String restAddress;
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftStateListener.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.raft;
+
+public interface RaftStateListener {
+    void onRaftLeaderChanged();
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftStateMachine.java

@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.raft;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.zip.Checksum;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.grpc.Pdpb;
+import org.springframework.util.CollectionUtils;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.core.StateMachineAdapter;
+import com.alipay.sofa.jraft.entity.LeaderChangeContext;
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.error.RaftException;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+import com.alipay.sofa.jraft.util.CRC64;
+import com.alipay.sofa.jraft.util.Utils;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class RaftStateMachine extends StateMachineAdapter {
+    private static final String SNAPSHOT_DIR_NAME = "snapshot";
+    private static final String SNAPSHOT_ARCHIVE_NAME = "snapshot.zip";
+    private final AtomicLong leaderTerm = new AtomicLong(-1);
+    private final List<RaftTaskHandler> taskHandlers;
+    private final List<RaftStateListener> stateListeners;
+
+    public RaftStateMachine() {
+        this.taskHandlers = new CopyOnWriteArrayList<>();
+        this.stateListeners = new CopyOnWriteArrayList<>();
+    }
+
+    public void addTaskHandler(RaftTaskHandler handler) {
+        taskHandlers.add(handler);
+    }
+
+    public void addStateListener(RaftStateListener listener) {
+        stateListeners.add(listener);
+    }
+
+    public boolean isLeader() {
+        return this.leaderTerm.get() > 0;
+    }
+
+    @Override
+    public void onApply(Iterator iter) {
+        while (iter.hasNext()) {
+            final RaftClosureAdapter done = (RaftClosureAdapter) iter.done();
+            try {
+                KVOperation kvOp;
+                if (done != null) {
+                    kvOp = done.op;
+                } else {
+                    kvOp = KVOperation.fromByteArray(iter.getData().array());
+                }
+                for (RaftTaskHandler taskHandler : taskHandlers) {
+                    taskHandler.invoke(kvOp, done);
+                }
+                if (done != null) {
+                    done.run(Status.OK());
+                }
+            } catch (Throwable t) {
+                log.error("StateMachine meet critical error: {}.", t);
+                if (done != null) {
+                    done.run(new Status(RaftError.EINTERNAL, t.getMessage()));
+                }
+            }
+            iter.next();
+        }
+    }
+
+    @Override
+    public void onError(final RaftException e) {
+        log.error("Raft StateMachine on error {}", e);
+    }
+
+    @Override
+    public void onShutdown() {
+        super.onShutdown();
+    }
+
+    @Override
+    public void onLeaderStart(final long term) {
+        this.leaderTerm.set(term);
+        super.onLeaderStart(term);
+
+        log.info("Raft becomes leader");
+        Utils.runInThread(() -> {
+            if (!CollectionUtils.isEmpty(stateListeners)) {
+                stateListeners.forEach(listener -> {
+                    listener.onRaftLeaderChanged();
+                });
+            }
+        });
+    }
+
+    @Override
+    public void onLeaderStop(final Status status) {
+        this.leaderTerm.set(-1);
+        super.onLeaderStop(status);
+        log.info("Raft  lost leader ");
+    }
+
+    @Override
+    public void onStartFollowing(final LeaderChangeContext ctx) {
+        super.onStartFollowing(ctx);
+        Utils.runInThread(() -> {
+            if (!CollectionUtils.isEmpty(stateListeners)) {
+                stateListeners.forEach(listener -> {
+                    listener.onRaftLeaderChanged();
+                });
+            }
+        });
+    }
+
+    @Override
+    public void onStopFollowing(final LeaderChangeContext ctx) {
+        super.onStopFollowing(ctx);
+    }
+
+
+    @Override
+    public void onConfigurationCommitted(final Configuration conf) {
+        log.info("Raft  onConfigurationCommitted {}", conf);
+    }
+
+    @Override
+    public void onSnapshotSave(final SnapshotWriter writer, final Closure done) {
+
+        String snapshotDir = writer.getPath() + File.separator + SNAPSHOT_DIR_NAME;
+        try {
+            FileUtils.deleteDirectory(new File(snapshotDir));
+            FileUtils.forceMkdir(new File(snapshotDir));
+        } catch (IOException e) {
+            log.error("Failed to create snapshot directory {}", snapshotDir);
+            done.run(new Status(RaftError.EIO, e.toString()));
+            return;
+        }
+
+        CountDownLatch latch = new CountDownLatch(taskHandlers.size());
+        for (RaftTaskHandler taskHandler : taskHandlers) {
+            Utils.runInThread(() -> {
+                try {
+                    KVOperation op = KVOperation.createSaveSnapshot(snapshotDir);
+                    taskHandler.invoke(op, null);
+                    log.info("Raft onSnapshotSave success");
+                    latch.countDown();
+                } catch (PDException e) {
+                    log.error("Raft onSnapshotSave failed. {}", e.toString());
+                    done.run(new Status(RaftError.EIO, e.toString()));
+                }
+            });
+        }
+        try {
+            latch.await();
+        } catch (InterruptedException e) {
+            log.error("Raft onSnapshotSave failed. {}", e.toString());
+            done.run(new Status(RaftError.EIO, e.toString()));
+            return;
+        }
+
+        // compress
+        try {
+            compressSnapshot(writer);
+            FileUtils.deleteDirectory(new File(snapshotDir));
+        } catch (Exception e) {
+            log.error("Failed to delete snapshot directory {}, {}", snapshotDir, e.toString());
+            done.run(new Status(RaftError.EIO, e.toString()));
+            return;
+        }
+        done.run(Status.OK());
+    }
+
+    @Override
+    public boolean onSnapshotLoad(final SnapshotReader reader) {
+        if (isLeader()) {
+            log.warn("Leader is not supposed to load snapshot");
+            return false;
+        }
+        String snapshotDir = reader.getPath() + File.separator + SNAPSHOT_DIR_NAME;
+        String snapshotArchive = reader.getPath() + File.separator + SNAPSHOT_ARCHIVE_NAME;
+        // 2. decompress snapshot archive
+        try {
+            decompressSnapshot(reader);
+        } catch (PDException e) {
+            log.error("Failed to delete snapshot directory {}, {}", snapshotDir, e.toString());
+            return true;
+        }
+
+        CountDownLatch latch = new CountDownLatch(taskHandlers.size());
+        for (RaftTaskHandler taskHandler : taskHandlers) {
+            try {
+                KVOperation op = KVOperation.createLoadSnapshot(snapshotDir);
+                taskHandler.invoke(op, null);
+                log.info("Raft onSnapshotLoad success");
+                latch.countDown();
+            } catch (PDException e) {
+                log.error("Raft onSnapshotLoad failed. {}", e.toString());
+                return false;
+            }
+        }
+        try {
+            latch.await();
+        } catch (InterruptedException e) {
+            log.error("Raft onSnapshotSave failed. {}", e.toString());
+            return false;
+        }
+
+
+        try {
+            // TODO: remove file from meta
+            // SnapshotReader 沒有提供刪除文件的接口
+            FileUtils.deleteDirectory(new File(snapshotDir));
+            File file = new File(snapshotArchive);
+            if (file.exists()) {
+                FileUtils.forceDelete(file);
+            }
+        } catch (IOException e) {
+            log.error("Failed to delete snapshot directory {} and file {}", snapshotDir,
+                      snapshotArchive);
+            return false;
+        }
+
+        return true;
+    }
+
+    private void compressSnapshot(final SnapshotWriter writer) throws PDException {
+        final Checksum checksum = new CRC64();
+        final String snapshotArchive = writer.getPath() + File.separator + SNAPSHOT_ARCHIVE_NAME;
+        try {
+            ZipUtils.compress(writer.getPath(), SNAPSHOT_DIR_NAME, snapshotArchive, checksum);
+            LocalFileMetaOutter.LocalFileMeta.Builder metaBuild =
+                    LocalFileMetaOutter.LocalFileMeta.newBuilder();
+            metaBuild.setChecksum(Long.toHexString(checksum.getValue()));
+            if (!writer.addFile(SNAPSHOT_ARCHIVE_NAME, metaBuild.build())) {
+                throw new PDException(Pdpb.ErrorType.ROCKSDB_SAVE_SNAPSHOT_ERROR_VALUE,
+                                      "failed to add file to LocalFileMeta");
+            }
+        } catch (IOException e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_SAVE_SNAPSHOT_ERROR_VALUE, e);
+        }
+    }
+
+    private void decompressSnapshot(final SnapshotReader reader) throws PDException {
+        final LocalFileMetaOutter.LocalFileMeta meta =
+                (LocalFileMetaOutter.LocalFileMeta) reader.getFileMeta(SNAPSHOT_ARCHIVE_NAME);
+        final Checksum checksum = new CRC64();
+        final String snapshotArchive = reader.getPath() + File.separator + SNAPSHOT_ARCHIVE_NAME;
+        try {
+            ZipUtils.decompress(snapshotArchive, reader.getPath(), checksum);
+            if (meta.hasChecksum()) {
+                if (!meta.getChecksum().equals(Long.toHexString(checksum.getValue()))) {
+                    throw new PDException(Pdpb.ErrorType.ROCKSDB_LOAD_SNAPSHOT_ERROR_VALUE,
+                                          "Snapshot checksum failed");
+                }
+            }
+        } catch (IOException e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_LOAD_SNAPSHOT_ERROR_VALUE, e);
+        }
+    }
+
+
+    public static class RaftClosureAdapter implements KVStoreClosure {
+        private final KVOperation op;
+        private final KVStoreClosure closure;
+
+        public RaftClosureAdapter(KVOperation op, KVStoreClosure closure) {
+            this.op = op;
+            this.closure = closure;
+        }
+
+        public KVStoreClosure getClosure() {
+            return closure;
+        }
+
+        @Override
+        public void run(Status status) {
+            closure.run(status);
+        }
+
+        @Override
+        public Pdpb.Error getError() {
+            return null;
+        }
+
+        @Override
+        public void setError(Pdpb.Error error) {
+
+        }
+
+        @Override
+        public Object getData() {
+            return null;
+        }
+
+        @Override
+        public void setData(Object data) {
+
+        }
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftTaskHandler.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.raft;
+
+import org.apache.hugegraph.pd.common.PDException;
+
+/**
+ * 接收raft发送的数据
+ */
+public interface RaftTaskHandler {
+    boolean invoke(final KVOperation op, KVStoreClosure response) throws PDException;
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/ZipUtils.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.raft;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.zip.CheckedInputStream;
+import java.util.zip.CheckedOutputStream;
+import java.util.zip.Checksum;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.NullOutputStream;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public final class ZipUtils {
+
+    public static void compress(final String rootDir, final String sourceDir,
+                                final String outputFile, final Checksum checksum) throws
+                                                                                  IOException {
+        try (final FileOutputStream fos = new FileOutputStream(outputFile);
+             final CheckedOutputStream cos = new CheckedOutputStream(fos, checksum);
+             final ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(cos))) {
+            ZipUtils.compressDirectoryToZipFile(rootDir, sourceDir, zos);
+            zos.flush();
+            fos.getFD().sync();
+        }
+    }
+
+    private static void compressDirectoryToZipFile(final String rootDir, final String sourceDir,
+                                                   final ZipOutputStream zos) throws IOException {
+        final String dir = Paths.get(rootDir, sourceDir).toString();
+        final File[] files = new File(dir).listFiles();
+        for (final File file : files) {
+            final String child = Paths.get(sourceDir, file.getName()).toString();
+            if (file.isDirectory()) {
+                compressDirectoryToZipFile(rootDir, child, zos);
+            } else {
+                zos.putNextEntry(new ZipEntry(child));
+                try (final FileInputStream fis = new FileInputStream(file);
+                     final BufferedInputStream bis = new BufferedInputStream(fis)) {
+                    IOUtils.copy(bis, zos);
+                }
+            }
+        }
+    }
+
+    public static void decompress(final String sourceFile, final String outputDir,
+                                  final Checksum checksum) throws IOException {
+        try (final FileInputStream fis = new FileInputStream(sourceFile);
+             final CheckedInputStream cis = new CheckedInputStream(fis, checksum);
+             final ZipInputStream zis = new ZipInputStream(new BufferedInputStream(cis))) {
+            ZipEntry entry;
+            while ((entry = zis.getNextEntry()) != null) {
+                final String fileName = entry.getName();
+                final File entryFile = new File(Paths.get(outputDir, fileName).toString());
+                FileUtils.forceMkdir(entryFile.getParentFile());
+                try (final FileOutputStream fos = new FileOutputStream(entryFile);
+                     final BufferedOutputStream bos = new BufferedOutputStream(fos)) {
+                    IOUtils.copy(zis, bos);
+                    bos.flush();
+                    fos.getFD().sync();
+                }
+            }
+            IOUtils.copy(cis, NullOutputStream.NULL_OUTPUT_STREAM);
+        }
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/BaseKVStoreClosure.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.store;
+
+import org.apache.hugegraph.pd.grpc.Pdpb;
+import org.apache.hugegraph.pd.raft.KVStoreClosure;
+
+public abstract class BaseKVStoreClosure implements KVStoreClosure {
+    private Pdpb.Error error;
+    private Object data;
+
+    @Override
+    public Pdpb.Error getError() {
+        return error;
+    }
+
+    @Override
+    public void setError(Pdpb.Error error) {
+        this.error = error;
+    }
+
+    @Override
+    public Object getData() {
+        return data;
+    }
+
+    @Override
+    public void setData(Object data) {
+        this.data = data;
+    }
+
+
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStore.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.store;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+
+public interface HgKVStore {
+    void init(PDConfig config);
+
+    void put(byte[] key, byte[] value) throws PDException;
+
+    byte[] get(byte[] key) throws PDException;
+
+    List<KV> scanPrefix(byte[] prefix);
+
+    long remove(byte[] bytes) throws PDException;
+
+    long removeByPrefix(byte[] bytes) throws PDException;
+
+    void putWithTTL(byte[] key, byte[] value, long ttl) throws PDException;
+
+    void putWithTTL(byte[] key, byte[] value, long ttl, TimeUnit timeUnit) throws PDException;
+
+    byte[] getWithTTL(byte[] key) throws PDException;
+
+    void removeWithTTL(byte[] key) throws PDException;
+
+    List<byte[]> getListWithTTL(byte[] key) throws PDException;
+
+    void clear() throws PDException;
+
+    void saveSnapshot(String snapshotPath) throws PDException;
+
+    void loadSnapshot(String snapshotPath) throws PDException;
+
+    List<KV> scanRange(byte[] start, byte[] end);
+
+    void close();
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java

@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.store;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.Pdpb;
+import org.apache.hugegraph.pd.grpc.discovery.RegisterInfo;
+import org.rocksdb.Checkpoint;
+import org.rocksdb.Options;
+import org.rocksdb.ReadOptions;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.RocksIterator;
+import org.rocksdb.Slice;
+
+import com.alipay.sofa.jraft.util.Utils;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.primitives.Bytes;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class HgKVStoreImpl implements HgKVStore {
+    private static final ConcurrentHashMap<String,
+            ConcurrentMap<String, Object>> CACHE = new ConcurrentHashMap();
+    private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+    private RocksDB db;
+    private String dbPath;
+    private Options dbOptions;
+
+    @Override
+    public void init(PDConfig config) {
+        dbOptions = new Options().setCreateIfMissing(true);
+
+        final Lock writeLock = this.readWriteLock.writeLock();
+        writeLock.lock();
+        try {
+            this.dbPath = config.getDataPath() + "/rocksdb/";
+            File file = new File(this.dbPath);
+            if (!file.exists()) {
+                try {
+                    FileUtils.forceMkdir(file);
+                } catch (IOException e) {
+                    log.warn("Failed to create data file,{}", e);
+                }
+            }
+            openRocksDB(dbPath);
+        } catch (PDException e) {
+            log.error("Failed to open data file,{}", e);
+        } finally {
+            writeLock.unlock();
+        }
+    }
+
+    @Override
+    public void put(byte[] key, byte[] value) throws PDException {
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            db.put(key, value);
+        } catch (RocksDBException e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_WRITE_ERROR_VALUE, e);
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public byte[] get(byte[] key) throws PDException {
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            return db.get(key);
+        } catch (RocksDBException e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_READ_ERROR_VALUE, e);
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public List<KV> scanPrefix(byte[] prefix) {
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try (ReadOptions options = new ReadOptions()
+                .setIterateLowerBound(new Slice(prefix))) {
+            List<KV> kvs = new ArrayList<>();
+            RocksIterator iterator = db.newIterator(options);
+            iterator.seekToFirst();
+            while (iterator.isValid() && 0 == Bytes.indexOf(iterator.key(), prefix)) {
+                kvs.add(new KV(iterator.key(), iterator.value()));
+                iterator.next();
+            }
+            return kvs;
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public long remove(byte[] key) throws PDException {
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            db.delete(key);
+        } catch (RocksDBException e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_DEL_ERROR_VALUE, e);
+        } finally {
+            readLock.unlock();
+        }
+        return 0;
+    }
+
+    @Override
+    public long removeByPrefix(byte[] prefix) throws PDException {
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try (ReadOptions options = new ReadOptions()
+                .setIterateLowerBound(new Slice(prefix))) {
+            RocksIterator iterator = db.newIterator(options);
+            iterator.seekToFirst();
+
+            while (iterator.isValid()) {
+                if (0 == Bytes.indexOf(iterator.key(), prefix)) {
+                    db.delete(iterator.key());
+                } else {
+                    break;
+                }
+                iterator.next();
+            }
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_WRITE_ERROR_VALUE, e);
+        } finally {
+            readLock.unlock();
+        }
+        return 0;
+    }
+
+    @Override
+    public void clear() throws PDException {
+        CACHE.clear();
+    }
+
+    @Override
+    public List getListWithTTL(byte[] key) throws PDException {
+        String storeKey = new String(key, Charset.defaultCharset());
+        LinkedList<byte[]> ts = new LinkedList<>();
+        CACHE.keySet().forEach((cacheKey) -> {
+            if (cacheKey.startsWith(storeKey)) {
+                ConcurrentMap map;
+                if ((map = CACHE.get(cacheKey)) == null) {
+                    return;
+                }
+                map.values().forEach((element) -> {
+                    ts.add((byte[]) element);
+                });
+            }
+        });
+        return ts;
+    }
+
+    @Override
+    public byte[] getWithTTL(byte[] key) throws PDException {
+        ConcurrentMap map;
+        String storeKey = new String(key, Charset.defaultCharset());
+        if ((map = CACHE.get(storeKey)) == null) {
+            return null;
+        }
+        Object value = map.get(storeKey);
+        return value == null ? null : (byte[]) value;
+    }
+
+    @Override
+    public void removeWithTTL(byte[] key) throws PDException {
+        ConcurrentMap map;
+        String storeKey = new String(key, Charset.defaultCharset());
+        if ((map = CACHE.get(storeKey)) == null) {
+            return;
+        }
+        map.remove(storeKey);
+    }
+
+    @Override
+    public void putWithTTL(byte[] key, byte[] value, long ttl) throws PDException {
+        this.putWithTTL(key, value, ttl, TimeUnit.SECONDS);
+    }
+
+    @Override
+    public void putWithTTL(byte[] key, byte[] value, long ttl, TimeUnit timeUnit) throws
+                                                                                  PDException {
+        try {
+            ConcurrentMap spaceNode = CacheBuilder.newBuilder().initialCapacity(200)
+                                                  .expireAfterWrite(ttl,
+                                                                    timeUnit)
+                                                  .<String, RegisterInfo>build().asMap();
+            String storeKey = new String(key, Charset.defaultCharset());
+            ConcurrentMap space = CACHE.putIfAbsent(storeKey, spaceNode);
+            if (space == null) {
+                space = spaceNode;
+            }
+            space.put(storeKey, value);
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_WRITE_ERROR_VALUE, e);
+        }
+    }
+
+    @Override
+    public void saveSnapshot(String snapshotPath) throws PDException {
+        log.info("begin save snapshot at {}", snapshotPath);
+        final Lock writeLock = this.readWriteLock.writeLock();
+        writeLock.lock();
+        try (final Checkpoint checkpoint = Checkpoint.create(this.db)) {
+            final String tempPath = Paths.get(snapshotPath) + "_temp";
+            final File tempFile = new File(tempPath);
+            FileUtils.deleteDirectory(tempFile);
+            checkpoint.createCheckpoint(tempPath);
+            final File snapshotFile = new File(snapshotPath);
+            FileUtils.deleteDirectory(snapshotFile);
+            if (!Utils.atomicMoveFile(tempFile, snapshotFile, true)) {
+                log.error("Fail to rename {} to {}", tempPath, snapshotPath);
+                throw new PDException(Pdpb.ErrorType.ROCKSDB_SAVE_SNAPSHOT_ERROR_VALUE,
+                                      String.format("Fail to rename %s to %s", tempPath,
+                                                    snapshotPath));
+            }
+        } catch (final PDException e) {
+            throw e;
+        } catch (final Exception e) {
+            log.error("Fail to write snapshot at path: {}", snapshotPath, e);
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_SAVE_SNAPSHOT_ERROR_VALUE, e);
+        } finally {
+            writeLock.unlock();
+        }
+        log.info("saved snapshot into {}", snapshotPath);
+    }
+
+    @Override
+    public void loadSnapshot(String snapshotPath) throws PDException {
+        log.info("begin load snapshot from {}", snapshotPath);
+        final Lock writeLock = this.readWriteLock.writeLock();
+        writeLock.lock();
+        try {
+            final File snapshotFile = new File(snapshotPath);
+            if (!snapshotFile.exists()) {
+                log.error("Snapshot file {} not exists.", snapshotPath);
+                return;
+            }
+            // close DB
+            closeRocksDB();
+            // replace rocksdb data with snapshot data
+            final File dbFile = new File(this.dbPath);
+            FileUtils.deleteDirectory(dbFile);
+            if (!Utils.atomicMoveFile(snapshotFile, dbFile, true)) {
+                log.error("Fail to rename {} to {}", snapshotPath, this.dbPath);
+                throw new PDException(Pdpb.ErrorType.ROCKSDB_LOAD_SNAPSHOT_ERROR_VALUE,
+                                      String.format("Fail to rename %s to %s", snapshotPath,
+                                                    this.dbPath));
+            }
+            // reopen the db
+            openRocksDB(this.dbPath);
+        } catch (final PDException e) {
+            throw e;
+        } catch (final Exception e) {
+            log.error("failed to load snapshot from {}", snapshotPath);
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_LOAD_SNAPSHOT_ERROR_VALUE, e);
+        } finally {
+            writeLock.unlock();
+        }
+        log.info("loaded snapshot from {}", snapshotPath);
+    }
+
+    @Override
+    public List<KV> scanRange(byte[] start, byte[] end) {
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try (ReadOptions options = new ReadOptions()
+                .setIterateLowerBound(new Slice(start))
+                .setIterateUpperBound(new Slice(end))) {
+            List<KV> kvs = new ArrayList<>();
+            RocksIterator iterator = db.newIterator(options);
+            iterator.seekToFirst();
+            while (iterator.isValid()) {
+                kvs.add(new KV(iterator.key(), iterator.value()));
+                iterator.next();
+            }
+            return kvs;
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public void close() {
+        closeRocksDB();
+    }
+
+
+    private void closeRocksDB() {
+        if (this.db != null) {
+            this.db.close();
+            this.db = null;
+        }
+    }
+
+    private void openRocksDB(String dbPath) throws PDException {
+        try {
+            this.db = RocksDB.open(dbOptions, dbPath);
+        } catch (RocksDBException e) {
+            log.error("Failed to open RocksDB from {}", dbPath, e);
+            throw new PDException(Pdpb.ErrorType.ROCKSDB_LOAD_SNAPSHOT_ERROR_VALUE, e);
+        }
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/KV.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.store;
+
+
+public class KV {
+    private byte[] key;
+    private byte[] value;
+
+    public KV(byte[] key, byte[] value) {
+        this.key = key;
+        this.value = value;
+    }
+
+    public byte[] getKey() {
+        return key;
+    }
+
+    public void setKey(byte[] key) {
+        this.key = key;
+    }
+
+    public byte[] getValue() {
+        return value;
+    }
+
+    public void setValue(byte[] value) {
+        this.value = value;
+    }
+}

hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/RaftKVStore.java

@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.store;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.Pdpb;
+import org.apache.hugegraph.pd.raft.KVOperation;
+import org.apache.hugegraph.pd.raft.KVStoreClosure;
+import org.apache.hugegraph.pd.raft.RaftEngine;
+import org.apache.hugegraph.pd.raft.RaftStateMachine;
+import org.apache.hugegraph.pd.raft.RaftTaskHandler;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.Task;
+import com.alipay.sofa.jraft.error.RaftError;
+
+import lombok.extern.slf4j.Slf4j;
+
+
+@Slf4j
+public class RaftKVStore implements HgKVStore, RaftTaskHandler {
+
+    private final RaftEngine engine;
+    private final HgKVStore store;
+
+    public RaftKVStore(RaftEngine engine, HgKVStore store) {
+        this.engine = engine;
+        this.store = store;
+    }
+
+    @Override
+    public void init(PDConfig config) {
+        this.store.init(config);
+        this.engine.addTaskHandler(this);
+    }
+
+    private BaseKVStoreClosure createClosure() {
+        return new BaseKVStoreClosure() {
+            @Override
+            public void run(Status status) {
+                if (!status.isOk()) {
+                    log.error("An exception occurred while performing the RAFT,{}",
+                              status.getErrorMsg());
+                } else {
+                    log.info("RAFT done!");
+                }
+            }
+        };
+    }
+
+    @Override
+    public void put(byte[] key, byte[] value) throws PDException {
+        KVOperation operation = KVOperation.createPut(key, value);
+        try {
+            applyOperation(operation).get();
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.UNKNOWN_VALUE, e.getMessage());
+        }
+    }
+
+    /**
+     * 查询可以不走raft，直接读取
+     */
+    @Override
+    public byte[] get(byte[] key) throws PDException {
+        return store.get(key);
+
+    }
+
+    @Override
+    public List<KV> scanPrefix(byte[] prefix) {
+        return store.scanPrefix(prefix);
+    }
+
+    @Override
+    public long remove(byte[] bytes) throws PDException {
+        try {
+            applyOperation(KVOperation.createRemove(bytes)).get();
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.UNKNOWN_VALUE, e.getMessage());
+        }
+        return 0;
+    }
+
+
+    @Override
+    public long removeByPrefix(byte[] bytes) throws PDException {
+        try {
+            applyOperation(KVOperation.createRemoveByPrefix(bytes)).get();
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.UNKNOWN_VALUE, e.getMessage());
+        }
+        return 0;
+    }
+
+    @Override
+    public void clear() throws PDException {
+        try {
+            applyOperation(KVOperation.createClear()).get();
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.UNKNOWN_VALUE, e.getMessage());
+        }
+    }
+
+    @Override
+    public void putWithTTL(byte[] key, byte[] value, long ttl) throws PDException {
+        try {
+            applyOperation(KVOperation.createPutWithTTL(key, value, ttl)).get();
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.UNKNOWN_VALUE, e.getMessage());
+        }
+    }
+
+    @Override
+    public void putWithTTL(byte[] key, byte[] value, long ttl, TimeUnit timeUnit) throws
+                                                                                  PDException {
+        try {
+            applyOperation(KVOperation.createPutWithTTL(key, value, ttl, timeUnit)).get();
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.UNKNOWN_VALUE, e.getMessage());
+        }
+    }
+
+    @Override
+    public List<byte[]> getListWithTTL(byte[] key) throws PDException {
+        return store.getListWithTTL(key);
+    }
+
+    @Override
+    public byte[] getWithTTL(byte[] key) throws PDException {
+        return store.getWithTTL(key);
+    }
+
+    @Override
+    public void removeWithTTL(byte[] key) throws PDException {
+        try {
+            applyOperation(KVOperation.createRemoveWithTTL(key)).get();
+        } catch (Exception e) {
+            throw new PDException(Pdpb.ErrorType.UNKNOWN_VALUE, e.getMessage());
+        }
+    }
+
+    @Override
+    public void saveSnapshot(String snapshotPath) throws PDException {
+        store.saveSnapshot(snapshotPath);
+    }
+
+    @Override
+    public void loadSnapshot(String snapshotPath) throws PDException {
+        store.loadSnapshot(snapshotPath);
+    }
+
+    @Override
+    public List<KV> scanRange(byte[] start, byte[] end) {
+        return store.scanRange(start, end);
+    }
+
+    @Override
+    public void close() {
+        store.close();
+    }
+
+    /**
+     * 需要走Raft的真实操作
+     */
+    private void doPut(byte[] key, byte[] value) throws PDException {
+
+        store.put(key, value);
+    }
+
+    public long doRemove(byte[] bytes) throws PDException {
+        return this.store.remove(bytes);
+    }
+
+    public long doRemoveByPrefix(byte[] bytes) throws PDException {
+        return this.store.removeByPrefix(bytes);
+    }
+
+    public void doRemoveWithTTL(byte[] key) throws PDException {
+        this.store.removeWithTTL(key);
+    }
+
+    public void doClear() throws PDException {
+        this.store.clear();
+    }
+
+    public void doPutWithTTL(byte[] key, byte[] value, long ttl) throws PDException {
+        this.store.putWithTTL(key, value, ttl);
+    }
+
+    public void doPutWithTTL(byte[] key, byte[] value, long ttl, TimeUnit timeUnit) throws
+                                                                                    PDException {
+        this.store.putWithTTL(key, value, ttl, timeUnit);
+    }
+
+    public void doSaveSnapshot(String snapshotPath) throws PDException {
+        this.store.saveSnapshot(snapshotPath);
+    }
+
+    public void doLoadSnapshot(String snapshotPath) throws PDException {
+        this.store.loadSnapshot(snapshotPath);
+    }
+
+    private <T> CompletableFuture<T> applyOperation(final KVOperation op) throws PDException {
+        CompletableFuture<T> future = new CompletableFuture<>();
+        try {
+            final Task task = new Task();
+            task.setData(ByteBuffer.wrap(op.toByteArray()));
+            task.setDone(new RaftStateMachine.RaftClosureAdapter(op, new KVStoreClosure() {
+                Object data;
+                Pdpb.Error error;
+
+                @Override
+                public Pdpb.Error getError() {
+                    return error;
+                }
+
+                @Override
+                public void setError(Pdpb.Error error) {
+                    this.error = error;
+                }
+
+                @Override
+                public Object getData() {
+                    return data;
+                }
+
+                @Override
+                public void setData(Object data) {
+                    this.data = data;
+                }
+
+                @Override
+                public void run(Status status) {
+                    if (status.isOk()) {
+                        future.complete((T) data);
+                    } else {
+                        RaftError raftError = status.getRaftError();
+                        Pdpb.ErrorType type;
+                        if (RaftError.EPERM.equals(raftError)) {
+                            type = Pdpb.ErrorType.NOT_LEADER;
+                        } else {
+                            type = Pdpb.ErrorType.UNKNOWN;
+                        }
+                        error = Pdpb.Error.newBuilder().setType(type)
+                                          .setMessage(status.getErrorMsg())
+                                          .build();
+                        future.completeExceptionally(
+                                new PDException(error.getTypeValue()));
+                    }
+                }
+            }));
+            this.engine.addTask(task);
+            return future;
+        } catch (Exception e) {
+            future.completeExceptionally(e);
+            return future;
+        }
+    }
+
+    private boolean isLeader() {
+        return this.engine.isLeader();
+    }
+
+    @Override
+    public boolean invoke(KVOperation op, KVStoreClosure response) throws PDException {
+        switch (op.getOp()) {
+            case KVOperation.GET:
+                break;
+            case KVOperation.PUT:
+                doPut(op.getKey(), op.getValue());
+                break;
+            case KVOperation.REMOVE:
+                doRemove(op.getKey());
+                break;
+            case KVOperation.PUT_WITH_TTL:
+                doPutWithTTL(op.getKey(), op.getValue(), (long) op.getArg());
+                break;
+            case KVOperation.PUT_WITH_TTL_UNIT:
+                Object[] arg = (Object[]) op.getArg();
+                doPutWithTTL(op.getKey(), op.getValue(), (long) arg[0], (TimeUnit) arg[1]);
+                break;
+            case KVOperation.REMOVE_BY_PREFIX:
+                doRemoveByPrefix(op.getKey());
+                break;
+            case KVOperation.REMOVE_WITH_TTL:
+                doRemoveWithTTL(op.getKey());
+                break;
+            case KVOperation.CLEAR:
+                doClear();
+                break;
+            case KVOperation.SAVE_SNAPSHOT:
+                doSaveSnapshot((String) op.getAttach());
+                break;
+            case KVOperation.LOAD_SNAPSHOT:
+                doLoadSnapshot((String) op.getAttach());
+                break;
+            default:
+                log.error("Err op {}", op.getOp());
+        }
+        return false;
+    }
+}

hugegraph-pd/hg-pd-core/src/test/java/org/apache/hugegraph/pd/MonitorServiceTest.java

@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import java.util.concurrent.ExecutionException;
+
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+// import org.junit.Test;
+
+public class MonitorServiceTest {
+    static PDConfig pdConfig;
+
+    @BeforeClass
+    public static void init() throws ExecutionException, InterruptedException {
+        pdConfig = new PDConfig() {{
+            this.setClusterId(100);
+            this.setPatrolInterval(1);
+        }};
+
+        //pdConfig.setEtcd(new PDConfig().new Etcd() {{
+        //    this.setAddress("http://localhost:2379");
+        //
+        //}});
+        pdConfig.setStore(new PDConfig().new Store() {{
+            this.setMaxDownTime(1);
+            this.setKeepAliveTimeout(5);
+        }});
+
+        pdConfig.setPartition(new PDConfig().new Partition() {{
+            this.setShardCount(3);
+            this.setTotalCount(10);
+        }});
+
+        clearClusterData();
+    }
+
+    public static void clearClusterData() throws ExecutionException, InterruptedException {
+        //Client client = Client.builder().endpoints(pdConfig.getEtcd().getAddress()).build();
+        //KV kvClient = client.getKVClient();
+        //
+        //ByteSequence key = ByteSequence.from("HUGEGRAPH/" + pdConfig.getClusterId(), Charset
+        // .forName("utf-8"));
+        //CompletableFuture<DeleteResponse> rsp = kvClient.delete(key, DeleteOption.newBuilder()
+        // .isPrefix(true).build());
+        //System.out.println("删除数量 : " + rsp.get().getDeleted());
+        //kvClient.close();
+        //client.close();
+    }
+
+    // @Test
+    public void testPatrolStores() throws PDException, InterruptedException {
+        StoreNodeService storeService = new StoreNodeService(pdConfig);
+        PartitionService partitionService = new PartitionService(pdConfig, storeService);
+        TaskScheduleService monitorService =
+                new TaskScheduleService(pdConfig, storeService, partitionService);
+        storeService.init(partitionService);
+        partitionService.init();
+        monitorService.init();
+
+        int count = 6;
+        Metapb.Store[] stores = new Metapb.Store[count];
+        for (int i = 0; i < count; i++) {
+            Metapb.Store store = Metapb.Store.newBuilder()
+                                             .setId(0)
+                                             .setAddress(String.valueOf(i))
+                                             .setDeployPath("/data")
+                                             .addLabels(Metapb.StoreLabel.newBuilder()
+                                                                         .setKey("namespace")
+                                                                         .setValue("default")
+                                                                         .build())
+                                             .build();
+            stores[i] = storeService.register(store);
+            System.out.println("新注册store， id = " + Long.toHexString(stores[i].getId()));
+        }
+        Metapb.Graph graph = Metapb.Graph.newBuilder()
+                                         .setGraphName("defaultGH")
+
+                                         .setPartitionCount(10)
+                                         .build();
+        partitionService.updateGraph(graph);
+        Thread.sleep(10000);
+        count = 0;
+        count += storeService.getStores("").stream()
+                             .filter(store -> store.getState() == Metapb.StoreState.Tombstone)
+                             .count();
+
+        Assert.assertEquals(6, count);
+
+    }
+
+
+}

hugegraph-pd/hg-pd-core/src/test/java/org/apache/hugegraph/pd/PartitionServiceTest.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.junit.Test;
+
+public class PartitionServiceTest {
+    @Test
+    public void testPartitionHeartbeat() {
+        List<Metapb.Shard> shardList = new ArrayList<>();
+        shardList.add(Metapb.Shard.newBuilder().setStoreId(1).build());
+        shardList.add(Metapb.Shard.newBuilder().setStoreId(2).build());
+        shardList.add(Metapb.Shard.newBuilder().setStoreId(3).build());
+        shardList = new ArrayList<>(shardList);
+        Metapb.PartitionStats stats = Metapb.PartitionStats.newBuilder()
+                                                           .addAllShard(shardList).build();
+        List<Metapb.Shard> shardList2 = new ArrayList<>(stats.getShardList());
+        Collections.shuffle(shardList2);
+        shardList2.forEach(shard -> {
+            System.out.println(shard.getStoreId());
+        });
+
+
+    }
+}

hugegraph-pd/hg-pd-core/src/test/java/org/apache/hugegraph/pd/StoreNodeServiceTest.java

@@ -0,0 +1,485 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.config.PDConfig;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.apache.hugegraph.pd.grpc.pulse.ChangeShard;
+import org.apache.hugegraph.pd.grpc.pulse.CleanPartition;
+import org.apache.hugegraph.pd.grpc.pulse.DbCompaction;
+import org.apache.hugegraph.pd.grpc.pulse.MovePartition;
+import org.apache.hugegraph.pd.grpc.pulse.PartitionKeyRange;
+import org.apache.hugegraph.pd.grpc.pulse.SplitPartition;
+import org.apache.hugegraph.pd.grpc.pulse.TransferLeader;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+public class StoreNodeServiceTest {
+    static PDConfig pdConfig;
+
+    @BeforeClass
+    public static void init() throws Exception {
+        String path = "tmp/unitTest";
+        deleteDirectory(new File(path));
+        pdConfig = new PDConfig() {{
+            this.setClusterId(100);
+            this.setInitialStoreList(
+                    "127.0.0.1:8500,127.0.0.1:8501,127.0.0.1:8502,127.0.0.1:8503,127.0.0.1:8504," +
+                    "127.0.0.1:8505");
+        }};
+
+        pdConfig.setStore(new PDConfig().new Store() {{
+            this.setMaxDownTime(3600);
+            this.setKeepAliveTimeout(3600);
+        }});
+
+        pdConfig.setPartition(new PDConfig().new Partition() {{
+            this.setShardCount(3);
+            this.setMaxShardsPerStore(3);
+        }});
+        pdConfig.setRaft(new PDConfig().new Raft() {{
+            this.setEnable(false);
+        }});
+        pdConfig.setDiscovery(new PDConfig().new Discovery());
+        pdConfig.setDataPath(path);
+        ConfigService configService = new ConfigService(pdConfig);
+        pdConfig = configService.loadConfig();
+    }
+
+    public static byte[] intToByteArray(int i) {
+        byte[] result = new byte[4];
+        result[0] = (byte) ((i >> 24) & 0xFF);
+        result[1] = (byte) ((i >> 16) & 0xFF);
+        result[2] = (byte) ((i >> 8) & 0xFF);
+        result[3] = (byte) (i & 0xFF);
+        return result;
+    }
+
+    public static void deleteDirectory(File dir) {
+        try {
+            FileUtils.deleteDirectory(dir);
+        } catch (IOException e) {
+            System.out.printf("Failed to start ....,%s%n", e.getMessage());
+        }
+    }
+
+    // @Test
+    public void testStoreNodeService() throws PDException {
+        Assert.assertEquals(pdConfig.getPartition().getTotalCount(),
+                            (long) pdConfig.getInitialStoreMap().size() *
+                            pdConfig.getPartition().getMaxShardsPerStore()
+                            / pdConfig.getPartition().getShardCount());
+        StoreNodeService storeService = new StoreNodeService(pdConfig);
+        int count = 6;
+        Metapb.Store[] stores = new Metapb.Store[count];
+        for (int i = 0; i < count; i++) {
+            Metapb.Store store = Metapb.Store.newBuilder()
+                                             .setId(0)
+                                             .setAddress("127.0.0.1:850" + i)
+                                             .setDeployPath("/data")
+                                             .addLabels(Metapb.StoreLabel.newBuilder()
+                                                                         .setKey("namespace")
+                                                                         .setValue("default")
+                                                                         .build())
+                                             .build();
+            stores[i] = storeService.register(store);
+            System.out.println("新注册store， id = " + stores[i].getId());
+        }
+        Assert.assertEquals(count, storeService.getStores("").size());
+
+        for (Metapb.Store store : stores) {
+            Metapb.StoreStats stats = Metapb.StoreStats.newBuilder()
+                                                       .setStoreId(store.getId())
+                                                       .build();
+            storeService.heartBeat(stats);
+        }
+
+        Assert.assertEquals(6, storeService.getActiveStores("").size());
+
+        Metapb.Graph graph = Metapb.Graph.newBuilder()
+                                         .setGraphName("defaultGH")
+                                         .setPartitionCount(10)
+                                         .build();
+        // 分配shard
+        List<Metapb.Shard> shards = storeService.allocShards(graph, 1);
+
+
+        Assert.assertEquals(3, shards.size());
+
+        Assert.assertEquals(pdConfig.getPartition().getTotalCount(),
+                            storeService.getShardGroups().size());        // 设置leader
+        Metapb.Shard leader = Metapb.Shard.newBuilder(shards.get(0))
+                                          .setRole(Metapb.ShardRole.Leader).build();
+        shards = new ArrayList<>(shards);
+        shards.set(0, leader);
+        // 增加shard
+        pdConfig.getPartition().setShardCount(5);
+
+        Metapb.ShardGroup shardGroup = Metapb.ShardGroup.newBuilder()
+                                                        .setId(1)
+                                                        .addAllShards(shards).build();
+        shards = storeService.reallocShards(shardGroup);
+
+        Assert.assertEquals(5, shards.size());
+        // 减少shard
+        pdConfig.getPartition().setShardCount(3);
+        shards = storeService.reallocShards(shardGroup);
+        Assert.assertEquals(3, shards.size());
+        // 包含leader，leader不能被删除
+        Assert.assertTrue(shards.contains(leader));
+
+        // 减少shard
+        pdConfig.getPartition().setShardCount(1);
+        graph = Metapb.Graph.newBuilder(graph).build();
+        shards = storeService.reallocShards(shardGroup);
+        Assert.assertEquals(1, shards.size());
+        // 包含leader，leader不能被删除
+        Assert.assertTrue(shards.contains(leader));
+
+        for (Metapb.Store store : stores) {
+            storeService.removeStore(store.getId());
+        }
+        Assert.assertEquals(0, storeService.getStores("").size());
+
+
+    }
+
+    // @Test
+    public void testSplitPartition() throws PDException {
+        StoreNodeService storeService = new StoreNodeService(pdConfig);
+        PartitionService partitionService = new PartitionService(pdConfig, storeService);
+
+        storeService.init(partitionService);
+        partitionService.addInstructionListener(new PartitionInstructionListener() {
+
+            @Override
+            public void changeShard(Metapb.Partition partition, ChangeShard changeShard) throws
+                                                                                         PDException {
+
+            }
+
+            @Override
+            public void transferLeader(Metapb.Partition partition,
+                                       TransferLeader transferLeader) throws PDException {
+
+            }
+
+            @Override
+            public void splitPartition(Metapb.Partition partition,
+                                       SplitPartition splitPartition) throws PDException {
+                splitPartition.getNewPartitionList().forEach(p -> {
+                    System.out.println("SplitPartition " + p.getId() + " " + p.getStartKey() + "," +
+                                       p.getEndKey());
+                });
+            }
+
+            @Override
+            public void dbCompaction(Metapb.Partition partition, DbCompaction dbCompaction) throws
+                                                                                            PDException {
+
+            }
+
+            @Override
+            public void movePartition(Metapb.Partition partition,
+                                      MovePartition movePartition) throws PDException {
+
+            }
+
+            @Override
+            public void cleanPartition(Metapb.Partition partition,
+                                       CleanPartition cleanPartition) throws PDException {
+
+            }
+
+            @Override
+            public void changePartitionKeyRange(Metapb.Partition partition,
+                                                PartitionKeyRange partitionKeyRange) throws
+                                                                                     PDException {
+
+            }
+        });
+        int count = 6;
+        Metapb.Store[] stores = new Metapb.Store[count];
+        for (int i = 0; i < count; i++) {
+            Metapb.Store store = Metapb.Store.newBuilder()
+                                             .setId(0)
+                                             .setAddress("127.0.0.1:850" + i)
+                                             .setDeployPath("/data")
+                                             .addLabels(Metapb.StoreLabel.newBuilder()
+                                                                         .setKey("namespace")
+                                                                         .setValue("default")
+                                                                         .build())
+                                             .build();
+            stores[i] = storeService.register(store);
+            System.out.println("新注册store， id = " + Long.toHexString(stores[i].getId()));
+        }
+        Assert.assertEquals(count, storeService.getStores().size());
+
+        Metapb.Graph graph = Metapb.Graph.newBuilder()
+                                         .setGraphName("defaultGH")
+                                         .build();
+        Metapb.PartitionShard ptShard =
+                partitionService.getPartitionByCode(graph.getGraphName(), 0);
+        System.out.println(ptShard.getPartition().getId());
+        {
+            Metapb.Partition pt = ptShard.getPartition();
+            System.out.println(pt.getId() + " " + pt.getStartKey() + "," + pt.getEndKey());
+        }
+
+        Assert.assertEquals(6, storeService.getShardGroups().size());
+        // storeService.splitShardGroups(ptShard.getPartition().getId(), 4);
+        Assert.assertEquals(9, storeService.getShardGroups().size());
+        storeService.getShardGroups().forEach(shardGroup -> {
+            System.out.println("shardGroup id = " + shardGroup.getId());
+        });
+    }
+
+    // @Test
+    public void testPartitionService() throws PDException, ExecutionException,
+                                              InterruptedException {
+        StoreNodeService storeService = new StoreNodeService(pdConfig);
+        int count = 6;
+        Metapb.Store[] stores = new Metapb.Store[count];
+        for (int i = 0; i < count; i++) {
+            Metapb.Store store = Metapb.Store.newBuilder()
+                                             .setId(0)
+                                             .setAddress(String.valueOf(i))
+                                             .setDeployPath("/data")
+                                             .addLabels(Metapb.StoreLabel.newBuilder()
+                                                                         .setKey("namespace")
+                                                                         .setValue("default")
+                                                                         .build())
+                                             .build();
+            stores[i] = storeService.register(store);
+            System.out.println("新注册store， id = " + Long.toHexString(stores[i].getId()));
+        }
+        Assert.assertEquals(count, storeService.getStores("").size());
+
+
+        PartitionService partitionService = new PartitionService(pdConfig, storeService);
+
+        Metapb.Graph graph = Metapb.Graph.newBuilder()
+                                         .setGraphName("defaultGH")
+
+                                         .setPartitionCount(10)
+                                         .build();
+        // 申请分区
+        Metapb.PartitionShard[] partitions = new Metapb.PartitionShard[10];
+        for (int i = 0; i < partitions.length; i++) {
+            partitions[i] =
+                    partitionService.getPartitionShard(graph.getGraphName(), intToByteArray(i));
+            Assert.assertEquals(3, storeService.getShardGroup(i).getShardsCount());
+        }
+        System.out.println(
+                "分区数量： " + partitionService.getPartitions(graph.getGraphName()).size());
+
+        int[] caseNo = {0}; //1 测试增加shard, 2 //测试store下线
+
+        Metapb.Shard leader = null;
+        int[] finalCaseNo = caseNo;
+
+        partitionService.addInstructionListener(new PartitionInstructionListener() {
+
+            @Override
+            public void changeShard(Metapb.Partition partition, ChangeShard changeShard) throws
+                                                                                         PDException {
+                switch (finalCaseNo[0]) {
+                    case 2:
+                        Assert.assertEquals(5, storeService.getShardGroup(partition.getId())
+                                                           .getShardsCount());
+                        break;
+                    case 3:
+                        storeService.getShardGroup(partition.getId()).getShardsList()
+                                    .forEach(shard -> {
+                                        Assert.assertNotEquals(shard.getStoreId(),
+                                                               stores[0].getId());
+                                    });
+                        break;
+                }
+
+            }
+
+            @Override
+            public void transferLeader(Metapb.Partition partition, TransferLeader transferLeader) {
+
+            }
+
+            @Override
+            public void splitPartition(Metapb.Partition partition, SplitPartition splitPartition) {
+            }
+
+            @Override
+            public void dbCompaction(Metapb.Partition partition, DbCompaction dbCompaction) throws
+                                                                                            PDException {
+
+            }
+
+            @Override
+            public void movePartition(Metapb.Partition partition,
+                                      MovePartition movePartition) throws PDException {
+
+            }
+
+            @Override
+            public void cleanPartition(Metapb.Partition partition,
+                                       CleanPartition cleanPartition) throws PDException {
+
+            }
+
+            @Override
+            public void changePartitionKeyRange(Metapb.Partition partition,
+                                                PartitionKeyRange partitionKeyRange)
+                    throws PDException {
+
+            }
+        });
+        Metapb.Partition partition = partitions[0].getPartition();
+        leader = Metapb.Shard.newBuilder(
+                storeService.getShardGroup(partition.getId()).getShardsList().get(0)).build();
+        Metapb.Shard finalLeader = leader;
+        partitionService.addStatusListener(new PartitionStatusListener() {
+            @Override
+            public void onPartitionChanged(Metapb.Partition partition,
+                                           Metapb.Partition newPartition) {
+
+            }
+
+            @Override
+            public void onPartitionRemoved(Metapb.Partition partition) {
+
+            }
+        });
+        // 测试修改图
+        caseNo[0] = 1;
+        partitionService.updateGraph(graph);
+        for (int i = 0; i < partitions.length; i++) {
+            partitions[i] =
+                    partitionService.getPartitionShard(graph.getGraphName(), intToByteArray(i));
+            Assert.assertEquals(3, storeService.getShardGroup(i).getShardsCount());
+        }
+
+        graph = Metapb.Graph.newBuilder(graph)
+                            .setGraphName("defaultGH")
+
+                            .setPartitionCount(10)
+                            .build();
+        caseNo[0] = 2;
+        partitionService.updateGraph(graph);
+
+        // 测试store离线
+        caseNo[0] = 3;
+        partitionService.storeOffline(stores[0]);
+
+
+        Metapb.PartitionStats stats = Metapb.PartitionStats.newBuilder()
+                                                           .addGraphName(partition.getGraphName())
+                                                           .setId(partition.getId())
+                                                           .setLeader(
+                                                                   Metapb.Shard.newBuilder(leader)
+                                                                               .setRole(
+                                                                                       Metapb.ShardRole.Leader))
+                                                           .build();
+        // 测试leader飘移
+        caseNo[0] = 4;
+        partitionService.partitionHeartbeat(stats);
+        AtomicReference<Metapb.Shard> shard = new AtomicReference<>();
+        Metapb.PartitionShard ss =
+                partitionService.getPartitionShardById(partition.getGraphName(), partition.getId());
+        storeService.getShardList(partition.getId()).forEach(s -> {
+            if (s.getRole() == Metapb.ShardRole.Leader) {
+                Assert.assertNull(shard.get());
+                shard.set(s);
+            }
+        });
+
+        Assert.assertEquals(leader.getStoreId(), shard.get().getStoreId());
+
+    }
+
+    // @Test
+    public void testMergeGraphParams() throws PDException {
+        StoreNodeService storeService = new StoreNodeService(pdConfig);
+        PartitionService partitionService = new PartitionService(pdConfig, storeService);
+
+        Metapb.Graph dfGraph = Metapb.Graph.newBuilder()
+
+                                           .setPartitionCount(
+                                                   pdConfig.getPartition().getTotalCount())
+
+                                           .build();
+
+        Metapb.Graph graph1 = Metapb.Graph.newBuilder()
+                                          .setGraphName("test")
+                                          .setPartitionCount(20)
+
+                                          .build();
+
+        Metapb.Graph graph2 = Metapb.Graph.newBuilder()
+                                          .setGraphName("test")
+                                          .setPartitionCount(7).build();
+        Metapb.Graph graph3 = Metapb.Graph.newBuilder()
+                                          .setGraphName("test")
+                                          .build();
+        Metapb.Graph graph4 = Metapb.Graph.newBuilder()
+                                          .setGraphName("test")
+                                          .build();
+
+        Metapb.Graph graph = Metapb.Graph.newBuilder(dfGraph).mergeFrom(graph2).build();
+        Assert.assertEquals(graph2.getGraphName(), graph.getGraphName());
+
+        Assert.assertEquals(graph2.getPartitionCount(), graph.getPartitionCount());
+
+
+        graph = Metapb.Graph.newBuilder(dfGraph).mergeFrom(graph3).build();
+        Assert.assertEquals(graph3.getGraphName(), graph.getGraphName());
+
+        Assert.assertEquals(dfGraph.getPartitionCount(), graph.getPartitionCount());
+
+
+        graph = Metapb.Graph.newBuilder(dfGraph).mergeFrom(graph4).build();
+        Assert.assertEquals(graph4.getGraphName(), graph.getGraphName());
+
+        Assert.assertEquals(dfGraph.getPartitionCount(), graph.getPartitionCount());
+
+    }
+
+    // @Test
+    public void test() {
+        int[] n = new int[3];
+
+
+        if (++n[2] > 1) {
+            System.out.println(n[2]);
+        }
+        if (++n[2] > 1) {
+            System.out.println(n[2]);
+        }
+        if (++n[2] > 1) {
+            System.out.println(n[2]);
+        }
+    }
+}

hugegraph-pd/hg-pd-core/src/test/java/org/apache/hugegraph/pd/UnitTestBase.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd;
+
+import java.io.File;
+
+public class UnitTestBase {
+    public static boolean deleteDir(File dir) {
+        if (dir.isDirectory()) {
+            for (File file : dir.listFiles()) {
+                deleteDir(file);
+            }
+        }
+        return dir.delete();
+    }
+}

hugegraph-pd/hg-pd-core/src/test/java/org/apache/hugegraph/pd/common/PartitionUtilsTest.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.pd.common;
+
+// import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.hugegraph.pd.common.PartitionUtils;
+
+public class PartitionUtilsTest {
+
+    // @Test
+    public void testHashCode() {
+        int partCount = 10;
+        int partSize = PartitionUtils.MAX_VALUE / partCount + 1;
+        int[] counter = new int[partCount];
+        for (int i = 0; i < 10000; i++) {
+            String s = String.format("BATCH-GET-UNIT-%02d", i);
+            int c = PartitionUtils.calcHashcode(s.getBytes(StandardCharsets.UTF_8));
+
+            counter[c / partSize]++;
+
+        }
+
+        for (int i = 0; i < counter.length; i++) {
+            System.out.println(i + " " + counter[i]);
+        }
+    }
+
+
+}