anph
/
hugegraph-sync
forked from hugegraph/hugegraph-sync
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refact: format root/parent modules & pd-service package name & rearrange code 

Change-Id: Id8ec0a0d754f646c59f7251158dd89de669b2016
This commit is contained in:
imbajin 2023-05-06 19:50:02 +08:00
parent 61ecb66e8f
commit fd13bb436b
49 changed files with 324 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.mvn/wrapper/MavenWrapperDownloader.java vendored
View File

@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.net.*;
import java.io.*;
import java.nio.channels.*;
@ -24,8 +25,9 @@ public class MavenWrapperDownloader {
/**
* Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
*/
private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
+ WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
private static final String DEFAULT_DOWNLOAD_URL =
"https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
+ WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
/**
* Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
@ -41,7 +43,8 @@ public class MavenWrapperDownloader {
".mvn/wrapper/maven-wrapper.jar";
/**
* Name of the property which should be used to override the default download url for the wrapper.
* Name of the property which should be used to override the default download url for the
* wrapper.
*/
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
@ -54,7 +57,7 @@ public class MavenWrapperDownloader {
// wrapperUrl parameter.
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
String url = DEFAULT_DOWNLOAD_URL;
if(mavenWrapperPropertyFile.exists()) {
if (mavenWrapperPropertyFile.exists()) {
FileInputStream mavenWrapperPropertyFileInputStream = null;
try {
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
@ -65,7 +68,7 @@ public class MavenWrapperDownloader {
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
} finally {
try {
if(mavenWrapperPropertyFileInputStream != null) {
if (mavenWrapperPropertyFileInputStream != null) {
mavenWrapperPropertyFileInputStream.close();
}
} catch (IOException e) {
@ -76,10 +79,11 @@ public class MavenWrapperDownloader {
System.out.println("- Downloading from: " + url);
File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
if(!outputFile.getParentFile().exists()) {
if(!outputFile.getParentFile().mkdirs()) {
if (!outputFile.getParentFile().exists()) {
if (!outputFile.getParentFile().mkdirs()) {
System.out.println(
"- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
"- ERROR creating output directory '" +
outputFile.getParentFile().getAbsolutePath() + "'");
}
}
System.out.println("- Downloading to: " + outputFile.getAbsolutePath());

16
.mvn/wrapper/maven-wrapper.properties vendored
View File

@ -1,2 +1,18 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.0/apache-maven-3.6.0-bin.zip
wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar

45
README.md
View File

@ -1,6 +1,9 @@
# PD部署说明
## PD配置
- 配置文件在application.yml
````
license:
# 验证使用的配置文件所在目录，包括主题、密码等
@ -23,15 +26,20 @@ partition:
# 默认每机器最大副本数,初始分区数= store-max-shard-count * store-number / default-shard-count
store-max-shard-count: 12
````
##store配置
-配置文件在application.yml,配置pdserver的address
````
pdserver:
# pd服务地址，多个pd地址用逗号分割
address: pdserver ip:端口
````
## Hugegraph配置
- 配置项在hugegraph的启动脚本start-hugegraph.sh中
````
if [ -z "$META_SERVERS" ];then
META_SERVERS="pdserver ip:端口"
@ -40,7 +48,9 @@ if [ -z "$PD_PEERS" ];then
PD_PEERS="pdserver ip:端口"
fi
````
## RESTFUL API
- pd提供了一些restful API可以获取集群分区，图，存储节点等一系列信息
###获取集群统计信息
@ -48,6 +58,7 @@ fi
#### 获取集群统计信息
###### Method & Url
```
GET http://localhost:8620/v1/cluster
```
@ -115,6 +126,7 @@ GET http://localhost:8620/v1/cluster
#### 获取pd集群成员信息
###### Method & Url
```
GET http://localhost:8620/v1/member
```
@ -160,19 +172,23 @@ GET http://localhost:8620/v1/member
"status": 0
}
```
###存储节点相关
#### 获取集群所有的store的信息
###### Method & Url
```
GET http://localhost:8620/v1/stores
```
###### Response Status
```json
200
```
###### Request Body
```json
@ -241,18 +257,23 @@ GET http://localhost:8620/v1/stores
"status": 0
}
```
#### 获取单个store的信息
###### Method & Url
```
GET http://localhost:8620/v1/store/{storeId}
```
###### Response Status
```json
200
```
###### Request Body
```json
{
"message": "OK",
@ -315,10 +336,13 @@ GET http://localhost:8620/v1/store/{storeId}
"status": 0
}
```
### 分区相关
#### 获取分区信息
###### Method & Url
```
GET http://localhost:8620/v1/highLevelPartitions
```
@ -475,10 +499,13 @@ GET http://localhost:8620/v1/highLevelPartitions
"status": 0
}
```
###获取图信息
#### 获取所有的图信息
###### Method & Url
```
GET http://localhost:8620/v1/graphs
```
@ -630,9 +657,11 @@ GET http://localhost:8620/v1/graphs
"status": 0
}
```
#### 获取单个图信息
###### Method & Url
```
GET http://localhost:8620/v1/graph/{graphName}
```
@ -726,10 +755,13 @@ GET http://localhost:8620/v1/graph/{graphName}
"status": 0
}
```
###获取shard的信息
#### 获取所有shard的信息
###### Method & Url
```
GET http://localhost:8620/v1/shards
```
@ -800,15 +832,19 @@ GET http://localhost:8620/v1/shards
"status": 0
}
```
###服务注册
#### 注册服务
###### Method & Url
```
POST http://127.0.0.1:8620/v1/registry
```
###### Request Body
```json
200
```
@ -826,6 +862,7 @@ POST http://127.0.0.1:8620/v1/registry
}
}
```
appName：所属服务名
version：所属服务版本号
address：服务实例地址+端口
@ -841,17 +878,21 @@ labels: 自定义标签，若服务名为'hg'即hugeserver时，需要提供key
"data": null
}
```
errorType：状态码
message：状态码为错误时的具体出错信息
data：无返回数据
#### 服务实例获取
###### Method & Url
```
POST http://127.0.0.1:8620/v1/registryInfo
```
###### Request Body
```json
200
```
@ -867,10 +908,11 @@ POST http://127.0.0.1:8620/v1/registryInfo
}
}
```
以下三项可全部为空，则获取所有服务节点的信息:
-- appName：过滤所属服务名的条件
-- version：过滤所属服务版本号的条件，此项有值，则appName不能为空
-- labels: 过滤自定义标签的条件
-- labels: 过滤自定义标签的条件
###### Response Body
@ -892,6 +934,7 @@ POST http://127.0.0.1:8620/v1/registryInfo
]
}
```
errorType：状态码
message：状态码为错误时的具体出错信息
data：获取的服务节点信息

17
build-pre.sh
View File

@ -1,4 +1,21 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
mkdir output
touch output/1
export JAVA_HOME=$ORACLEJDK_1_8_0_HOME

17
build.sh
View File

@ -1,4 +1,21 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
export PATH=$MAVEN_3_5_3_BIN:$ORACLEJDK_11_0_7_BIN:$PATH
export JAVA_HOME=$ORACLEJDK_11_0_7_HOME
export MAVEN_HOME=$MAVEN_3_5_3_HOME

22
ci.yml
View File

@ -1,22 +1,22 @@
Global:
tool : build_submitter
tool: build_submitter
languages:
# Java Version: 1.7, 1.8, 9, 10
- language : java
version : 11
- language: java
version: 11
envs:
# Maven Version: 3.0.4, 3.1.1, 3.2.5, 3.3.9, 3.5.3
- env : maven
version : 3.5.3
- env: maven
version: 3.5.3
Default:
profile : [dev]
profile: [ dev ]
Profiles:
- profile:
name : dev
tool : build_submitter
env : DECK_CENTOS7U5_K3
command : sh build.sh
release : true
name: dev
tool: build_submitter
env: DECK_CENTOS7U5_K3
command: sh build.sh
release: true
- profile:
name: deploy

17
deploy-release.sh
View File

@ -1,4 +1,21 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
readonly VER=3.6.3
readonly REPO_URL=http://10.14.139.8:8081/artifactory/star-local
#mvn -DnewVersion=${VER}-SNAPSHOT -DprocessAllModules=true -DgenerateBackupPoms=false versions:set

17
deploy-snapshot.sh
View File

@ -1,4 +1,21 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
readonly VER=3.6.3
readonly REPO_URL=http://10.14.139.8:8081/artifactory/star-local
#mvn -DnewVersion=${VER}-SNAPSHOT -DprocessAllModules=true -DgenerateBackupPoms=false versions:set

2
hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClientStubProxy.java
View File

@ -28,9 +28,9 @@ import io.grpc.stub.AbstractStub;
**/
public class AbstractClientStubProxy {
private final LinkedList<String> hostList = new LinkedList<>();
private AbstractBlockingStub blockingStub;
private AbstractStub stub;
private final LinkedList<String> hostList = new LinkedList<>();
public AbstractClientStubProxy(String[] hosts) {
for (String host : hosts) if (!host.isEmpty()) hostList.offer(host);

4
hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClient.java
View File

@ -45,12 +45,12 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class DiscoveryClient implements Closeable, Discoverable {
private final Timer timer = new Timer("serverHeartbeat", true);
private final AtomicBoolean requireResetStub = new AtomicBoolean(false);
protected int period; //心跳周期
LinkedList<String> pdAddresses = new LinkedList<>();
ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
private final Timer timer = new Timer("serverHeartbeat", true);
private volatile int currentIndex; // 当前在用pd地址位置
private final AtomicBoolean requireResetStub = new AtomicBoolean(false);
private int maxTime = 6;
private ManagedChannel channel = null;
private DiscoveryServiceGrpc.DiscoveryServiceBlockingStub registerStub;

53
hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/KvClient.java
View File

@ -171,6 +171,21 @@ public class KvClient<T extends WatchResponse> extends AbstractClient implements
}
};
}
public void listen(String key, Consumer<T> consumer) throws PDException {
StreamObserver<WatchResponse> observer = getObserver(key, consumer, listenWrapper);
acquire();
WatchRequest k = WatchRequest.newBuilder().setClientId(clientId.get()).setKey(key).build();
streamingCall(KvServiceGrpc.getWatchMethod(), k, observer, 1);
}
public void listenPrefix(String prefix, Consumer<T> consumer) throws PDException {
StreamObserver<WatchResponse> observer = getObserver(prefix, consumer, prefixListenWrapper);
acquire();
WatchRequest k =
WatchRequest.newBuilder().setClientId(clientId.get()).setKey(prefix).build();
streamingCall(KvServiceGrpc.getWatchPrefixMethod(), k, observer, 1);
} BiConsumer<String, Consumer> listenWrapper = (key, consumer) -> {
try {
listen(key, consumer);
@ -183,31 +198,6 @@ public class KvClient<T extends WatchResponse> extends AbstractClient implements
}
};
public void listen(String key, Consumer<T> consumer) throws PDException {
StreamObserver<WatchResponse> observer = getObserver(key, consumer, listenWrapper);
acquire();
WatchRequest k = WatchRequest.newBuilder().setClientId(clientId.get()).setKey(key).build();
streamingCall(KvServiceGrpc.getWatchMethod(), k, observer, 1);
} BiConsumer<String, Consumer> prefixListenWrapper = (key, consumer) -> {
try {
listenPrefix(key, consumer);
} catch (PDException e) {
try {
log.warn("start listenPrefix with warning:", e);
Thread.sleep(1000);
} catch (InterruptedException ex) {
}
}
};
public void listenPrefix(String prefix, Consumer<T> consumer) throws PDException {
StreamObserver<WatchResponse> observer = getObserver(prefix, consumer, prefixListenWrapper);
acquire();
WatchRequest k =
WatchRequest.newBuilder().setClientId(clientId.get()).setKey(prefix).build();
streamingCall(KvServiceGrpc.getWatchPrefixMethod(), k, observer, 1);
}
private void acquire() {
if (clientId.get() == 0L) {
try {
@ -247,7 +237,17 @@ public class KvClient<T extends WatchResponse> extends AbstractClient implements
values.put(key, value);
}
return values;
}
} BiConsumer<String, Consumer> prefixListenWrapper = (key, consumer) -> {
try {
listenPrefix(key, consumer);
} catch (PDException e) {
try {
log.warn("start listenPrefix with warning:", e);
Thread.sleep(1000);
} catch (InterruptedException ex) {
}
}
};
public LockResponse lock(String key, long ttl) throws PDException {
acquire();
@ -307,4 +307,5 @@ public class KvClient<T extends WatchResponse> extends AbstractClient implements
}

2
hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDClient.java
View File

@ -1201,8 +1201,8 @@ public class PDClient {
}
static class StubProxy {
private volatile PDGrpc.PDBlockingStub stub;
private final LinkedList<String> hostList = new LinkedList<>();
private volatile PDGrpc.PDBlockingStub stub;
public StubProxy(String[] hosts) {
for (String host : hosts) if (!host.isEmpty()) hostList.offer(host);

2
hg-pd-client/src/test/java/org/apache/hugegraph/pd/client/PDPulseTest.java
View File

@ -78,8 +78,8 @@ public class PDPulseTest {
private class PulseListener<T> implements PDPulse.Listener<T> {
CountDownLatch latch = new CountDownLatch(10);
private final String listenerName;
CountDownLatch latch = new CountDownLatch(10);
private PulseListener(CountDownLatch latch, String listenerName) {
this.latch = latch;

2
hg-pd-client/src/test/java/org/apache/hugegraph/pd/client/PDWatchTest.java
View File

@ -63,8 +63,8 @@ public class PDWatchTest {
}
private class WatchListener<T> implements PDWatch.Listener<T> {
CountDownLatch latch = new CountDownLatch(10);
private final String watcherName;
CountDownLatch latch = new CountDownLatch(10);
private WatchListener(CountDownLatch latch, String watcherName) {
this.latch = latch;

9
hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionCache.java
View File

@ -34,19 +34,16 @@ import com.google.common.collect.TreeRangeMap;
public class PartitionCache {
// 读写锁对象
private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
Lock writeLock = readWriteLock.writeLock();
// 每张图一个缓存
private volatile Map<String, RangeMap<Long, Integer>> keyToPartIdCache;
// graphName + PartitionID组成key
private volatile Map<String, Metapb.Partition> partitionCache;
private volatile Map<Integer, Metapb.ShardGroup> shardGroupCache;
private volatile Map<Long, Metapb.Store> storeCache;
private volatile Map<String, Metapb.Graph> graphCache;
// 读写锁对象
private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
Lock writeLock = readWriteLock.writeLock();
public PartitionCache() {
keyToPartIdCache = new HashMap<>();

2
hg-pd-core/src/main/java/org/apache/hugegraph/pd/ConfigService.java
View File

@ -32,8 +32,8 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class ConfigService implements RaftStateListener {
private PDConfig pdConfig;
private final ConfigMetaStore meta;
private PDConfig pdConfig;
public ConfigService(PDConfig config) {
this.pdConfig = config;

2
hg-pd-core/src/main/java/org/apache/hugegraph/pd/IdService.java
View File

@ -25,8 +25,8 @@ import com.baidu.hugegraph.pd.common.PDException;
public class IdService {
private PDConfig pdConfig;
private final IdMetaStore meta;
private PDConfig pdConfig;
public IdService(PDConfig config) {
this.pdConfig = config;

2
hg-pd-core/src/main/java/org/apache/hugegraph/pd/KvService.java
View File

@ -51,8 +51,8 @@ public class KvService {
private static final String LOCK_PREFIX = "L";
private static final String KV_PREFIX_DELIMITER = KV_PREFIX + KV_DELIMITER;
private static final byte[] EMPTY_VALUE = new byte[0];
private PDConfig pdConfig;
private final MetadataRocksDBStore meta;
private PDConfig pdConfig;
public KvService(PDConfig config) {
this.pdConfig = config;

4
hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreNodeService.java
View File

@ -59,14 +59,13 @@ public class StoreNodeService {
// Store状态监听
private final List<StoreStatusListener> statusListeners;
private final List<ShardGroupStatusListener> shardGroupStatusListeners;
private PartitionService partitionService;
private final StoreInfoMeta storeInfoMeta;
private final TaskInfoMeta taskInfoMeta;
private final Random random = new Random(System.currentTimeMillis());
private Metapb.ClusterStats clusterStats;
private final KvService kvService;
private final ConfigService configService;
private final PDConfig pdConfig;
private PartitionService partitionService;
private final Runnable quotaChecker = () -> {
try {
getQuota();
@ -76,6 +75,7 @@ public class StoreNodeService {
e);
}
};
private Metapb.ClusterStats clusterStats;
public StoreNodeService(PDConfig config) {
this.pdConfig = config;

5
hg-pd-core/src/main/java/org/apache/hugegraph/pd/TaskScheduleService.java
View File

@ -66,8 +66,6 @@ public class TaskScheduleService {
private final StoreMonitorDataService storeMonitorDataService;
private final KvService kvService;
private final LogService logService;
private long lastStoreTurnoffTime = 0;
private long lastBalanceLeaderTime = 0;
// 先按照value排序，再按照key排序
private final Comparator<KVPair<Long, Integer>> kvPairComparatorAsc = (o1, o2) -> {
if (o1.getValue() == o2.getValue()) {
@ -75,7 +73,6 @@ public class TaskScheduleService {
}
return o1.getValue().compareTo(o2.getValue());
};
// 先按照value排序(倒序)，再按照key排序(升序）
private final Comparator<KVPair<Long, Integer>> kvPairComparatorDesc = (o1, o2) -> {
if (o1.getValue() == o2.getValue()) {
@ -83,6 +80,8 @@ public class TaskScheduleService {
}
return o2.getValue().compareTo(o1.getValue());
};
private long lastStoreTurnoffTime = 0;
private long lastBalanceLeaderTime = 0;
public TaskScheduleService(PDConfig config, StoreNodeService storeService,

1
hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/DiscoveryMetaStore.java
View File

@ -41,6 +41,7 @@ public class DiscoveryMetaStore extends MetadataRocksDBStore {
//appName --> address --> registryInfo
private static final String PREFIX = "REGIS-";
private static final String SPLITTER = "-";
public DiscoveryMetaStore(PDConfig pdConfig) {
super(pdConfig);
}

1
hg-pd-core/src/main/java/org/apache/hugegraph/pd/meta/IdMetaStore.java
View File

@ -49,6 +49,7 @@ public class IdMetaStore extends MetadataRocksDBStore {
private static final ConcurrentHashMap<String, Object> SEQUENCES = new ConcurrentHashMap<>();
public static long CID_DEL_TIMEOUT = 24 * 3600 * 1000;
private final long clusterId;
public IdMetaStore(PDConfig pdConfig) {
super(pdConfig);
this.clusterId = pdConfig.getClusterId();

1
hg-pd-core/src/main/java/org/apache/hugegraph/pd/raft/RaftRpcProcessor.java
View File

@ -30,6 +30,7 @@ public class RaftRpcProcessor<T extends RaftRpcProcessor.BaseRequest> implements
private final Class<?> requestClass;
private final RaftEngine raftEngine;
public RaftRpcProcessor(Class<?> requestClass, RaftEngine raftEngine) {
this.requestClass = requestClass;
this.raftEngine = raftEngine;

2
hg-pd-dist/src/assembly/static/bin/stop-hugegraph-pd.sh
View File

@ -27,7 +27,7 @@ abs_path() {
echo "$( cd -P "$( dirname "$SOURCE" )" && pwd )"
}
BIN=`abs_path`
BIN=$(abs_path)
TOP="$(cd $BIN/../ && pwd)"
. "$BIN"/util.sh

2
hg-pd-dist/src/assembly/static/conf/application.yml
View File

@ -24,7 +24,7 @@ grpc:
server:
# rest服务端口号
port : 8620
port: 8620
pd:
# 存储路径

6
hg-pd-service/src/main/java/org/apache/hugegraph/pd/license/LicenseVerifierService.java
View File

@ -70,15 +70,15 @@ import lombok.extern.slf4j.Slf4j;
public class LicenseVerifierService {
private static final Duration CHECK_INTERVAL = Duration.ofMinutes(10);
private static LicenseContent content;
private static KvService kvService;
private static final String contentKey = "contentKey";
private static final Gson mapper = new Gson();
private static LicenseContent content;
private static KvService kvService;
private static volatile boolean installed = false;
private final MachineInfo machineInfo;
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
private final PDConfig pdConfig;
private final Instant lastCheckTime = Instant.now();
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// private final LicenseVerifyParam verifyParam;
private LicenseVerifyManager manager;
private ManagedChannel channel;

3
hg-pd-service/src/main/java/org/apache/hugegraph/pd/model/PromTargetsModel.java
View File

@ -30,9 +30,8 @@ public class PromTargetsModel {
private static final String LABEL_SCHEME = "__scheme__";
private static final String LABEL_JOB_NAME = "job";
private static final String LABEL_CLUSTER = "cluster";
private Set<String> targets = new HashSet<>();
private final Map<String, String> labels = new HashMap<>();
private Set<String> targets = new HashSet<>();
private PromTargetsModel() {
}

2
hg-pd-service/src/main/java/org/apache/hugegraph/pd/notice/NoticeBroadcaster.java
View File

@ -29,9 +29,9 @@ import lombok.extern.slf4j.Slf4j;
*/
@Slf4j
public class NoticeBroadcaster {
private final Supplier<Long> noticeSupplier;
private long noticeId;
private String durableId;
private final Supplier<Long> noticeSupplier;
private Supplier<String> durableSupplier;
private Function<String, Boolean> removeFunction;
private int state; //0=ready; 1=notified; 2=done ack; -1=error

2
hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/KvServiceGrpcImpl.java
View File

@ -64,12 +64,12 @@ import lombok.extern.slf4j.Slf4j;
public class KvServiceGrpcImpl extends KvServiceGrpc.KvServiceImplBase implements RaftStateListener,
ServiceGrpc {
private final ManagedChannel channel = null;
KvService kvService;
AtomicLong count = new AtomicLong();
String msg = "node is not leader,it is necessary to redirect to the leader on the client";
@Autowired
private PDConfig pdConfig;
private final ManagedChannel channel = null;
private KvWatchSubject subjects;
private ScheduledExecutorService executor;

1
hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/PDPulseService.java
View File

@ -53,6 +53,7 @@ public class PDPulseService extends HgPdPulseGrpc.HgPdPulseImplBase {
@Autowired
private PDConfig pdConfig;
private QueueStore queueStore = null;
public PDPulseService() {
PDPulseSubject.setQueueRetrieveFunction(() -> getQueue());
PDPulseSubject.setQueueDurableFunction(getQueueDurableFunction());

7
hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/PDService.java
View File

@ -84,9 +84,10 @@ import lombok.extern.slf4j.Slf4j;
public class PDService extends PDGrpc.PDImplBase implements RaftStateListener {
static String TASK_ID_KEY = "task_id";
private final Pdpb.ResponseHeader okHeader = Pdpb.ResponseHeader.newBuilder().setError(
Pdpb.Error.newBuilder().setType(Pdpb.ErrorType.OK)).build();
@Autowired
private PDConfig pdConfig;
private StoreNodeService storeNodeService;
private PartitionService partitionService;
private TaskScheduleService taskService;
@ -94,11 +95,7 @@ public class PDService extends PDGrpc.PDImplBase implements RaftStateListener {
private ConfigService configService;
private LogService logService;
private LicenseVerifierService licenseVerifierService;
private StoreMonitorDataService storeMonitorDataService;
private final Pdpb.ResponseHeader okHeader = Pdpb.ResponseHeader.newBuilder().setError(
Pdpb.Error.newBuilder().setType(Pdpb.ErrorType.OK)).build();
private ManagedChannel channel;
private Pdpb.ResponseHeader newErrorHeader(int errorCode, String errorMsg) {

4
hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/PromTargetsService.java
View File

@ -62,13 +62,13 @@ public class PromTargetsService {
.setScheme("http")
.setMetricsPath(
"/actuator/prometheus");
private final HgMapCache<String, Set<String>> targetsCache =
HgMapCache.expiredOf(24 * 60 * 60 * 1000);// expired after 24H.
@Autowired
private PDConfig pdConfig;
@Autowired
private PDService pdService;
private RegistryService register;
private final HgMapCache<String, Set<String>> targetsCache =
HgMapCache.expiredOf(24 * 60 * 60 * 1000);// expired after 24H.
private RegistryService getRegister() {
if (this.register == null) {

5
hg-pd-service/src/main/java/org/apache/hugegraph/pd/service/UpgradeService.java
View File

@ -17,12 +17,13 @@
package com.baidu.hugegraph.pd.service;
import org.apache.hugegraph.pd.upgrade.VersionScriptFactory;
import org.apache.hugegraph.pd.upgrade.VersionUpgradeScript;
import com.baidu.hugegraph.pd.KvService;
import com.baidu.hugegraph.pd.common.PDException;
import com.baidu.hugegraph.pd.config.PDConfig;
import com.baidu.hugegraph.pd.rest.API;
import com.baidu.hugegraph.pd.upgrade.VersionScriptFactory;
import com.baidu.hugegraph.pd.upgrade.VersionUpgradeScript;
import lombok.extern.slf4j.Slf4j;

9
hg-pd-service/src/main/java/com/baidu/hugegraph/pd/upgrade/VersionScriptFactory.java → hg-pd-service/src/main/java/org/apache/hugegraph/pd/upgrade/VersionScriptFactory.java
View File

@ -15,18 +15,17 @@
* under the License.
*/
package com.baidu.hugegraph.pd.upgrade;
package org.apache.hugegraph.pd.upgrade;
import java.util.LinkedList;
import java.util.List;
import com.baidu.hugegraph.pd.upgrade.scripts.PartitionMetaUpgrade;
import com.baidu.hugegraph.pd.upgrade.scripts.TaskCleanUpgrade;
import org.apache.hugegraph.pd.upgrade.scripts.PartitionMetaUpgrade;
import org.apache.hugegraph.pd.upgrade.scripts.TaskCleanUpgrade;
public class VersionScriptFactory {
private static volatile VersionScriptFactory factory;
private static final List<VersionUpgradeScript> scripts = new LinkedList<>();
private static volatile VersionScriptFactory factory;
static {
registerScript(new PartitionMetaUpgrade());

2
hg-pd-service/src/main/java/com/baidu/hugegraph/pd/upgrade/VersionUpgradeScript.java → hg-pd-service/src/main/java/org/apache/hugegraph/pd/upgrade/VersionUpgradeScript.java
View File

@ -15,7 +15,7 @@
* under the License.
*/
package com.baidu.hugegraph.pd.upgrade;
package org.apache.hugegraph.pd.upgrade;
import com.baidu.hugegraph.pd.config.PDConfig;

5
hg-pd-service/src/main/java/com/baidu/hugegraph/pd/upgrade/scripts/PartitionMetaUpgrade.java → hg-pd-service/src/main/java/org/apache/hugegraph/pd/upgrade/scripts/PartitionMetaUpgrade.java
View File

@ -15,16 +15,17 @@
* under the License.
*/
package com.baidu.hugegraph.pd.upgrade.scripts;
package org.apache.hugegraph.pd.upgrade.scripts;
import java.util.HashSet;
import org.apache.hugegraph.pd.upgrade.VersionUpgradeScript;
import com.baidu.hugegraph.pd.common.PDException;
import com.baidu.hugegraph.pd.config.PDConfig;
import com.baidu.hugegraph.pd.grpc.Metapb;
import com.baidu.hugegraph.pd.meta.MetadataKeyHelper;
import com.baidu.hugegraph.pd.meta.MetadataRocksDBStore;
import com.baidu.hugegraph.pd.upgrade.VersionUpgradeScript;
import lombok.extern.slf4j.Slf4j;

5
hg-pd-service/src/main/java/com/baidu/hugegraph/pd/upgrade/scripts/TaskCleanUpgrade.java → hg-pd-service/src/main/java/org/apache/hugegraph/pd/upgrade/scripts/TaskCleanUpgrade.java
View File

@ -15,13 +15,14 @@
* under the License.
*/
package com.baidu.hugegraph.pd.upgrade.scripts;
package org.apache.hugegraph.pd.upgrade.scripts;
import org.apache.hugegraph.pd.upgrade.VersionUpgradeScript;
import com.baidu.hugegraph.pd.common.PDException;
import com.baidu.hugegraph.pd.config.PDConfig;
import com.baidu.hugegraph.pd.meta.MetadataKeyHelper;
import com.baidu.hugegraph.pd.meta.MetadataRocksDBStore;
import com.baidu.hugegraph.pd.upgrade.VersionUpgradeScript;
import lombok.extern.slf4j.Slf4j;

2
hg-pd-service/src/main/java/org/apache/hugegraph/pd/watch/KvWatchSubject.java
View File

@ -52,9 +52,9 @@ public class KvWatchSubject {
public static final long WATCH_TTL = 20000L;
private static final ConcurrentMap<String, StreamObserver<WatchResponse>> clients =
new ConcurrentHashMap<>();
private final KvService kvService;
BiPredicate<String, String> equal = (kvKey, watchKey) -> kvKey.equals(watchKey);
BiPredicate<String, String> startWith = (kvKey, watchKey) -> kvKey.startsWith(watchKey);
private final KvService kvService;
/**
* 会使用以下三组key:

2
hg-pd-service/src/main/resources/application.yml
View File

@ -24,7 +24,7 @@ license:
license-path: 'conf/hugegraph.license'
server:
port : 8620
port: 8620
pd:
# 定期检查集群是否健康的时间间隔，单位秒

4
hg-pd-service/src/test/resources/application-server0.yml
View File

@ -18,12 +18,12 @@ grpc:
max-inbound-message-size: 100MB
server:
port : 8620
port: 8620
pd:
patrol-interval: 3000000
data-path: tmp/8686
data-path: tmp/8686
# 最少节点数，少于该数字，集群停止入库
initial-store-count: 1
# 初始store列表，在列表内的store自动激活

4
hg-pd-service/src/test/resources/application-server1.yml
View File

@ -18,12 +18,12 @@ grpc:
max-inbound-message-size: 100MB
server:
port : 8620
port: 8620
pd:
patrol-interval: 3000000
data-path: tmp/8686
data-path: tmp/8686
# 最少节点数，少于该数字，集群停止入库
initial-store-count: 1
# 初始store列表，在列表内的store自动激活

2
hg-pd-service/src/test/resources/application-server2.yml
View File

@ -19,7 +19,7 @@ grpc:
max-inbound-message-size: 100MB
server:
port : 8621
port: 8621
pd:
# 集群ID，区分不同的PD集群

2
hg-pd-service/src/test/resources/application-server3.yml
View File

@ -19,7 +19,7 @@ grpc:
max-inbound-message-size: 100MB
server:
port : 8622
port: 8622
pd:
# 集群ID，区分不同的PD集群

6
hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/HgAssertTest.java
View File

@ -110,8 +110,10 @@ public class HgAssertTest {
@Test
public void testIsContains() {
assertTrue(HgAssert.isContains(new Object[]{Integer.valueOf(1), Long.valueOf(2)}, Long.valueOf(2)));
assertFalse(HgAssert.isContains(new Object[]{Integer.valueOf(1), Long.valueOf(2)}, Long.valueOf(3)));
assertTrue(HgAssert.isContains(new Object[]{Integer.valueOf(1), Long.valueOf(2)},
Long.valueOf(2)));
assertFalse(HgAssert.isContains(new Object[]{Integer.valueOf(1), Long.valueOf(2)},
Long.valueOf(3)));
}
@Test

17
local-release.sh
View File

@ -1,4 +1,21 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
readonly VER=3.6.3
mvn -DnewVersion=${VER}-SNAPSHOT -DprocessAllModules=true -DgenerateBackupPoms=false versions:set

28
mvnw vendored
View File

@ -1,22 +1,20 @@
#!/bin/sh
# ----------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# ----------------------------------------------------------------------------
#
# ----------------------------------------------------------------------------
# Maven Start Up Batch script

21
pom.xml
View File

@ -1,6 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

25
settings.xml
View File

@ -1,6 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<settings xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<!-- 修改这里更改本地缓存路径
<localRepository>
@ -71,7 +88,9 @@
</repository>
<repository>
<id>Baidu_Local_Snapshots</id>
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots</url>
<url>
http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots
</url>
<releases>
<enabled>false</enabled>
</releases>

17
start_pd_server.sh
View File

@ -1,3 +1,20 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
function rename()
{
cfilelist=$(find -maxdepth 1 -type d -printf '%f\n' )