forked from hugegraph/hugegraph-sync
history merged until 2023.4.23
This commit is contained in:
V_Galaxy
commit
03b6d3761b
|
|
@ -0,0 +1,53 @@
|
|||
**/target/
|
||||
**/output/
|
||||
.DS_Store
|
||||
**/tmp/
|
||||
**/tmp*/
|
||||
|
||||
*.log
|
||||
.idea/
|
||||
*.iml
|
||||
**[^storage]/log/
|
||||
**/rocksdb-data/
|
||||
dist/
|
||||
.settings/
|
||||
.project
|
||||
.classpath
|
||||
.factorypath
|
||||
|
||||
# vscode
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
*.code-workspace
|
||||
|
||||
# Local History for Visual Studio Code
|
||||
.history/
|
||||
|
||||
|
||||
# Compiled class file
|
||||
*.class
|
||||
|
||||
# Log file
|
||||
*.log
|
||||
|
||||
# BlueJ files
|
||||
*.ctxt
|
||||
|
||||
# Mobile Tools for Java (J2ME)
|
||||
.mtj.tmp/
|
||||
|
||||
# Package Files #
|
||||
*.jar
|
||||
*.war
|
||||
*.nar
|
||||
*.ear
|
||||
*.zip
|
||||
*.tar.gz
|
||||
*.rar
|
||||
|
||||
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
|
||||
hs_err_pid*
|
||||
/logs/
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
/*
|
||||
* Copyright 2007-present the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import java.net.*;
|
||||
import java.io.*;
|
||||
import java.nio.channels.*;
|
||||
import java.util.Properties;
|
||||
|
||||
public class MavenWrapperDownloader {
|
||||
|
||||
private static final String WRAPPER_VERSION = "0.5.6";
|
||||
/**
|
||||
* Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
|
||||
*/
|
||||
private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
|
||||
+ WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
|
||||
|
||||
/**
|
||||
* Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
|
||||
* use instead of the default one.
|
||||
*/
|
||||
private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
|
||||
".mvn/wrapper/maven-wrapper.properties";
|
||||
|
||||
/**
|
||||
* Path where the maven-wrapper.jar will be saved to.
|
||||
*/
|
||||
private static final String MAVEN_WRAPPER_JAR_PATH =
|
||||
".mvn/wrapper/maven-wrapper.jar";
|
||||
|
||||
/**
|
||||
* Name of the property which should be used to override the default download url for the wrapper.
|
||||
*/
|
||||
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
|
||||
|
||||
public static void main(String args[]) {
|
||||
System.out.println("- Downloader started");
|
||||
File baseDirectory = new File(args[0]);
|
||||
System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
|
||||
|
||||
// If the maven-wrapper.properties exists, read it and check if it contains a custom
|
||||
// wrapperUrl parameter.
|
||||
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
|
||||
String url = DEFAULT_DOWNLOAD_URL;
|
||||
if(mavenWrapperPropertyFile.exists()) {
|
||||
FileInputStream mavenWrapperPropertyFileInputStream = null;
|
||||
try {
|
||||
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
|
||||
Properties mavenWrapperProperties = new Properties();
|
||||
mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
|
||||
url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
|
||||
} catch (IOException e) {
|
||||
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
|
||||
} finally {
|
||||
try {
|
||||
if(mavenWrapperPropertyFileInputStream != null) {
|
||||
mavenWrapperPropertyFileInputStream.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
// Ignore ...
|
||||
}
|
||||
}
|
||||
}
|
||||
System.out.println("- Downloading from: " + url);
|
||||
|
||||
File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
|
||||
if(!outputFile.getParentFile().exists()) {
|
||||
if(!outputFile.getParentFile().mkdirs()) {
|
||||
System.out.println(
|
||||
"- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
|
||||
}
|
||||
}
|
||||
System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
|
||||
try {
|
||||
downloadFileFromURL(url, outputFile);
|
||||
System.out.println("Done");
|
||||
System.exit(0);
|
||||
} catch (Throwable e) {
|
||||
System.out.println("- Error downloading");
|
||||
e.printStackTrace();
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
private static void downloadFileFromURL(String urlString, File destination) throws Exception {
|
||||
if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
|
||||
String username = System.getenv("MVNW_USERNAME");
|
||||
char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
|
||||
Authenticator.setDefault(new Authenticator() {
|
||||
@Override
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(username, password);
|
||||
}
|
||||
});
|
||||
}
|
||||
URL website = new URL(urlString);
|
||||
ReadableByteChannel rbc;
|
||||
rbc = Channels.newChannel(website.openStream());
|
||||
FileOutputStream fos = new FileOutputStream(destination);
|
||||
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
|
||||
fos.close();
|
||||
rbc.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip
|
||||
wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
## 启用无PD模式
|
||||
- hgstore设置yml文件 **app.fake-pd: true**
|
||||
- hugegraph设置properties文件 **pd.fake=true**，设置**hstore.peers=** 指向hgstore的grpc地址，多个地址逗号分割
|
||||
|
||||
````
|
||||
backend=hstore
|
||||
serializer=binary
|
||||
pd.peers=localhost:9000
|
||||
pd.fake=true
|
||||
hstore.peers=127.0.0.1:9081,127.0.0.1:9082,127.0.0.1:9083
|
||||
store=hugegraph
|
||||
````
|
||||
|
||||
###FakePD集群配置
|
||||
如果启用fakePD模式的集群部署，需要在hgstore的yml文件增加fake-pd配置
|
||||
+ **store-list:** store集群列表
|
||||
+ **peers-list:** raft集群列表
|
||||
|
||||
````
|
||||
fake-pd:
|
||||
store-list: 127.0.0.1:9080,127.0.0.1:9081,127.0.0.1:9082
|
||||
peers-list: 127.0.0.1:8080,127.0.0.1:8081,127.0.0.1:8082
|
||||
````
|
||||
|
||||
###本地打包
|
||||
|
||||
项目根目录下执行：
|
||||
````
|
||||
./mvnw -Dmaven.test.skip=true package
|
||||
````
|
||||
文件保存在项目根目录下的 dist 文件夹中。
|
||||
|
||||
###本地单测
|
||||
- 在跑单测之前需要启动一个pd和store服务
|
||||
- 所以的单测统一在hg-store-test模块中撰写，client、common、core等与项目的模块一一对应，相应模块下的单测写到对应的文件夹下
|
||||
- 保持目录结构与项目模块对应，文件命名必须包含"Test"字样，把编写好的类加到对应的SuiteTest类中，由Suite调度执行
|
||||
- 单测可以用编译工具单个执行，也可以到代码的根目录下执行：mvn verify -Pjacoco
|
||||
- 代码覆盖率查看，需要先执行：mvn verify -Pjacoco 后，在到根目录的target->site->jacoco->index.html
|
||||
|
|
@ -0,0 +1,521 @@
|
|||
# HStore存储部署说明
|
||||
##安装etcd
|
||||
- 配置etcd时需要确保其他服务器可以访问，注意配置文件中以下修改
|
||||
````
|
||||
name: etcd-1 #节点名称
|
||||
data-dir: default.etcd/ #节点数据存储目录
|
||||
listen-client-urls: http://0.0.0.0:2379 #监听etcd客户发送信息的地址
|
||||
advertise-client-urls: http://0.0.0.0:2379 #客户与本节点交互信息所用地址
|
||||
````
|
||||
## PD配置
|
||||
- 配置文件在application.yml
|
||||
````
|
||||
grpc:
|
||||
port: 9000 #内部grpc通信用端口，如心跳等使用
|
||||
max-inbound-message-size: 100MB #grpc最大入参的大小
|
||||
server:
|
||||
port : 9001 #restful端口，如通过http的get方法获取storeNode信息(/v1/stores)等
|
||||
etcd:
|
||||
address: http://localhost:2379 #etcd地址
|
||||
timeout: 3 #etcd连接超时时间
|
||||
store:
|
||||
keepAlive-timeout: 60 #store心跳超时时间，超过该时间，认为store临时不可用，转移Leader到其他副本,单位秒
|
||||
down-time: 1800 #store下线时间。超过该时间，认为store永久不可用，分配副本到其他机器，单位秒
|
||||
partition:
|
||||
default-total-count: 12 #默认分区总数
|
||||
default-shard-count: 3 #默认每个分区副本数
|
||||
````
|
||||
## StoreNode配置
|
||||
````
|
||||
grpc:
|
||||
port: 9080
|
||||
host: 127.0.0.1 #grpc的服务地址，给HugeServer使用
|
||||
netty-server:
|
||||
max-inbound-message-size: 100MB #grpc最大入参的大小
|
||||
server:
|
||||
port : 9090 #restful端口，如通过http的get方法获取监控统计信息(/metrics)等
|
||||
app:
|
||||
rocks:
|
||||
config-path: "/hugegraph.properties" #写入Rocksdb的参数配置文件
|
||||
raft:
|
||||
address: 127.0.0.1:8081 #raft通信地址
|
||||
data-path: /test/raft/8081 #raft日志写入目录
|
||||
timeout: 10 #raft读写超时时间,单位秒
|
||||
metrics: false #统计raft监控统计信息，打开可能对性能有影响
|
||||
pdserver:
|
||||
address: localhost:9000 #PD的grpc地址
|
||||
````
|
||||
StoreNode在启动后，如果在PD的日志中打印出其注册信息，则代表注册成功，如：Store register, id = **** address = ****
|
||||
## Hugegraph配置
|
||||
- 配置文件在properties文件，如hugegraph.properties
|
||||
````
|
||||
backend=hstore #后端分布式存储类型，固定为hstore
|
||||
serializer=binary #序列化器，固定为binary
|
||||
pd.peers=localhost:9000 #PD的grpc地址
|
||||
````
|
||||
## RESTFUL API
|
||||
- pd提供了一些restful API可以获取分区、存储节点等一系列信息
|
||||
|
||||
###图相关
|
||||
|
||||
#### 获取图的partition、shard信息
|
||||
|
||||
###### Method & Url
|
||||
```
|
||||
GET http://localhost:9001/v1/graphs
|
||||
```
|
||||
|
||||
###### Response Status
|
||||
|
||||
```json
|
||||
200
|
||||
```
|
||||
|
||||
###### Response Body
|
||||
|
||||
```json
|
||||
{
|
||||
"status": 0,
|
||||
"graphs": [
|
||||
{
|
||||
"graphName": "default/hugegraph/g",
|
||||
"partitionCount": 60,
|
||||
"shardCount": 3
|
||||
},
|
||||
{
|
||||
"graphName": "default/hugegraph/m",
|
||||
"partitionCount": 60,
|
||||
"shardCount": 3
|
||||
},
|
||||
{
|
||||
"graphName": "default/hugegraph/s",
|
||||
"partitionCount": 60,
|
||||
"shardCount": 3
|
||||
},
|
||||
{
|
||||
"graphName": "default/system/g",
|
||||
"partitionCount": 60,
|
||||
"shardCount": 3
|
||||
},
|
||||
{
|
||||
"graphName": "default/system/m",
|
||||
"partitionCount": 60,
|
||||
"shardCount": 3
|
||||
},
|
||||
{
|
||||
"graphName": "default/system/s",
|
||||
"partitionCount": 60,
|
||||
"shardCount": 3
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
#### 获取指定图的partition、shard信息
|
||||
|
||||
###### Method & Url
|
||||
```
|
||||
GET http://localhost:9001/v1/graph/default/hugegraph/g
|
||||
```
|
||||
|
||||
###### Response Status
|
||||
|
||||
```json
|
||||
200
|
||||
```
|
||||
|
||||
###### Response Body
|
||||
|
||||
```json
|
||||
{
|
||||
"status": 0,
|
||||
"graph": {
|
||||
"graphName": "default/hugegraph/g",
|
||||
"partitionCount": 60,
|
||||
"shardCount": 3
|
||||
}
|
||||
}
|
||||
```
|
||||
#### 修改指定图的partition、shard信息
|
||||
|
||||
###### Method & Url
|
||||
```
|
||||
POST http://localhost:9001/v1/graph/default/hugegraph/g
|
||||
```
|
||||
###### Request Body
|
||||
|
||||
```json
|
||||
{
|
||||
"partitionCount":90,
|
||||
"shardCount": 4,
|
||||
"workMode":"Batch_Import"
|
||||
}
|
||||
```
|
||||
|
||||
###### Response Status
|
||||
|
||||
```json
|
||||
200
|
||||
```
|
||||
|
||||
###### Response Body
|
||||
|
||||
```json
|
||||
{
|
||||
"status": 0,
|
||||
"graph": {
|
||||
"graphName": "default/hugegraph/g",
|
||||
"partitionCount": 60,
|
||||
"shardCount": 4,
|
||||
"workMode": "Batch_Import"
|
||||
}
|
||||
}
|
||||
```
|
||||
### 分区相关
|
||||
#### 获取分区信息
|
||||
|
||||
###### Method & Url
|
||||
```
|
||||
GET http://localhost:9001/v1/partitions
|
||||
```
|
||||
|
||||
###### Response Status
|
||||
|
||||
```json
|
||||
200
|
||||
```
|
||||
|
||||
###### Response Body
|
||||
|
||||
```json
|
||||
{
|
||||
"partitions": [
|
||||
{
|
||||
"endKey": 2731,
|
||||
"graphName": "DEFAULT/hugegraph18e1/g",
|
||||
"id": 0,
|
||||
"shards": [
|
||||
{
|
||||
"address": "10.14.139.56:8500",
|
||||
"committedIndex": "7396553",
|
||||
"role": "Follower",
|
||||
"state": "online",
|
||||
"storeId": "6140115507143799097"
|
||||
},
|
||||
{
|
||||
"address": "10.14.139.61:8500",
|
||||
"committedIndex": "7399964",
|
||||
"role": "Follower",
|
||||
"state": "online",
|
||||
"storeId": "3661113370728329034"
|
||||
},
|
||||
{
|
||||
"address": "10.14.139.57:8500",
|
||||
"committedIndex": "7394615",
|
||||
"role": "Leader",
|
||||
"state": "online",
|
||||
"storeId": "3384290400781292563"
|
||||
}
|
||||
],
|
||||
"startKey": 0,
|
||||
"timestamp": "2022-03-28 17:29:56",
|
||||
"version": 23,
|
||||
"workMode": "Normal",
|
||||
"workState": "State_Normal"
|
||||
}
|
||||
],
|
||||
"status": 0
|
||||
}
|
||||
```
|
||||
###存储节点相关
|
||||
#### 获取存储节点信息
|
||||
|
||||
###### Method & Url
|
||||
```
|
||||
GET http://localhost:9001/v1/stores
|
||||
```
|
||||
|
||||
###### Response Status
|
||||
|
||||
```json
|
||||
200
|
||||
```
|
||||
|
||||
###### Response Body
|
||||
|
||||
```json
|
||||
{
|
||||
"status": 0,
|
||||
"stores": [
|
||||
{
|
||||
"address": "10.14.139.60:8500",
|
||||
"dataVersion": 1,
|
||||
"id": "1779882393067551022",
|
||||
"labels": [
|
||||
{
|
||||
"key": "rest.port",
|
||||
"value": "8520"
|
||||
}
|
||||
],
|
||||
"lastHeartbeat": "1648459470389",
|
||||
"raftAddress": "10.14.139.60:8510",
|
||||
"startTimestamp": "1648395903516",
|
||||
"state": "Up",
|
||||
"stats": {
|
||||
"available": "4081010491392",
|
||||
"capacity": "7545950388224",
|
||||
"graphStats": [
|
||||
{
|
||||
"approximateKeys": "1933169027",
|
||||
"approximateSize": "191628815980",
|
||||
"committedIndex": "7301460",
|
||||
"graphName": "DEFAULT/hugegraph18e1/g",
|
||||
"partitionId": 1,
|
||||
"role": "Leader",
|
||||
"workMode": "Normal",
|
||||
"workState": "State_Normal"
|
||||
},
|
||||
{
|
||||
"approximateKeys": "1919472248",
|
||||
"approximateSize": "189192672333",
|
||||
"committedIndex": "7301705",
|
||||
"graphName": "DEFAULT/hugegraph18e1/g",
|
||||
"partitionId": 2,
|
||||
"role": "Leader",
|
||||
"workMode": "Normal",
|
||||
"workState": "State_Normal"
|
||||
}
|
||||
],
|
||||
"partitionCount": 12,
|
||||
"startTime": 1648395902,
|
||||
"storeId": "1779882393067551022",
|
||||
"usedSize": "4851786575920"
|
||||
},
|
||||
"version": "3.6.3"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
#### 删除存储节点
|
||||
|
||||
###### Method & Url
|
||||
```
|
||||
DELETE http://localhost:9001/v1/store/2549332866463202176
|
||||
```
|
||||
|
||||
###### Response Status
|
||||
|
||||
```json
|
||||
200
|
||||
```
|
||||
|
||||
###### Response Body
|
||||
````
|
||||
OK
|
||||
````
|
||||
#### 服务注册
|
||||
|
||||
###### Method & Url
|
||||
```
|
||||
post http://127.0.0.1:8620/v1/registry
|
||||
```
|
||||
###### Request Body
|
||||
|
||||
```json
|
||||
{
|
||||
"appName":"aaaa",
|
||||
"version":"version1",
|
||||
"address":"address1",
|
||||
"interval":"9223372036854775807",
|
||||
"labels": {
|
||||
"aaa": "aaaavalue"
|
||||
}
|
||||
}
|
||||
```
|
||||
````
|
||||
appName：所属服务名
|
||||
version：所属服务版本号
|
||||
address：服务实例地址+端口
|
||||
interval：实例心跳间隔，字符串，最大9223372036854775807
|
||||
labels: 自定义标签
|
||||
````
|
||||
###### Response Status
|
||||
|
||||
```json
|
||||
200
|
||||
```
|
||||
|
||||
###### Response Body
|
||||
```json
|
||||
{
|
||||
"errorType": "OK",
|
||||
"message": "",
|
||||
"data": null
|
||||
}
|
||||
````
|
||||
````
|
||||
errorType：状态码
|
||||
message：状态码为错误时的具体出错信息
|
||||
data：无返回数据
|
||||
````
|
||||
#### 服务实例获取
|
||||
|
||||
###### Method & Url
|
||||
```
|
||||
post http://127.0.0.1:8620/v1/registryInfo
|
||||
```
|
||||
###### Request Body
|
||||
|
||||
```json
|
||||
{
|
||||
"appName":"aaaa",
|
||||
"version":"version1",
|
||||
"labels": {
|
||||
"aaa": "aaaavalue"
|
||||
}
|
||||
}
|
||||
```
|
||||
````
|
||||
以下三项可全部为空，则获取所有服务节点的信息
|
||||
appName：过滤所属服务名的条件
|
||||
version：过滤所属服务版本号的条件，此项有值，则appName不能为空
|
||||
labels: 过滤自定义标签的条件
|
||||
````
|
||||
###### Response Status
|
||||
|
||||
```json
|
||||
200
|
||||
```
|
||||
|
||||
###### Response Body
|
||||
```json
|
||||
{
|
||||
"errorType": "OK",
|
||||
"message": null,
|
||||
"data": [
|
||||
{
|
||||
"id": null,
|
||||
"appName": "aaaa",
|
||||
"version": "version1",
|
||||
"address": "address1",
|
||||
"interval": "9223372036854775807",
|
||||
"labels": {
|
||||
"aaa": "aaaavalue"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
````
|
||||
errorType：状态码
|
||||
message：状态码为错误时的具体出错信息
|
||||
data：获取的服务节点信息
|
||||
````
|
||||
#### 服务注册
|
||||
|
||||
###### Method & Url
|
||||
```
|
||||
post http://127.0.0.1:8620/v1/registry
|
||||
```
|
||||
###### Request Body
|
||||
|
||||
```json
|
||||
{
|
||||
"appName":"aaaa",
|
||||
"version":"version1",
|
||||
"address":"address1",
|
||||
"interval":"9223372036854775807",
|
||||
"labels": {
|
||||
"aaa": "aaaavalue"
|
||||
}
|
||||
}
|
||||
```
|
||||
````
|
||||
appName：所属服务名
|
||||
version：所属服务版本号
|
||||
address：服务实例地址+端口
|
||||
interval：实例心跳间隔，字符串，最大9223372036854775807
|
||||
labels: 自定义标签
|
||||
````
|
||||
###### Response Status
|
||||
|
||||
```json
|
||||
200
|
||||
```
|
||||
|
||||
###### Response Body
|
||||
```json
|
||||
{
|
||||
"errorType": "OK",
|
||||
"message": "",
|
||||
"data": null
|
||||
}
|
||||
````
|
||||
````
|
||||
errorType：状态码
|
||||
message：状态码为错误时的具体出错信息
|
||||
data：无返回数据
|
||||
````
|
||||
#### 获取pd成员
|
||||
|
||||
###### Method & Url
|
||||
```
|
||||
GET http://127.0.0.1:8620/v1/members
|
||||
```
|
||||
###### Response Status
|
||||
|
||||
```json
|
||||
200
|
||||
```
|
||||
|
||||
###### Response Body
|
||||
```json
|
||||
{
|
||||
"status": 0,
|
||||
"graphs": {
|
||||
"members": [
|
||||
{
|
||||
"clusterId": "1",
|
||||
"raftUrl": "10.14.139.56:8620",
|
||||
"grpcUrl": "10.14.139.56:8686",
|
||||
"restUrl": "10.14.139.56:8620",
|
||||
"dataPath": "./pd_data",
|
||||
"state": "Up"
|
||||
},
|
||||
{
|
||||
"clusterId": "1",
|
||||
"raftUrl": "10.14.139.57:8620",
|
||||
"grpcUrl": "10.14.139.57:8686",
|
||||
"restUrl": "10.14.139.57:8620",
|
||||
"dataPath": "./pd_data",
|
||||
"state": "Up"
|
||||
},
|
||||
{
|
||||
"clusterId": "1",
|
||||
"raftUrl": "10.14.139.58:8620",
|
||||
"grpcUrl": "10.14.139.58:8686",
|
||||
"restUrl": "10.14.139.58:8620",
|
||||
"dataPath": "./pd_data",
|
||||
"state": "Up"
|
||||
}
|
||||
],
|
||||
"leader": {
|
||||
"clusterId": "1",
|
||||
"raftUrl": "10.14.139.56:8610",
|
||||
"grpcUrl": "10.14.139.56:8686",
|
||||
"dataPath": "./pd_data",
|
||||
"state": "Up"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
````
|
||||
clusterId：pd集群ID
|
||||
raftUrl、grpcUrl、restUrl：pd节点的地址信息
|
||||
dataPath：pd数据存储目录
|
||||
state: 节点当前状态
|
||||
````
|
||||
#### Store Mertics 地址
|
||||
```
|
||||
GET http://127.0.0.1:8520/actuator/prometheus
|
||||
|
||||
```
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
#!/bin/bash
|
||||
export PATH=$MAVEN_3_5_3_BIN:$ORACLEJDK_11_0_7_BIN:$PATH
|
||||
export JAVA_HOME=$ORACLEJDK_11_0_7_HOME
|
||||
export MAVEN_HOME=$MAVEN_3_5_3_HOME
|
||||
|
||||
readonly VER=3.6.3
|
||||
readonly REPO_URL=http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots
|
||||
|
||||
$MAVEN_3_5_3_BIN/mvn clean install -Dmaven.test.skip=true -DaltDeploymentRepository=Baidu_Local_Snapshots::default::${REPO_URL}
|
||||
|
||||
echo "mv dist...."
|
||||
mv dist output
|
||||
ls output
|
||||
echo "mv dist done"
|
||||
echo "show output...."
|
||||
ls output
|
||||
echo "show output done"
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
#!/bin/bash
|
||||
mkdir output
|
||||
touch output/1
|
||||
export JAVA_HOME=$ORACLEJDK_1_8_0_HOME
|
||||
|
||||
readonly VER=3.6.3
|
||||
readonly REPO_URL=http://10.14.139.8:8081/artifactory/star-local
|
||||
|
||||
$MAVEN_3_5_3_BIN/mvn -DremoveSnapshot=true -DprocessAllModules=true -DgenerateBackupPoms=true versions:set
|
||||
$MAVEN_3_5_3_BIN/mvn --settings ./settings.xml -Dmaven.test.skip=true -DaltDeploymentRepository=star-local::default::${REPO_URL} clean deploy
|
||||
$MAVEN_3_5_3_BIN/mvn versions:revert
|
||||
|
||||
#------------------repo-server-----------------------
|
||||
readonly FILE_NAME=hugegraph-store-3.6.3.tar.gz
|
||||
readonly REPO_URL_FILE=http://10.14.139.8:8081/artifactory/star-file
|
||||
|
||||
localFilePath=dist/${FILE_NAME}
|
||||
targetFolder="${REPO_URL_FILE}/dist/$(date '+%Y-%m-%d')/"
|
||||
artifactoryUser="admin"
|
||||
artifactoryPassword="JFrog12345"
|
||||
|
||||
md5Value="$(md5sum "$localFilePath")"
|
||||
md5Value="${md5Value:0:32}"
|
||||
sha1Value="$(sha1sum "$localFilePath")"
|
||||
sha1Value="${sha1Value:0:40}"
|
||||
sha256Value="$(sha256sum "$localFilePath")"
|
||||
sha256Value="${sha256Value:0:65}"
|
||||
|
||||
#curl -X PUT -u admin:JFrog12345 -T ${localFilePath} "${REPO_URL_FILE}/dist/${data_folder}/"
|
||||
echo "INFO: Uploading $localFilePath to $targetFolder"
|
||||
curl -i -X PUT -u "$artifactoryUser:$artifactoryPassword" \
|
||||
-H "X-Checksum-Md5: $md5Value" \
|
||||
-H "X-Checksum-Sha1: $sha1Value" \
|
||||
-H "X-Checksum-Sha256: $sha256Value" \
|
||||
-T "$localFilePath" \
|
||||
"$targetFolder"
|
||||
|
||||
|
||||
#------------------repo-client-----------------------
|
||||
readonly REPO_SNAPSHOT_PATH=http://10.14.139.8:8082/artifactory/star
|
||||
readonly REPO_FILE_PATH=http://10.14.139.8:8082/artifactory/star-file
|
||||
readonly REPO_BAIDU_LOCAL_PATH=http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local
|
||||
readonly PACKAGE_PATH=${REPO_SNAPSHOT_PATH}/com/baidu/hugegraph
|
||||
|
||||
dest=store-client-${VER}
|
||||
mkdir ${dest}
|
||||
|
||||
curl ${PACKAGE_PATH}/hg-store-client/${VER}/hg-store-client-${VER}.jar -o ./${dest}/hg-store-client-${VER}.jar
|
||||
curl ${PACKAGE_PATH}/hg-store-client/${VER}/hg-store-client-${VER}.pom -o ./${dest}/hg-store-client-pom.xml
|
||||
|
||||
curl ${PACKAGE_PATH}/hg-store-grpc/${VER}/hg-store-grpc-${VER}.jar -o ./${dest}/hg-store-grpc-${VER}.jar
|
||||
curl ${PACKAGE_PATH}/hg-store-grpc/${VER}/hg-store-grpc-${VER}.pom -o ./${dest}/hg-store-grpc-pom.xml
|
||||
|
||||
curl ${PACKAGE_PATH}/hg-store-common/${VER}/hg-store-common-${VER}.jar -o ./${dest}/hg-store-common-${VER}.jar
|
||||
curl ${PACKAGE_PATH}/hg-store-common/${VER}/hg-store-common-${VER}.pom -o ./${dest}/hg-store-common-pom.xml
|
||||
|
||||
curl ${PACKAGE_PATH}/hg-pd-client/${VER}/hg-pd-client-${VER}.jar -o ./${dest}/hg-pd-client-${VER}.jar
|
||||
curl ${PACKAGE_PATH}/hg-pd-client/${VER}/hg-pd-client-${VER}.pom -o ./${dest}/hg-pd-client-pom.xml
|
||||
|
||||
curl ${PACKAGE_PATH}/hg-pd-grpc/${VER}/hg-pd-grpc-${VER}.jar -o ./${dest}/hg-pd-grpc-${VER}.jar
|
||||
curl ${PACKAGE_PATH}/hg-pd-grpc/${VER}/hg-pd-grpc-${VER}.pom -o ./${dest}/hg-pd-grpc-pom.xml
|
||||
|
||||
curl ${PACKAGE_PATH}/hg-pd-common/${VER}/hg-pd-common-${VER}.jar -o ./${dest}/hg-pd-common-${VER}.jar
|
||||
curl ${PACKAGE_PATH}/hg-pd-common/${VER}/hg-pd-common-${VER}.pom -o ./${dest}/hg-pd-common-pom.xml
|
||||
|
||||
tar -cvf ${dest}.tar ${dest}
|
||||
|
||||
localFilePath=${dest}.tar
|
||||
targetFolder="${REPO_URL_FILE}/dist/$(date '+%Y-%m-%d')/"
|
||||
artifactoryUser="admin"
|
||||
artifactoryPassword="JFrog12345"
|
||||
|
||||
md5Value="$(md5sum "$localFilePath")"
|
||||
md5Value="${md5Value:0:32}"
|
||||
sha1Value="$(sha1sum "$localFilePath")"
|
||||
sha1Value="${sha1Value:0:40}"
|
||||
sha256Value="$(sha256sum "$localFilePath")"
|
||||
sha256Value="${sha256Value:0:65}"
|
||||
|
||||
#curl -X PUT -u admin:JFrog12345 -T ${localFilePath} "${REPO_URL_FILE}/dist/${data_folder}/"
|
||||
echo "INFO: Uploading $localFilePath to $targetFolder"
|
||||
curl -i -X PUT -u "$artifactoryUser:$artifactoryPassword" \
|
||||
-H "X-Checksum-Md5: $md5Value" \
|
||||
-H "X-Checksum-Sha1: $sha1Value" \
|
||||
-H "X-Checksum-Sha256: $sha256Value" \
|
||||
-T "$localFilePath" \
|
||||
"$targetFolder"
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
#!/bin/bash
|
||||
export PATH=$MAVEN_3_5_3_BIN:$ORACLEJDK_11_0_7_BIN:$PATH
|
||||
export JAVA_HOME=$ORACLEJDK_11_0_7_HOME
|
||||
export MAVEN_HOME=$MAVEN_3_5_3_HOME
|
||||
|
||||
readonly VER=3.6.3
|
||||
readonly REPO_URL=http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots
|
||||
|
||||
$MAVEN_3_5_3_BIN/mvn -Dmaven.test.skip=true -DaltDeploymentRepository=Baidu_Local_Snapshots::default::${REPO_URL} -U clean deploy
|
||||
echo "mv dist...."
|
||||
mv dist output
|
||||
ls output
|
||||
echo "mv dist done"
|
||||
echo "show output...."
|
||||
ls output
|
||||
echo "show output done"
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
Global:
|
||||
tool : build_submitter
|
||||
languages:
|
||||
# Java Version: 1.7, 1.8, 9, 10
|
||||
- language : java
|
||||
version : 11
|
||||
envs:
|
||||
# Maven Version: 3.0.4, 3.1.1, 3.2.5, 3.3.9, 3.5.3
|
||||
- env : maven
|
||||
version : 3.5.3
|
||||
Default:
|
||||
profile : [dev]
|
||||
Profiles:
|
||||
- profile:
|
||||
name : dev
|
||||
tool : build_submitter
|
||||
env : DECK_CENTOS7U5_K3
|
||||
command : sh build.sh
|
||||
release : true
|
||||
|
||||
- profile:
|
||||
name : change
|
||||
tool : build_submitter
|
||||
env : DECK_CENTOS7U5_K3
|
||||
command : sh build-change.sh
|
||||
release : true
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
management:
|
||||
metrics:
|
||||
export:
|
||||
prometheus:
|
||||
enabled: true
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: "*"
|
||||
|
||||
rocksdb:
|
||||
# rocksdb使用的总内存大小,达到该值强制写盘
|
||||
total_memory_size: 32000000000
|
||||
# rocksdb使用的memtable大小
|
||||
write_buffer_size: 32000000
|
||||
# 对于每个rocksdb来说，memtable个数达到该值进行写盘
|
||||
min_write_buffer_number_to_merge: 16
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
#!/bin/bash
|
||||
readonly VER=3.6.3
|
||||
readonly REPO_URL=http://10.14.139.8:8081/artifactory/star-local
|
||||
#mvn -DnewVersion=${VER}-SNAPSHOT -DprocessAllModules=true -DgenerateBackupPoms=false versions:set
|
||||
|
||||
./mvnw -DremoveSnapshot=true -DprocessAllModules=true -DgenerateBackupPoms=true versions:set
|
||||
./mvnw --settings ./settings.xml -Dmaven.test.skip=true -DaltDeploymentRepository=star-local::default::${REPO_URL} clean deploy
|
||||
./mvnw versions:revert
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
readonly REPO_URL=http://10.14.139.8:8081/artifactory/star-snapshot
|
||||
|
||||
mvn --settings ./settings.xml -Dmaven.test.skip=true -DaltDeploymentRepository=star-snapshot::default::${REPO_URL} clean deploy
|
||||
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
#!/bin/bash
|
||||
readonly REPO_URL=http://10.14.139.8:8081/artifactory/star-snapshot
|
||||
|
||||
mvn --settings ../settings.xml -Dmaven.test.skip=true -DaltDeploymentRepository=star-local::default::${REPO_URL} clean deploy
|
||||
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>hugegraph-store-root</artifactId>
|
||||
<groupId>com.baidu.hugegraph</groupId>
|
||||
<version>3.6.3-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>hg-store-cli</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-logging</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.baidu.hugegraph</groupId>
|
||||
<artifactId>hg-store-client</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.baidu.hugegraph</groupId>
|
||||
<artifactId>hg-store-grpc</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.baidu.hugegraph</groupId>
|
||||
<artifactId>hg-pd-client</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<mainClass>
|
||||
com.baidu.hugegraph.store.cli.StoreConsoleApplication
|
||||
</mainClass>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<distributionManagement>
|
||||
<repository>
|
||||
<id>Baidu_Local</id>
|
||||
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local</url>
|
||||
</repository>
|
||||
<snapshotRepository>
|
||||
<id>Baidu_Local_Snapshots</id>
|
||||
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots</url>
|
||||
</snapshotRepository>
|
||||
</distributionManagement>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
package com.baidu.hugegraph.store.cli;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2022/2/15
|
||||
*/
|
||||
@Data
|
||||
@Component
|
||||
public class AppConfig {
|
||||
|
||||
@Value("${pd.address}")
|
||||
private String pdAddress;
|
||||
|
||||
@Value("${net.kv.scanner.page.size}")
|
||||
private int scannerPageSize;
|
||||
|
||||
@Value("${scanner.graph}")
|
||||
private String scannerGraph;
|
||||
|
||||
@Value("${scanner.table}")
|
||||
private String scannerTable;
|
||||
|
||||
@Value("${scanner.max}")
|
||||
private int scannerMax;
|
||||
|
||||
@Value("${scanner.mod}")
|
||||
private int scannerModNumber;
|
||||
|
||||
@Value("${committer.graph}")
|
||||
private String committerGraph;
|
||||
|
||||
@Value("${committer.table}")
|
||||
private String committerTable;
|
||||
|
||||
@Value("${committer.amount}")
|
||||
private int committerAmount;
|
||||
}
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
package com.baidu.hugegraph.store.cli;
|
||||
|
||||
import com.baidu.hugegraph.pd.client.PDClient;
|
||||
import com.baidu.hugegraph.pd.client.PDConfig;
|
||||
import com.baidu.hugegraph.pd.common.PDException;
|
||||
import com.baidu.hugegraph.store.HgStoreClient;
|
||||
import com.baidu.hugegraph.store.cli.loader.HgThread2DB;
|
||||
import com.baidu.hugegraph.store.cli.scan.GrpcShardScanner;
|
||||
import com.baidu.hugegraph.store.cli.scan.HgStoreCommitter;
|
||||
import com.baidu.hugegraph.store.cli.scan.HgStoreScanner;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2022/2/14
|
||||
*/
|
||||
@SpringBootApplication
|
||||
@Slf4j
|
||||
public class StoreConsoleApplication implements CommandLineRunner {
|
||||
|
||||
@Autowired
|
||||
private AppConfig appConfig;
|
||||
|
||||
public static void main(String[] args) {
|
||||
log.info("Starting StoreConsoleApplication");
|
||||
SpringApplication.run(StoreConsoleApplication.class, args);
|
||||
log.info("StoreConsoleApplication finished.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(String... args) throws IOException, InterruptedException, PDException {
|
||||
if (args.length <= 0){
|
||||
// HgThread2DB hB = new HgThread2DB("localhost:8686");
|
||||
// hB.startMultiprocessQuery("12", "10");
|
||||
System.out.println("参数类型 cmd[-load, -query, -scan]");
|
||||
}else {
|
||||
switch (args[0]) {
|
||||
case "-load":
|
||||
HgThread2DB hgThread2DB = new HgThread2DB(args[1]);
|
||||
if (!args[3].isEmpty()) {
|
||||
hgThread2DB.setGraphName(args[3]);
|
||||
}
|
||||
try {
|
||||
if (args[2].equals("order")){
|
||||
hgThread2DB.testOrder(args[4]);
|
||||
}else {
|
||||
hgThread2DB.startMultiprocessInsert(args[2]);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
break;
|
||||
case "-query":
|
||||
HgThread2DB hgDB = new HgThread2DB(args[1]);
|
||||
try {
|
||||
hgDB.startMultiprocessQuery("12", args[2]);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
break;
|
||||
case "-scan":
|
||||
if ( args.length < 4){
|
||||
System.out.println("参数类型 -scan pd graphName tableName");
|
||||
}else {
|
||||
doScan(args[1], args[2], args[3]);
|
||||
}
|
||||
break;
|
||||
case "-shard":
|
||||
GrpcShardScanner scanner = new GrpcShardScanner();
|
||||
scanner.getData();
|
||||
break;
|
||||
case "-shard-single":
|
||||
scanner = new GrpcShardScanner();
|
||||
scanner.getDataSingle();
|
||||
break;
|
||||
default:
|
||||
System.out.println("参数类型错误，未执行任何程序");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void doCommit(){
|
||||
HgStoreCommitter committer=HgStoreCommitter.of(appConfig.getCommitterGraph());
|
||||
committer.put(appConfig.getScannerTable(),appConfig.getCommitterAmount());
|
||||
}
|
||||
|
||||
private void doScan(String pd, String graphName, String tableName) throws PDException {
|
||||
HgStoreClient storeClient = HgStoreClient.create(PDConfig.of(pd)
|
||||
.setEnableCache(true));
|
||||
|
||||
HgStoreScanner storeScanner = HgStoreScanner.of(storeClient, graphName);
|
||||
storeScanner.scanTable2(tableName);
|
||||
// storeScanner.scanHash();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,545 @@
|
|||
package com.baidu.hugegraph.store.cli.loader;
|
||||
|
||||
import com.baidu.hugegraph.pd.client.PDClient;
|
||||
import com.baidu.hugegraph.pd.client.PDConfig;
|
||||
import com.baidu.hugegraph.store.HgStoreClient;
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.HgStoreSession;
|
||||
import com.baidu.hugegraph.store.HgKvEntry;
|
||||
import com.baidu.hugegraph.store.HgKvIterator;
|
||||
import com.baidu.hugegraph.store.HgScanQuery;
|
||||
import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator;
|
||||
import com.baidu.hugegraph.store.client.util.MetricX;
|
||||
import com.baidu.hugegraph.store.cli.util.HgCliUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.BufferedReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
|
||||
/**
|
||||
* 使用pd，支持raft
|
||||
* 读取文件并多线程进行入库
|
||||
*/
|
||||
@Slf4j
|
||||
public class HgThread2DB {
|
||||
private static PDClient pdClient;
|
||||
public String graphName = "hugegraphtest";
|
||||
/*正在进行和在排队的任务的总数*/
|
||||
private static AtomicInteger taskTotal = new AtomicInteger(0);
|
||||
private static AtomicInteger queryTaskTotal = new AtomicInteger(0);
|
||||
private static ThreadPoolExecutor threadPool = null;
|
||||
private static ThreadPoolExecutor queryThreadPool = null;
|
||||
private static AtomicLong insertDataCount = new AtomicLong();
|
||||
private static AtomicLong queryCount = new AtomicLong();
|
||||
private static AtomicLong totalQueryCount = new AtomicLong();
|
||||
volatile long startTime = System.currentTimeMillis();
|
||||
private static int limitScanBatchCount = 100;
|
||||
private static ArrayBlockingQueue listQueue = null;
|
||||
private static AtomicLong longId = new AtomicLong();
|
||||
|
||||
private static CountDownLatch countDownLatch = null;
|
||||
private HgStoreClient storeClient;
|
||||
|
||||
public HgThread2DB(String pdAddr) {
|
||||
int threadCount = Runtime.getRuntime().availableProcessors();
|
||||
|
||||
listQueue = new ArrayBlockingQueue<List<HgOwnerKey>>(100000000);
|
||||
queryThreadPool = new ThreadPoolExecutor(500, 1000,
|
||||
200, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>( 1000));
|
||||
threadPool = new ThreadPoolExecutor(threadCount*2, threadCount * 3,
|
||||
200, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(threadCount + 100));
|
||||
storeClient = HgStoreClient.create(PDConfig.of(pdAddr)
|
||||
.setEnableCache(true));
|
||||
pdClient = storeClient.getPdClient();
|
||||
}
|
||||
|
||||
public void setGraphName(String graphName){
|
||||
this.graphName = graphName;
|
||||
log.info("setGraphName {}", graphName);
|
||||
}
|
||||
|
||||
public boolean singlePut(String tableName
|
||||
, List<String> keys) throws InterruptedException {
|
||||
HgStoreSession session = storeClient.openSession(graphName);
|
||||
session.beginTx();
|
||||
|
||||
keys.forEach((strKey) -> {
|
||||
insertDataCount.getAndIncrement();
|
||||
int j = strKey.indexOf("\t");
|
||||
// byte[] key = HgCliUtil.toBytes(strKey.substring(0, j));
|
||||
HgOwnerKey hgKey = HgCliUtil.toOwnerKey(strKey.substring(0, j), strKey);
|
||||
byte[] value = HgCliUtil.toBytes(strKey.substring(j + 1));
|
||||
session.put(tableName, hgKey, value);
|
||||
|
||||
});
|
||||
if ( insertDataCount.get() > 10000000){
|
||||
synchronized (insertDataCount){
|
||||
long count = insertDataCount.get();
|
||||
insertDataCount.set(0);
|
||||
if (count > 10000000){
|
||||
log.info("count : " + count + " qps : " + count * 1000 / (System.currentTimeMillis() - startTime)
|
||||
+" threadCount : " + taskTotal);
|
||||
startTime = System.currentTimeMillis();
|
||||
}
|
||||
}
|
||||
}
|
||||
// Thread.sleep(2000);
|
||||
if (!keys.isEmpty()) {
|
||||
if (session.isTx()) {
|
||||
session.commit();
|
||||
} else {
|
||||
session.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean singlePut(String tableName) throws InterruptedException {
|
||||
HgStoreSession session = storeClient.openSession(graphName);
|
||||
session.beginTx();
|
||||
|
||||
int maxlist = 100;
|
||||
|
||||
for (int y = 0; y < maxlist; y++) {
|
||||
insertDataCount.getAndIncrement();
|
||||
String strLine = "" + getLong() + getLong() + getLong() + getLong();
|
||||
// log.info("========{}",strLine);
|
||||
HgOwnerKey hgKey = HgCliUtil.toOwnerKey(strLine, strLine);
|
||||
byte[] value = HgCliUtil.toBytes(strLine);
|
||||
session.put(tableName, hgKey, value);
|
||||
}
|
||||
|
||||
if ( insertDataCount.get() > 10000000){
|
||||
synchronized (insertDataCount){
|
||||
long count = insertDataCount.get();
|
||||
insertDataCount.set(0);
|
||||
if (count > 10000000){
|
||||
log.info("count : " + count + " qps : " + count * 1000 / (System.currentTimeMillis() - startTime)
|
||||
+" threadCount : " + taskTotal);
|
||||
startTime = System.currentTimeMillis();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (session.isTx()) {
|
||||
session.commit();
|
||||
} else {
|
||||
session.rollback();
|
||||
}
|
||||
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean testOrder(String input) {
|
||||
String tableName = "hugegraph02";
|
||||
HgStoreSession session = storeClient.openSession(graphName);
|
||||
session.beginTx();
|
||||
int loop = Integer.parseInt(input);
|
||||
if (loop == 0){
|
||||
loop = 2000;
|
||||
}
|
||||
for (int i = 0; i < loop; i++) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
HgOwnerKey hgOwnerKey = HgCliUtil.toOwnerKey(startTime + "owner:" + i, startTime + "k:" + i);
|
||||
session.put(tableName, hgOwnerKey, HgCliUtil.toBytes(i));
|
||||
}
|
||||
|
||||
if (session.isTx()) {
|
||||
session.commit();
|
||||
} else {
|
||||
session.rollback();
|
||||
}
|
||||
|
||||
try {
|
||||
HgKvIterator<HgKvEntry> iterable = session.scanIterator(tableName);
|
||||
int x = 0;
|
||||
while (iterable.hasNext()) {
|
||||
HgKvEntry entry = iterable.next();
|
||||
// log.info("data:{}-{}", toStr(entry.key()), entry.value());
|
||||
x++;
|
||||
}
|
||||
log.info("x={}", x);
|
||||
} catch (Exception e) {
|
||||
log.error("query error, message ");
|
||||
// e.printStackTrace();
|
||||
}
|
||||
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 多线程读取文件入库
|
||||
* @throws IOException
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
public void startMultiprocessInsert(String filepath) throws IOException {
|
||||
log.info("--- start startMultiprocessInsert---");
|
||||
startTime = System.currentTimeMillis();
|
||||
File readfile = new File(filepath);
|
||||
MetricX metrics = null;
|
||||
long dataCount = 0;
|
||||
if (readfile.exists()) {
|
||||
// 读取文件
|
||||
InputStreamReader isr = new InputStreamReader(new FileInputStream(readfile), "UTF-8");
|
||||
BufferedReader reader = new BufferedReader(isr);
|
||||
|
||||
String strLine = null;
|
||||
String tableName = HgCliUtil.TABLE_NAME;
|
||||
// 积攒到多少个后执行线程入库,10万
|
||||
int maxlist = 100000;
|
||||
List<String> keys = new ArrayList<>(maxlist);
|
||||
metrics = MetricX.ofStart();
|
||||
try {
|
||||
while ((strLine = reader.readLine()) != null) {
|
||||
keys.add(strLine);
|
||||
dataCount++;
|
||||
|
||||
// 读取文件中的10000条数据，启一个线程入库
|
||||
if (dataCount % maxlist == 0) {
|
||||
List<String> finalKeys = keys;
|
||||
Runnable task = new Runnable() {
|
||||
public void run() {
|
||||
try {
|
||||
if (!finalKeys.isEmpty()) {
|
||||
boolean ret = singlePut(tableName, finalKeys);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
taskTotal.decrementAndGet();
|
||||
synchronized (taskTotal){
|
||||
taskTotal.notifyAll();
|
||||
}
|
||||
}
|
||||
};
|
||||
taskTotal.getAndIncrement();
|
||||
threadPool.execute(task);
|
||||
|
||||
while ( taskTotal.get() > 100){
|
||||
synchronized (taskTotal){
|
||||
taskTotal.wait();
|
||||
}
|
||||
}
|
||||
// keys.remove(0);
|
||||
keys = new ArrayList<>(maxlist);
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
isr.close();
|
||||
reader.close();
|
||||
// 把剩余的入库
|
||||
if (!keys.isEmpty()){
|
||||
List<String> finalKeys1 = keys;
|
||||
Runnable task = new Runnable() {
|
||||
public void run() {
|
||||
try {
|
||||
boolean ret = singlePut(tableName, finalKeys1);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
taskTotal.decrementAndGet();
|
||||
synchronized (taskTotal){
|
||||
taskTotal.notifyAll();
|
||||
}
|
||||
}
|
||||
};
|
||||
threadPool.execute(task);
|
||||
taskTotal.getAndIncrement();
|
||||
}
|
||||
while ( taskTotal.get() > 0){
|
||||
synchronized (taskTotal){
|
||||
try {
|
||||
taskTotal.wait(1000);
|
||||
if ( taskTotal.get() > 0){
|
||||
System.out.println("wait thread exit " + taskTotal.get());
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
threadPool.shutdown();
|
||||
|
||||
}else {
|
||||
System.out.println("样本文件不存在：" + filepath);
|
||||
}
|
||||
metrics.end();
|
||||
log.info("*************************************************");
|
||||
log.info(" 主进程执行时间:" + metrics.past() / 1000 + "秒，一共执行：" + dataCount + "条");
|
||||
log.info("*************************************************");
|
||||
System.out.println(" 主进程执行时间 " + metrics.past() / 1000 + "秒");
|
||||
System.out.println("-----主进程执行结束---------");
|
||||
}
|
||||
|
||||
/**
|
||||
* 多线程读取文件入库
|
||||
* @throws IOException
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
public void autoMultiprocessInsert() throws IOException {
|
||||
log.info("--- start autoMultiprocessInsert---");
|
||||
startTime = System.currentTimeMillis();
|
||||
|
||||
|
||||
MetricX metrics = null;
|
||||
long dataCount = 0;
|
||||
|
||||
String strLine = null;
|
||||
String tableName = HgCliUtil.TABLE_NAME;
|
||||
// 积攒到多少个后执行线程入库,10万
|
||||
int maxlist = 100000;
|
||||
List<String> keys = new ArrayList<>(maxlist);
|
||||
for (int x = 0; x < 10000000; x++) {
|
||||
metrics = MetricX.ofStart();
|
||||
try {
|
||||
Runnable task = new Runnable() {
|
||||
public void run() {
|
||||
try {
|
||||
boolean ret = singlePut(tableName);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
taskTotal.decrementAndGet();
|
||||
synchronized (taskTotal) {
|
||||
taskTotal.notifyAll();
|
||||
}
|
||||
}
|
||||
};
|
||||
taskTotal.getAndIncrement();
|
||||
threadPool.execute(task);
|
||||
|
||||
while (taskTotal.get() > 100) {
|
||||
synchronized (taskTotal) {
|
||||
taskTotal.wait();
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
while ( taskTotal.get() > 0){
|
||||
synchronized (taskTotal){
|
||||
try {
|
||||
taskTotal.wait(1000);
|
||||
if ( taskTotal.get() > 0){
|
||||
System.out.println("wait thread exit " + taskTotal.get());
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
threadPool.shutdown();
|
||||
|
||||
|
||||
metrics.end();
|
||||
log.info("*************************************************");
|
||||
log.info(" 主进程执行时间:" + metrics.past() / 1000 + "秒，一共执行：" + dataCount + "条");
|
||||
log.info("*************************************************");
|
||||
System.out.println(" 主进程执行时间 " + metrics.past() / 1000 + "秒");
|
||||
System.out.println("-----主进程执行结束---------");
|
||||
}
|
||||
|
||||
public String getLong() {
|
||||
//如果需要更长 或者更大冗余空间, 只需要 time * 10^n 即可
|
||||
//当前可保证1毫秒 生成 10000条不重复
|
||||
return String.format("%019x", longId.getAndIncrement());
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 执行查询，并将查询的结果做为下一次迭代的点放入队列
|
||||
*/
|
||||
private void queryAnd2Queue() {
|
||||
try {
|
||||
HgStoreSession session = storeClient.openSession(graphName);
|
||||
HashSet<String> hashSet = new HashSet<String>();
|
||||
while (!listQueue.isEmpty()) {
|
||||
|
||||
log.info(" ====== start scanBatch2 count:{} list:{}=============",
|
||||
queryThreadPool.getActiveCount(), listQueue.size());
|
||||
List<HgOwnerKey> keys = (List<HgOwnerKey>) listQueue.take();
|
||||
List<HgOwnerKey> newQueryList = new ArrayList<>();
|
||||
|
||||
KvCloseableIterator<HgKvIterator<HgKvEntry>> iterators =
|
||||
session.scanBatch2(
|
||||
HgScanQuery.prefixIteratorOf(HgCliUtil.TABLE_NAME, keys.iterator())
|
||||
);
|
||||
|
||||
while (iterators.hasNext()) {
|
||||
HgKvIterator<HgKvEntry> iterator = iterators.next();
|
||||
int insertQueueCount = 0;
|
||||
while (iterator.hasNext()) {
|
||||
HgKvEntry entry = iterator.next();
|
||||
String newPoint = HgCliUtil.toStr(entry.value());
|
||||
// log.info("query_key =" + newPoint);
|
||||
// 统计查询次数
|
||||
if (!newPoint.isEmpty() && hashSet.add(newPoint)) {
|
||||
queryCount.getAndIncrement();
|
||||
totalQueryCount.getAndIncrement();
|
||||
|
||||
HgOwnerKey hgKey = HgCliUtil.toOwnerKey(newPoint, newPoint);
|
||||
newQueryList.add(hgKey);
|
||||
|
||||
if (queryCount.get() > 1000000) {
|
||||
synchronized (queryCount) {
|
||||
long count = queryCount.get();
|
||||
queryCount.set(0);
|
||||
if (count > 1000000) {
|
||||
log.info("count : " + count + " qps : " + count * 1000 /
|
||||
(System.currentTimeMillis() - startTime)
|
||||
+ " threadCount : " + queryThreadPool.getActiveCount() + " queueSize:"
|
||||
+ listQueue.size());
|
||||
startTime = System.currentTimeMillis();
|
||||
}
|
||||
}
|
||||
}
|
||||
// 达到1万个点后，去查询一次
|
||||
if (newQueryList.size() > 10000 && listQueue.size() < 10000) {
|
||||
listQueue.put(newQueryList);
|
||||
insertQueueCount++;
|
||||
newQueryList = new ArrayList<>();
|
||||
if (insertQueueCount > 2) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// 一次查询如果不够1万，单独提交一次查询，确保所有的结果都能执行查询
|
||||
if (!newQueryList.isEmpty() && listQueue.size() < 1000) {
|
||||
listQueue.put(newQueryList);
|
||||
}
|
||||
|
||||
iterators.close();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
log.info("============= thread done ==============");
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
|
||||
/**
|
||||
* 多线程查询
|
||||
* @param point 起始查询点，后续根据这个点查询到的value做为下一次的查询条件进行迭代
|
||||
* @param scanCount 允许启动的线程数量
|
||||
* @throws IOException
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
public void startMultiprocessQuery(String point, String scanCount) throws IOException, InterruptedException {
|
||||
log.info("--- start startMultiprocessQuery---");
|
||||
startTime = System.currentTimeMillis();
|
||||
MetricX metrics = MetricX.ofStart();
|
||||
limitScanBatchCount = Integer.parseInt(scanCount);
|
||||
|
||||
CountDownLatch latch = new CountDownLatch(limitScanBatchCount);
|
||||
HgStoreSession session = storeClient.openSession(graphName);
|
||||
|
||||
final AtomicLong[] counter = {new AtomicLong()};
|
||||
final long[] start = {System.currentTimeMillis()};
|
||||
|
||||
LinkedBlockingQueue[] queue = new LinkedBlockingQueue[limitScanBatchCount];
|
||||
for(int i =0; i<limitScanBatchCount; i++) {
|
||||
queue[i] = new LinkedBlockingQueue();
|
||||
}
|
||||
List<String> strKey = Arrays.asList(new String[]{"20727483", "50329304", "26199460", "1177521", "27960125",
|
||||
"30440025", "15833920", "15015183", "33153097", "21250581"});
|
||||
strKey.forEach(key->{
|
||||
log.info("newkey:{}", key);
|
||||
HgOwnerKey hgKey = HgCliUtil.toOwnerKey(key, key);
|
||||
queue[0].add(hgKey);
|
||||
});
|
||||
|
||||
for(int i =0; i<limitScanBatchCount; i++) {
|
||||
int finalI = i;
|
||||
KvCloseableIterator<HgKvIterator<HgKvEntry>> iterators =
|
||||
session.scanBatch2(
|
||||
HgScanQuery.prefixIteratorOf(HgCliUtil.TABLE_NAME, new Iterator<HgOwnerKey>() {
|
||||
HgOwnerKey current = null;
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
while (current == null) {
|
||||
try {
|
||||
current = (HgOwnerKey) queue[finalI].poll(1, TimeUnit.SECONDS);
|
||||
} catch (InterruptedException e) {
|
||||
//
|
||||
}
|
||||
}
|
||||
if (current == null){
|
||||
log.info("===== current is null ==========");
|
||||
}
|
||||
return current != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgOwnerKey next() {
|
||||
return current;
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
new Thread(()->{
|
||||
while (iterators.hasNext()) {
|
||||
HgKvIterator<HgKvEntry> iterator = iterators.next();
|
||||
long c = 0;
|
||||
while (iterator.hasNext()) {
|
||||
String newPoint = HgCliUtil.toStr(iterator.next().value());
|
||||
HgOwnerKey newHgKey = HgCliUtil.toOwnerKey(newPoint, newPoint);
|
||||
if ( queue[(int) (c % limitScanBatchCount)].size() < 1000000) {
|
||||
queue[(int) (c % limitScanBatchCount)].add(newHgKey);
|
||||
}
|
||||
c++;
|
||||
}
|
||||
if ( counter[0].addAndGet(c) > 1000000){
|
||||
synchronized (counter) {
|
||||
if ( counter[0].get() > 10000000) {
|
||||
log.info("count {}, qps {}", counter[0].get(), counter[0].get() * 1000 / (System.currentTimeMillis() - start[0]));
|
||||
start[0] = System.currentTimeMillis();
|
||||
counter[0].set(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}, "client query thread:" + i ).start();
|
||||
log.info("===== read thread exit ==========");
|
||||
}
|
||||
latch.await();
|
||||
|
||||
|
||||
metrics.end();
|
||||
log.info("*************************************************");
|
||||
log.info(" 主进程执行时间:" + metrics.past() / 1000 + "秒; 查询：" + totalQueryCount.get()
|
||||
+ "次，qps:" + totalQueryCount.get() * 1000 / metrics.past());
|
||||
log.info("*************************************************");
|
||||
System.out.println("-----主进程执行结束---------");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,156 @@
|
|||
package com.baidu.hugegraph.store.cli.scan;
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import com.baidu.hugegraph.store.grpc.GraphStoreGrpc;
|
||||
import com.baidu.hugegraph.store.grpc.GraphStoreGrpc.GraphStoreStub;
|
||||
import com.baidu.hugegraph.store.grpc.Graphpb.ScanPartitionRequest;
|
||||
import com.baidu.hugegraph.store.grpc.Graphpb.ScanPartitionRequest.Reply;
|
||||
import com.baidu.hugegraph.store.grpc.Graphpb.ScanResponse;
|
||||
|
||||
import io.grpc.ManagedChannel;
|
||||
import io.grpc.ManagedChannelBuilder;
|
||||
import io.grpc.stub.StreamObserver;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* @author zhangyingjie
|
||||
* @date 2022/11/1
|
||||
**/
|
||||
|
||||
@Slf4j
|
||||
public class GrpcShardScanner {
|
||||
private volatile boolean closed = false;
|
||||
private AtomicInteger sum = new AtomicInteger();
|
||||
private ConcurrentHashMap<Long, StreamObserver<ScanPartitionRequest>>
|
||||
observers = new ConcurrentHashMap<>();
|
||||
|
||||
public void getData() {
|
||||
ExecutorService service = new ThreadPoolExecutor(500, Integer.MAX_VALUE,
|
||||
0L,
|
||||
TimeUnit.MILLISECONDS,
|
||||
new LinkedBlockingQueue<>());
|
||||
long start = System.currentTimeMillis();
|
||||
|
||||
String[] addresses = new String[]{"10.14.139.71:8500",
|
||||
"10.14.139.70:8500",
|
||||
"10.14.139.69:8500"};
|
||||
int pSize = 72;
|
||||
int size = pSize * addresses.length;
|
||||
CountDownLatch latch = new CountDownLatch(size);
|
||||
for (int j = 0; j < pSize; j++) {
|
||||
for (int i = 0; i < addresses.length; i++) {
|
||||
String address = addresses[i];
|
||||
int finalJ = j;
|
||||
service.execute(() -> getData(finalJ, latch, address));
|
||||
}
|
||||
}
|
||||
try {
|
||||
latch.await();
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
long cost = end - start;
|
||||
log.info("all rows are: {}, cost: {},avg: {}", sum.get(),
|
||||
cost, sum.get() / cost * 1000);
|
||||
}
|
||||
|
||||
public void getData(int pId, CountDownLatch latch, String address) {
|
||||
try {
|
||||
ScanPartitionRequest.Builder builder =
|
||||
ScanPartitionRequest.newBuilder();
|
||||
ScanPartitionRequest.Request.Builder srb =
|
||||
ScanPartitionRequest.Request.newBuilder();
|
||||
ScanPartitionRequest.Request request =
|
||||
srb.setGraphName("DEFAULT/hugegraph2/g")
|
||||
.setScanType(
|
||||
ScanPartitionRequest.ScanType.SCAN_EDGE)
|
||||
.setTable("g+oe").setBoundary(0x10)
|
||||
.setPartitionId(pId).build();
|
||||
ManagedChannel c =
|
||||
ManagedChannelBuilder.forTarget(address)
|
||||
.usePlaintext().build();
|
||||
int maxSize = 1024 * 1024 * 1024;
|
||||
GraphStoreStub stub;
|
||||
stub = GraphStoreGrpc.newStub(c)
|
||||
.withMaxInboundMessageSize(maxSize)
|
||||
.withMaxOutboundMessageSize(maxSize);
|
||||
|
||||
AtomicInteger count = new AtomicInteger();
|
||||
long start = System.currentTimeMillis();
|
||||
long id = Thread.currentThread().getId();
|
||||
StreamObserver<ScanResponse> ro =
|
||||
new StreamObserver<ScanResponse>() {
|
||||
@Override
|
||||
public void onNext(ScanResponse value) {
|
||||
try {
|
||||
int edgeSize = value.getEdgeCount();
|
||||
int vertexSize = value.getVertexCount();
|
||||
if (request.getScanType().equals(
|
||||
ScanPartitionRequest.ScanType.SCAN_VERTEX)) {
|
||||
count.getAndAdd(vertexSize);
|
||||
} else {
|
||||
count.getAndAdd(edgeSize);
|
||||
}
|
||||
// String print = JsonFormat.printer().print
|
||||
// (value);
|
||||
// System.out.println(print);
|
||||
ScanPartitionRequest.Builder builder =
|
||||
ScanPartitionRequest.newBuilder();
|
||||
builder.setScanRequest(request);
|
||||
Reply.Builder reply = Reply.newBuilder();
|
||||
reply.setSeqNo(1);
|
||||
builder.setReplyRequest(reply);
|
||||
observers.get(id).onNext(builder.build());
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(Throwable t) {
|
||||
log.warn("调用grpc接口发生错误", t);
|
||||
latch.countDown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onCompleted() {
|
||||
long time = System.currentTimeMillis() - start;
|
||||
log.info("scan id : {}, complete: {} ,time:{}",
|
||||
pId, count.get(), time);
|
||||
sum.addAndGet(count.get());
|
||||
latch.countDown();
|
||||
}
|
||||
};
|
||||
StreamObserver<ScanPartitionRequest> observer =
|
||||
stub.scanPartition(ro);
|
||||
observers.put(id, observer);
|
||||
builder.setScanRequest(request);
|
||||
observer.onNext(builder.build());
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void getDataSingle() {
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
new Thread(() -> getData(58, latch, "10.14.139.71:8500")).start();
|
||||
try {
|
||||
latch.await();
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
log.info("all rows are: {}", sum.get());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
package com.baidu.hugegraph.store.cli.scan;
|
||||
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.HgSessionManager;
|
||||
import com.baidu.hugegraph.store.HgStoreSession;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeManager;
|
||||
|
||||
import static com.baidu.hugegraph.store.cli.util.HgCliUtil.*;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2022/2/28
|
||||
*/
|
||||
public class HgStoreCommitter {
|
||||
|
||||
protected final static HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance();
|
||||
|
||||
private final String graph;
|
||||
|
||||
|
||||
public static HgStoreCommitter of(String graph) {
|
||||
return new HgStoreCommitter(graph);
|
||||
}
|
||||
|
||||
private HgStoreCommitter(String graph) {
|
||||
this.graph = graph;
|
||||
}
|
||||
|
||||
protected HgStoreSession getStoreSession() {
|
||||
return HgSessionManager.getInstance().openSession(this.graph);
|
||||
}
|
||||
|
||||
protected HgStoreSession getStoreSession(String graphName) {
|
||||
return HgSessionManager.getInstance().openSession(graphName);
|
||||
}
|
||||
|
||||
public void put(String tableName,int amount) {
|
||||
//*************** Put Benchmark **************//*
|
||||
String keyPrefix = "PUT-BENCHMARK";
|
||||
HgStoreSession session = getStoreSession();
|
||||
|
||||
int length = String.valueOf(amount).length();
|
||||
|
||||
session.beginTx();
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
for (int i = 0; i < amount; i++) {
|
||||
HgOwnerKey key = toOwnerKey(keyPrefix + "-" + padLeftZeros(String.valueOf(i), length));
|
||||
byte[] value = toBytes(keyPrefix + "-V-" + i);
|
||||
|
||||
session.put(tableName, key, value);
|
||||
|
||||
if ((i + 1) % 100_000 == 0) {
|
||||
println("---------- " + (i + 1) + " --------");
|
||||
println("Preparing took: " + (System.currentTimeMillis() - start) + " ms.");
|
||||
session.commit();
|
||||
println("Committing took: " + (System.currentTimeMillis() - start) + " ms.");
|
||||
start = System.currentTimeMillis();
|
||||
session.beginTx();
|
||||
}
|
||||
}
|
||||
|
||||
if (session.isTx()) {
|
||||
session.commit();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,203 @@
|
|||
package com.baidu.hugegraph.store.cli.scan;
|
||||
|
||||
import com.baidu.hugegraph.pd.client.PDClient;
|
||||
import com.baidu.hugegraph.pd.common.PDException;
|
||||
import com.baidu.hugegraph.pd.grpc.Metapb;
|
||||
import com.baidu.hugegraph.store.*;
|
||||
import com.baidu.hugegraph.store.cli.util.HgCliUtil;
|
||||
import com.baidu.hugegraph.store.cli.util.HgMetricX;
|
||||
import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConfig;
|
||||
import com.baidu.hugegraph.store.client.util.MetricX;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2022/2/14
|
||||
*/
|
||||
@Slf4j
|
||||
public class HgStoreScanner {
|
||||
|
||||
private HgStoreClient storeClient;
|
||||
private final String graphName;
|
||||
private long modNumber = 1_000_000;
|
||||
private int max = 10_000_000;
|
||||
|
||||
public long getModNumber() {
|
||||
return modNumber;
|
||||
}
|
||||
|
||||
public void setModNumber(int modNumber) {
|
||||
if (modNumber <= 0) {
|
||||
return;
|
||||
}
|
||||
this.modNumber = modNumber;
|
||||
}
|
||||
|
||||
public int getMax() {
|
||||
return max;
|
||||
}
|
||||
|
||||
public void setMax(int max) {
|
||||
if (modNumber <= 0) {
|
||||
return;
|
||||
}
|
||||
this.max = max;
|
||||
}
|
||||
|
||||
public static HgStoreScanner of(HgStoreClient storeClient, String graph) {
|
||||
return new HgStoreScanner(storeClient, graph);
|
||||
}
|
||||
|
||||
private HgStoreScanner(HgStoreClient storeClient, String graph) {
|
||||
this.storeClient = storeClient;
|
||||
this.graphName = graph;
|
||||
}
|
||||
|
||||
protected HgStoreSession getStoreSession() {
|
||||
return HgSessionManager.getInstance().openSession(this.graphName);
|
||||
}
|
||||
|
||||
protected HgStoreSession getStoreSession(String graphName) {
|
||||
return HgSessionManager.getInstance().openSession(graphName);
|
||||
}
|
||||
|
||||
public void scanTable(String tableName) {
|
||||
log.info("Starting scan table [{}] of graph [{}] ...", tableName, graphName);
|
||||
HgMetricX hgMetricX = HgMetricX.ofStart();
|
||||
HgStoreSession session = getStoreSession();
|
||||
int count = 0;
|
||||
KvCloseableIterator<HgKvIterator<HgKvEntry>> iterator = session.scanBatch2(HgScanQuery.tableOf(tableName));
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
while (iterator.hasNext()) {
|
||||
HgKvIterator<HgKvEntry> iterator2 = iterator.next();
|
||||
while (iterator2.hasNext()) {
|
||||
|
||||
count++;
|
||||
iterator2.next();
|
||||
if (count % (modNumber) == 0) {
|
||||
log.info("Scanning keys: " + count + " time is " + modNumber * 1000
|
||||
/ (System.currentTimeMillis() - start));
|
||||
start = System.currentTimeMillis();
|
||||
}
|
||||
if (count == max) {
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
iterator.close();
|
||||
|
||||
hgMetricX.end();
|
||||
log.info("*************************************************");
|
||||
log.info("************* Scanning Completed **************");
|
||||
log.info("Graph: {}", graphName);
|
||||
log.info("Table: {}", tableName);
|
||||
log.info("Keys: {}", count);
|
||||
log.info("Max: {}", max);
|
||||
log.info("Waiting: {} seconds.", MetricX.getIteratorWait() / 1000);
|
||||
log.info("Total: {} seconds.", hgMetricX.past() / 1000);
|
||||
log.info("Iterator: [{}]", iterator.getClass().getSimpleName());
|
||||
log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize());
|
||||
log.info("*************************************************");
|
||||
}
|
||||
|
||||
public void scanHash() {
|
||||
|
||||
String tableName = "g+i";
|
||||
HgMetricX hgMetricX = HgMetricX.ofStart();
|
||||
String graphName = "/DEFAULT/graphs/hugegraph1/";
|
||||
HgStoreSession session = getStoreSession(graphName);
|
||||
int count = 0;
|
||||
String query = "{\"conditions\":[{\"cls\":\"S\",\"el\":{\"key\":\"ID\",\"relation\":\"SCAN\",\"value\"" +
|
||||
":{\"start\":\"61180\",\"end\":\"63365\",\"length\":0}}}],\"optimizedType\":\"NONE\",\"ids\":[]," +
|
||||
"\"mustSortByInput\":true,\"resultType\":\"EDGE\",\"offset\":0," +
|
||||
"\"actualOffset\":0,\"actualStoreOffset\":" +
|
||||
"0,\"limit\":9223372036854775807,\"capacity\":-1,\"showHidden\":false,\"showDeleting\":false," +
|
||||
"\"showExpired\":false,\"olap\":false,\"withProperties\":false,\"olapPks\":[]}";
|
||||
//HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,0,715827883, HgKvStore.SCAN_ANY,null);
|
||||
|
||||
//HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,61180,63365, 348, null);
|
||||
//HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,0,65535, 348, null);
|
||||
HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName);
|
||||
while (iterator.hasNext()) {
|
||||
|
||||
count++;
|
||||
//iterator.next();
|
||||
// if (count % (modNumber) == 0) {
|
||||
// log.info("Scanning keys: " + count);
|
||||
HgCliUtil.println(Arrays.toString(iterator.next().key()));
|
||||
// }
|
||||
if (count == max) {
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
hgMetricX.end();
|
||||
log.info("*************************************************");
|
||||
log.info("************* Scanning Completed **************");
|
||||
log.info("Graph: {}", this.graphName);
|
||||
log.info("Table: {}", tableName);
|
||||
log.info("Keys: {}", count);
|
||||
log.info("Max: {}", max);
|
||||
log.info("Waiting: {} seconds.", MetricX.getIteratorWait() / 1000);
|
||||
log.info("Total: {} seconds.", hgMetricX.past() / 1000);
|
||||
log.info("Iterator: [{}]", iterator.getClass().getSimpleName());
|
||||
log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize());
|
||||
log.info("*************************************************");
|
||||
}
|
||||
|
||||
public static final byte[] EMPTY_BYTES = new byte[0];
|
||||
|
||||
public void scanTable2(String tableName) throws PDException {
|
||||
// java -jar hg-store-cli-3.6.0-SNAPSHOT.jar -scan 10.45.30.212:8989 "DEFAULT/case_112/g" g+ie
|
||||
PDClient pdClient = storeClient.getPdClient();
|
||||
List<Metapb.Partition> partitions = pdClient.getPartitions(0, graphName);
|
||||
HgStoreSession session = storeClient.openSession(graphName);
|
||||
int count = 0;
|
||||
byte[] position = null;
|
||||
HgMetricX hgMetricX = HgMetricX.ofStart();
|
||||
for (Metapb.Partition partition : partitions) {
|
||||
while (true) {
|
||||
try (HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,
|
||||
(int) (partition.getStartKey()), (int) (partition.getEndKey()),
|
||||
HgKvStore.SCAN_HASHCODE, EMPTY_BYTES)) {
|
||||
if (position != null) {
|
||||
iterator.seek(position);
|
||||
}
|
||||
while (iterator.hasNext()) {
|
||||
iterator.next();
|
||||
count++;
|
||||
if (count % 3000 == 0) {
|
||||
if (iterator.hasNext()) {
|
||||
iterator.next();
|
||||
position = iterator.position();
|
||||
System.out.println("count is " + count);
|
||||
} else {
|
||||
position = null;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!iterator.hasNext()) {
|
||||
position = null;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
hgMetricX.end();
|
||||
log.info("*************************************************");
|
||||
log.info("************* Scanning Completed **************");
|
||||
log.info("Graph: {}", graphName);
|
||||
log.info("Table: {}", tableName);
|
||||
log.info("Keys: {}", count);
|
||||
log.info("Total: {} seconds.", hgMetricX.past() / 1000);
|
||||
log.info("*************************************************");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,219 @@
|
|||
package com.baidu.hugegraph.store.cli.util;
|
||||
|
||||
import com.baidu.hugegraph.store.HgKvEntry;
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.HgStoreSession;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConst;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2022/2/14
|
||||
*/
|
||||
public class HgCliUtil {
|
||||
|
||||
public final static String TABLE_NAME = "cli-table";
|
||||
|
||||
|
||||
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String keyPrefix) {
|
||||
return batchPut(session, keyPrefix, 100);
|
||||
}
|
||||
|
||||
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String keyPrefix, int loop) {
|
||||
return batchPut(session, TABLE_NAME, keyPrefix, loop);
|
||||
}
|
||||
|
||||
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
|
||||
, String keyPrefix, int loop) {
|
||||
return batchPut(session, tableName, keyPrefix, loop,1, key -> toOwnerKey(key));
|
||||
}
|
||||
|
||||
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
|
||||
, String keyPrefix, int loop,int start) {
|
||||
return batchPut(session, tableName, keyPrefix, loop,start, key -> toOwnerKey(key));
|
||||
}
|
||||
|
||||
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
|
||||
, String keyPrefix, int loop, Function<String, HgOwnerKey> f){
|
||||
return batchPut(session,tableName,keyPrefix,loop,1,f);
|
||||
}
|
||||
|
||||
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
|
||||
, String keyPrefix, int loop, int start,Function<String, HgOwnerKey> f) {
|
||||
|
||||
Map<HgOwnerKey, byte[]> res = new LinkedHashMap<>();
|
||||
|
||||
int length = String.valueOf(loop).length();
|
||||
|
||||
session.beginTx();
|
||||
for (int i = start; i <= loop; i++) {
|
||||
|
||||
HgOwnerKey key = f.apply(keyPrefix + "-" + padLeftZeros(String.valueOf(i), length));
|
||||
|
||||
byte[] value = toBytes(keyPrefix + "-V-" + i);
|
||||
res.put(key, value);
|
||||
session.put(tableName, key, value);
|
||||
|
||||
if ((i + 1) % 10000 == 0) {
|
||||
println("commit: " + (i+1));
|
||||
session.commit();
|
||||
session.beginTx();
|
||||
}
|
||||
}
|
||||
if (session.isTx()) {
|
||||
session.commit();
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
public static void printNum(List<HgKvEntry> list, String title) {
|
||||
if (list == null) return;
|
||||
|
||||
println(title + " size: " + list.size());
|
||||
}
|
||||
|
||||
public static void println(Iterator<HgKvEntry> iterator) {
|
||||
if (iterator == null) return;
|
||||
while (iterator.hasNext()) {
|
||||
println(iterator.next());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static void printOwner(List<HgOwnerKey> list) {
|
||||
if (list == null) return;
|
||||
|
||||
for (HgOwnerKey entry : list) {
|
||||
println(entry);
|
||||
}
|
||||
}
|
||||
|
||||
public static void println(List<HgKvEntry> list) {
|
||||
if (list == null) return;
|
||||
|
||||
for (HgKvEntry entry : list) {
|
||||
println(entry);
|
||||
}
|
||||
}
|
||||
|
||||
public static void println(List<HgKvEntry> list, int mod) {
|
||||
if (list == null) return;
|
||||
|
||||
for (int i = 0; i < list.size(); i++) {
|
||||
if (i % mod == 0) {
|
||||
println(list.get(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void println(HgKvEntry kv) {
|
||||
if (kv == null) {
|
||||
System.out.println("null");
|
||||
return;
|
||||
}
|
||||
println("[ " + toStr(kv.key()) + " : " + toStr(kv.value()) + " ]");
|
||||
}
|
||||
|
||||
public static void println(HgOwnerKey key) {
|
||||
if (key == null) {
|
||||
System.out.println("null");
|
||||
return;
|
||||
}
|
||||
println("[ " + toInt(key.getOwner()) + " : " + toStr(key.getKey()) + " ]");
|
||||
}
|
||||
|
||||
public static void println(String str) {
|
||||
System.out.println(str);
|
||||
}
|
||||
|
||||
public static HgOwnerKey toOwnerKey(String key) {
|
||||
return new HgOwnerKey(getOwner(key), toBytes(key));
|
||||
}
|
||||
|
||||
public static HgOwnerKey toOwnerKey(byte[] key) {
|
||||
return new HgOwnerKey(getOwner(key), key);
|
||||
}
|
||||
|
||||
private static byte[] getOwner(String key) {
|
||||
return getOwner(toBytes(key));
|
||||
}
|
||||
|
||||
private static byte[] getOwner(byte[] key) {
|
||||
return toBytes(Arrays.hashCode(key));
|
||||
}
|
||||
|
||||
public static HgOwnerKey toAllPartitionKey(String key) {
|
||||
return HgOwnerKey.of(HgStoreClientConst.ALL_PARTITION_OWNER, toBytes(key));
|
||||
}
|
||||
|
||||
public static HgOwnerKey toOwnerKey(String owner, String key) {
|
||||
return HgOwnerKey.of(toBytes(owner), toBytes(key));
|
||||
}
|
||||
|
||||
public static String toStr(byte[] b) {
|
||||
if (b == null) return "";
|
||||
if (b.length == 0) return "";
|
||||
return new String(b, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public static byte[] toBytes(String str) {
|
||||
if (str == null) return null;
|
||||
return str.getBytes(StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public static byte[] toBytes(long l) {
|
||||
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
|
||||
buffer.putLong(l);
|
||||
return buffer.array();
|
||||
}
|
||||
|
||||
private static byte[] toBytes(final int i) {
|
||||
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
|
||||
buffer.putInt(i);
|
||||
return buffer.array();
|
||||
}
|
||||
|
||||
public static long toLong(byte[] bytes) {
|
||||
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
|
||||
buffer.put(bytes);
|
||||
buffer.flip();//need flip
|
||||
return buffer.getLong();
|
||||
}
|
||||
|
||||
public static long toInt(byte[] bytes) {
|
||||
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
|
||||
buffer.put(bytes);
|
||||
buffer.flip();//need flip
|
||||
return buffer.getInt();
|
||||
}
|
||||
|
||||
public static String padLeftZeros(String str, int n) {
|
||||
return String.format("%1$" + n + "s", str).replace(' ', '0');
|
||||
}
|
||||
|
||||
public static String toSuffix(int num, int length) {
|
||||
return "-" + padLeftZeros(String.valueOf(num), length);
|
||||
}
|
||||
|
||||
public static int amountOf(List list) {
|
||||
if (list == null) {
|
||||
return 0;
|
||||
}
|
||||
return list.size();
|
||||
}
|
||||
|
||||
public static int amountOf(Iterator iterator) {
|
||||
if (iterator == null) return 0;
|
||||
int count = 0;
|
||||
while (iterator.hasNext()) {
|
||||
iterator.next();
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
package com.baidu.hugegraph.store.cli.util;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2022/1/29
|
||||
*/
|
||||
public class HgMetricX {
|
||||
private long start;
|
||||
private long end;
|
||||
|
||||
private long waitStart = System.currentTimeMillis();
|
||||
private long waitTotal;
|
||||
|
||||
public static HgMetricX ofStart() {
|
||||
return new HgMetricX(System.currentTimeMillis());
|
||||
}
|
||||
|
||||
private HgMetricX(long start) {
|
||||
this.start = start;
|
||||
}
|
||||
|
||||
;
|
||||
|
||||
public long start() {
|
||||
return this.start = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
public long end() {
|
||||
return this.end = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
public long past() {
|
||||
return this.end - this.start;
|
||||
}
|
||||
|
||||
public long getWaitTotal() {
|
||||
return this.waitTotal;
|
||||
}
|
||||
|
||||
public void startWait() {
|
||||
this.waitStart = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
public void appendWait() {
|
||||
this.waitTotal += System.currentTimeMillis() - waitStart;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
net:
|
||||
kv:
|
||||
scanner:
|
||||
buffer.size: 10000
|
||||
page.size: 20000
|
||||
chan.size: 1
|
||||
|
||||
pd:
|
||||
address: 10.14.139.71:8686
|
||||
|
||||
scanner:
|
||||
#graph: 'default/icbc_hstoregraph/g'
|
||||
graph: 'DEFAULT/hugegraph/g'
|
||||
|
||||
table: 'g+ie'
|
||||
#table: 'g+v'
|
||||
max: 10_000_000
|
||||
mod: 1_000_000
|
||||
|
||||
committer:
|
||||
graph: 'STORE_CLI'
|
||||
table: 'benchmark'
|
||||
amount: 1_000_000
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#grpc.timeout.seconds=10
|
||||
#grpc.max.inbound.message.size=
|
||||
#grpc.max.outbound.message.size=
|
||||
net.kv.scanner.buffer.size=10000
|
||||
net.kv.scanner.page.size = 20000
|
||||
#Unit:second
|
||||
net.kv.scanner.have.next.timeout=1000
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<!-- Config will be auto loaded every 60s -->
|
||||
<configuration status="error" monitorInterval="60">
|
||||
<properties>
|
||||
<property name="LOG_PATH">logs</property>
|
||||
<property name="FILE_NAME">hg-store-client</property>
|
||||
</properties>
|
||||
|
||||
<appenders>
|
||||
<Console name="console" target="SYSTEM_OUT">
|
||||
<ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
|
||||
<PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
|
||||
<!--JsonLayout compact="false" locationInfo="true">
|
||||
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
|
||||
</JsonLayout-->
|
||||
</Console>
|
||||
|
||||
<!-- Normal server log config -->
|
||||
<RollingRandomAccessFile name="file" fileName="${LOG_PATH}/${FILE_NAME}.log"
|
||||
filePattern="${LOG_PATH}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log"
|
||||
bufferedIO="false" immediateFlush="true">
|
||||
<ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
|
||||
<PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
|
||||
<!--JsonLayout compact="true" eventEol="true" complete="true" locationInfo="true">
|
||||
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
|
||||
</JsonLayout-->
|
||||
<!-- Trigger after exceeding 1day or 50MB -->
|
||||
<Policies>
|
||||
<SizeBasedTriggeringPolicy size="50MB"/>
|
||||
<TimeBasedTriggeringPolicy interval="1" modulate="true" />
|
||||
</Policies>
|
||||
<!-- Keep 5 files per day & auto Delete after over 2GB or 100 files -->
|
||||
<DefaultRolloverStrategy max="5">
|
||||
<Delete basePath="${LOG_PATH}" maxDepth="2">
|
||||
<IfFileName glob="*/*.log"/>
|
||||
<!-- Limit log amount & size -->
|
||||
<IfAny>
|
||||
<IfAccumulatedFileSize exceeds="2GB" />
|
||||
<IfAccumulatedFileCount exceeds="100" />
|
||||
</IfAny>
|
||||
</Delete>
|
||||
</DefaultRolloverStrategy>
|
||||
</RollingRandomAccessFile>
|
||||
|
||||
</appenders>
|
||||
|
||||
<loggers>
|
||||
<logger name="io.grpc.netty" level="INFO" additivity="false">
|
||||
<appender-ref ref="console"/>
|
||||
</logger>
|
||||
<logger name="org.springframework" level="INFO" additivity="false">
|
||||
<appender-ref ref="console"/>
|
||||
</logger>
|
||||
<root level="INFO">
|
||||
<appender-ref ref="console"/>
|
||||
</root>
|
||||
|
||||
</loggers>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
readonly REPO_URL=http://10.14.139.8:8081/artifactory/star-snapshot
|
||||
|
||||
mvn --settings ../settings.xml -Dmaven.test.skip=true -DaltDeploymentRepository=star-snapshot::default::${REPO_URL} clean deploy
|
||||
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>com.baidu.hugegraph</groupId>
|
||||
<artifactId>hg-store-client</artifactId>
|
||||
<version>3.6.3-SNAPSHOT</version>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<maven.test.skip>true</maven.test.skip>
|
||||
<log4j2.version>2.15.0</log4j2.version>
|
||||
<lombok.version>1.18.20</lombok.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.baidu.hugegraph</groupId>
|
||||
<artifactId>hg-store-grpc</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.baidu.hugegraph</groupId>
|
||||
<artifactId>hg-store-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<version>${lombok.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-slf4j-impl</artifactId>
|
||||
<version>${log4j2.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.baidu.hugegraph</groupId>
|
||||
<artifactId>hg-pd-client</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>com.baidu.hugegraph</groupId>
|
||||
<artifactId>hg-pd-grpc</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<!-- test -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.13.2</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java-util</artifactId>
|
||||
<version>3.17.2</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<version>2.7</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<version>2.13.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
<version>1.15</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<distributionManagement>
|
||||
<repository>
|
||||
<id>Baidu_Local</id>
|
||||
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local</url>
|
||||
</repository>
|
||||
<snapshotRepository>
|
||||
<id>Baidu_Local_Snapshots</id>
|
||||
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots</url>
|
||||
</snapshotRepository>
|
||||
</distributionManagement>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com
|
||||
*/
|
||||
public interface HgKvEntry {
|
||||
byte[] key();
|
||||
|
||||
byte[] value();
|
||||
|
||||
default int code(){
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.util.Iterator;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/21
|
||||
*/
|
||||
public interface HgKvIterator<E> extends Iterator<E>, HgSeekAble, Closeable {
|
||||
|
||||
byte[] key();
|
||||
|
||||
byte[] value();
|
||||
|
||||
void close();
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2022/03/10
|
||||
*/
|
||||
public interface HgKvOrderedIterator<E> extends HgKvIterator<E>, Comparable<HgKvOrderedIterator> {
|
||||
long getSequence();
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/24
|
||||
*/
|
||||
public interface HgKvPagingIterator<E> extends HgKvIterator<E>, HgPageSize {
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator;
|
||||
import com.baidu.hugegraph.store.grpc.common.Kv;
|
||||
import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com
|
||||
* @version 0.2.0
|
||||
*/
|
||||
public interface HgKvStore {
|
||||
/**
|
||||
* CAUTION: THE CONST BELOW MUST KEEP CONSISTENCE TO ScanIterator.Trait.
|
||||
*/
|
||||
int SCAN_ANY = 0x80;
|
||||
int SCAN_PREFIX_BEGIN = 0x01;
|
||||
int SCAN_PREFIX_END = 0x02;
|
||||
int SCAN_GT_BEGIN = 0x04;
|
||||
int SCAN_GTE_BEGIN = 0x0c;
|
||||
int SCAN_LT_END = 0x10;
|
||||
int SCAN_LTE_END = 0x30;
|
||||
int SCAN_KEYONLY = 0x40;
|
||||
int SCAN_HASHCODE = 0x100;
|
||||
|
||||
boolean put(String table, HgOwnerKey ownerKey, byte[] value);
|
||||
|
||||
/**
|
||||
* 该版本被store内部使用。向分区写入数据，
|
||||
* partitionId与key.keyCode必须与pd存储的分区信息保持一致。
|
||||
*/
|
||||
boolean directPut(String table, int partitionId, HgOwnerKey key, byte[] value);
|
||||
byte[] get(String table, HgOwnerKey ownerKey);
|
||||
|
||||
boolean clean(int partId);
|
||||
|
||||
boolean delete(String table, HgOwnerKey ownerKey);
|
||||
|
||||
boolean deleteSingle(String table, HgOwnerKey ownerKey);
|
||||
|
||||
boolean deletePrefix(String table, HgOwnerKey prefix);
|
||||
|
||||
boolean deleteRange(String table, HgOwnerKey start, HgOwnerKey end);
|
||||
|
||||
boolean merge(String table, HgOwnerKey key, byte[] value);
|
||||
@Deprecated
|
||||
List<HgKvEntry> batchGetOwner(String table, List<HgOwnerKey> keyList);
|
||||
|
||||
HgKvIterator<HgKvEntry> scanIterator(String table);
|
||||
|
||||
HgKvIterator<HgKvEntry> scanIterator(String table, byte[] query);
|
||||
|
||||
HgKvIterator<HgKvEntry> scanIterator(String table, long limit);
|
||||
|
||||
HgKvIterator<HgKvEntry> scanIterator(String table, long limit, byte[] query);
|
||||
|
||||
HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey keyPrefix);
|
||||
|
||||
HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey keyPrefix, long limit);
|
||||
|
||||
HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey keyPrefix, long limit, byte[] query);
|
||||
|
||||
HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey);
|
||||
|
||||
HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit);
|
||||
|
||||
HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit, byte[] query);
|
||||
|
||||
HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit, int scanType, byte[] query);
|
||||
|
||||
HgKvIterator<HgKvEntry> scanIterator(String table, int codeFrom, int codeTo, int scanType, byte[] query);
|
||||
|
||||
// HgKvIterator<HgKvEntry> scanIterator(ScanStreamReq scanReq);
|
||||
|
||||
HgKvIterator<HgKvEntry> scanIterator(ScanStreamReq.Builder scanReqBuilder);
|
||||
|
||||
boolean truncate();
|
||||
|
||||
default boolean existsTable(String table) {
|
||||
return false;
|
||||
}
|
||||
|
||||
boolean createTable(String table);
|
||||
|
||||
boolean deleteTable(String table);
|
||||
|
||||
boolean dropTable(String table);
|
||||
|
||||
boolean deleteGraph(String graph);
|
||||
List<HgKvIterator<HgKvEntry>> scanBatch(HgScanQuery scanQuery);
|
||||
|
||||
KvCloseableIterator<HgKvIterator<HgKvEntry>> scanBatch2(HgScanQuery scanQuery);
|
||||
KvCloseableIterator<HgKvIterator<HgKvEntry>> scanBatch3(HgScanQuery scanQuery, KvCloseableIterator iterator);
|
||||
|
||||
HgKvIterator<HgKvEntry> batchPrefix(String table, List<HgOwnerKey> prefixList);
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,127 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientUtil;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Arrays;
|
||||
|
||||
import static com.baidu.hugegraph.store.client.util.HgAssert.isArgumentNotNull;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_OWNER_KEY;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/14
|
||||
* @version 1.1.0 add argument assertion.
|
||||
* @version 1.2.0 add newEmpty
|
||||
* @version 1.3.0 add canceled assert
|
||||
*/
|
||||
public class HgOwnerKey implements Serializable {
|
||||
private int keyCode = 0;// TODO: Be here OK?
|
||||
private final byte[] owner;
|
||||
private byte[] key;
|
||||
private int serialNo; //顺序号，用于批量查询保证返回结果的顺序性
|
||||
|
||||
public static HgOwnerKey emptyOf(){
|
||||
return EMPTY_OWNER_KEY;
|
||||
}
|
||||
|
||||
public static HgOwnerKey newEmpty(){
|
||||
return HgOwnerKey.of(EMPTY_BYTES,EMPTY_BYTES);
|
||||
}
|
||||
|
||||
public static HgOwnerKey ownerOf(byte[] owner){
|
||||
return new HgOwnerKey(owner,EMPTY_BYTES);
|
||||
}
|
||||
public static HgOwnerKey codeOf(int code){
|
||||
return HgOwnerKey.of(EMPTY_BYTES, EMPTY_BYTES).setKeyCode(code);
|
||||
}
|
||||
|
||||
public static HgOwnerKey of(byte[] owner, byte[] key){
|
||||
return new HgOwnerKey(owner,key);
|
||||
}
|
||||
public static HgOwnerKey of(int keyCode, byte[] key){
|
||||
return new HgOwnerKey(keyCode, key);
|
||||
}
|
||||
|
||||
/**
|
||||
* @see HgOwnerKey:of(byte[] owner, byte[] key)
|
||||
* @param owner
|
||||
* @param key
|
||||
*/
|
||||
@Deprecated
|
||||
public HgOwnerKey(byte[] owner, byte[] key) {
|
||||
if (owner == null) {
|
||||
owner = EMPTY_BYTES;
|
||||
}
|
||||
if (key == null) {
|
||||
key = EMPTY_BYTES;
|
||||
}
|
||||
this.owner = owner;
|
||||
this.key = key;
|
||||
}
|
||||
|
||||
public HgOwnerKey(int code, byte[] key) {
|
||||
if (key == null) {
|
||||
key = EMPTY_BYTES;
|
||||
}
|
||||
this.owner = EMPTY_BYTES;
|
||||
this.key = key;
|
||||
this.keyCode = code;
|
||||
}
|
||||
|
||||
public byte[] getOwner() {
|
||||
return owner;
|
||||
}
|
||||
|
||||
public byte[] getKey() {
|
||||
return key;
|
||||
}
|
||||
|
||||
public int getKeyCode() {
|
||||
return keyCode;
|
||||
}
|
||||
|
||||
public HgOwnerKey setKeyCode(int keyCode) {
|
||||
this.keyCode = keyCode;
|
||||
return this;
|
||||
}
|
||||
|
||||
public HgOwnerKey codeToKey(int keyCode) {
|
||||
this.keyCode = keyCode;
|
||||
this.key = HgStoreClientUtil.toIntBytes(keyCode);
|
||||
return this;
|
||||
}
|
||||
|
||||
public HgOwnerKey setSerialNo(int serialNo){
|
||||
this.serialNo = serialNo;
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getSerialNo(){
|
||||
return this.serialNo;
|
||||
}
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
HgOwnerKey that = (HgOwnerKey) o;
|
||||
return Arrays.equals(owner, that.owner) && Arrays.equals(key, that.key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = Arrays.hashCode(owner);
|
||||
result = 31 * result + Arrays.hashCode(key);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HgOwnerKey{" +
|
||||
"owner=" + Arrays.toString(owner) +
|
||||
", key=" + Arrays.toString(key) +
|
||||
", code=" + keyCode +
|
||||
'}';
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
/**
|
||||
* Return the amount of records returned by one query in pageable-query.
|
||||
*
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/24
|
||||
*/
|
||||
public interface HgPageSize {
|
||||
long getPageSize();
|
||||
default boolean isPageEmpty(){
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com
|
||||
*/
|
||||
public final class HgPrivate {
|
||||
|
||||
private static HgPrivate instance = new HgPrivate();
|
||||
|
||||
private HgPrivate() {
|
||||
}
|
||||
|
||||
static HgPrivate of() {
|
||||
return instance;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,305 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
import com.baidu.hugegraph.store.client.util.HgAssert;
|
||||
import com.baidu.hugegraph.store.grpc.common.ScanOrderType;
|
||||
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2022/3/4
|
||||
* @version 0.5.0
|
||||
*/
|
||||
public interface HgScanQuery {
|
||||
String getTable();
|
||||
|
||||
HgScanQuery.ScanMethod getScanMethod();
|
||||
|
||||
List<HgOwnerKey> getPrefixList();
|
||||
|
||||
Iterator<HgOwnerKey> getPrefixItr();
|
||||
|
||||
List<HgOwnerKey> getStartList();
|
||||
|
||||
List<HgOwnerKey> getEndList();
|
||||
|
||||
long getLimit();
|
||||
|
||||
long getPerKeyLimit();
|
||||
|
||||
long getPerKeyMax();
|
||||
|
||||
long getSkipDegree();
|
||||
|
||||
int getScanType();
|
||||
ScanOrderType getOrderType();
|
||||
|
||||
boolean isOnlyKey();
|
||||
|
||||
byte[] getQuery();
|
||||
|
||||
enum ScanMethod {
|
||||
ALL, PREFIX, RANGE
|
||||
}
|
||||
|
||||
enum SortType {
|
||||
UNSORTED, SORT_BY_EDGE, SORT_BY_VERTEX
|
||||
}
|
||||
|
||||
|
||||
ScanBuilder builder();
|
||||
|
||||
static HgScanQuery tableOf(String table) {
|
||||
return ScanBuilder.tableOf(table).build();
|
||||
}
|
||||
|
||||
static HgScanQuery rangeOf(String table, List<HgOwnerKey> startList, List<HgOwnerKey> endList) {
|
||||
return ScanBuilder.rangeOf(table, startList, endList).build();
|
||||
}
|
||||
|
||||
static HgScanQuery prefixOf(String table, List<HgOwnerKey> prefixList) {
|
||||
return ScanBuilder.prefixOf(table, prefixList).build();
|
||||
}
|
||||
|
||||
static HgScanQuery prefixOf(String table, List<HgOwnerKey> prefixList, ScanOrderType orderType) {
|
||||
return ScanBuilder.prefixOf(table, prefixList).setOrderType(orderType).build();
|
||||
}
|
||||
|
||||
static HgScanQuery prefixIteratorOf(String table, Iterator<HgOwnerKey> prefixItr) {
|
||||
return ScanBuilder.prefixIteratorOf(table, prefixItr).build();
|
||||
}
|
||||
|
||||
static HgScanQuery prefixIteratorOf(String table, Iterator<HgOwnerKey> prefixItr, ScanOrderType orderType) {
|
||||
return ScanBuilder.prefixIteratorOf(table, prefixItr).setOrderType(orderType).build();
|
||||
}
|
||||
|
||||
class ScanBuilder {
|
||||
private final String table;
|
||||
private final HgScanQuery.ScanMethod sanMethod;
|
||||
private long limit = Integer.MAX_VALUE;
|
||||
private long perKeyLimit = Integer.MAX_VALUE;
|
||||
private long perKeyMax = Integer.MAX_VALUE;
|
||||
private int scanType;
|
||||
private ScanOrderType orderType;
|
||||
|
||||
private long skipDegree;
|
||||
|
||||
private boolean onlyKey;
|
||||
private byte[] query;
|
||||
private List<HgOwnerKey> prefixList;
|
||||
private List<HgOwnerKey> startList;
|
||||
private List<HgOwnerKey> endList;
|
||||
private Iterator<HgOwnerKey> prefixItr;
|
||||
|
||||
public static ScanBuilder rangeOf(String table, List<HgOwnerKey> startList, List<HgOwnerKey> endList) {
|
||||
HgAssert.isArgumentValid(table, "table");
|
||||
HgAssert.isArgumentValid(startList, "startList");
|
||||
HgAssert.isArgumentValid(endList, "endList");
|
||||
HgAssert.isTrue(startList.size() == endList.size()
|
||||
, "The size of startList not equals endList's.");
|
||||
|
||||
ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.RANGE, table);
|
||||
res.startList = startList;
|
||||
res.endList = endList;
|
||||
res.scanType = HgKvStore.SCAN_GTE_BEGIN | HgKvStore.SCAN_LTE_END;
|
||||
return res;
|
||||
}
|
||||
|
||||
public static ScanBuilder prefixOf(String table, List<HgOwnerKey> prefixList) {
|
||||
HgAssert.isArgumentValid(table, "table");
|
||||
HgAssert.isArgumentValid(prefixList, "prefixList");
|
||||
|
||||
ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.PREFIX, table);
|
||||
res.prefixList = prefixList;
|
||||
return res;
|
||||
|
||||
}
|
||||
|
||||
public static ScanBuilder prefixIteratorOf(String table, Iterator<HgOwnerKey> prefixItr) {
|
||||
HgAssert.isArgumentValid(table, "table");
|
||||
|
||||
ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.PREFIX, table);
|
||||
res.prefixItr = prefixItr;
|
||||
return res;
|
||||
|
||||
}
|
||||
|
||||
public static ScanBuilder tableOf(String table) {
|
||||
HgAssert.isArgumentValid(table, "table");
|
||||
|
||||
return new ScanBuilder(HgScanQuery.ScanMethod.ALL, table);
|
||||
}
|
||||
|
||||
ScanBuilder(HgScanQuery.ScanMethod sanMethod, String table) {
|
||||
this.table = table;
|
||||
this.sanMethod = sanMethod;
|
||||
this.orderType = ScanOrderType.ORDER_NONE;
|
||||
}
|
||||
|
||||
public ScanBuilder setLimit(long limit) {
|
||||
this.limit = limit;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanBuilder setPerKeyLimit(long limit) {
|
||||
this.perKeyLimit = limit;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanBuilder setPerKeyMax(long max) {
|
||||
this.perKeyMax = max;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanBuilder setScanType(int scanType) {
|
||||
this.scanType = scanType;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanBuilder setOrderType(ScanOrderType orderType){
|
||||
this.orderType = orderType;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanBuilder setQuery(byte[] query) {
|
||||
this.query = query;
|
||||
return this;
|
||||
}
|
||||
public ScanBuilder setSkipDegree(long skipDegree){
|
||||
this.skipDegree = skipDegree;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanBuilder setOnlyKey(boolean onlyKey){
|
||||
this.onlyKey = onlyKey;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
public HgScanQuery build() {
|
||||
return this.new BatchScanQuery();
|
||||
}
|
||||
|
||||
private class BatchScanQuery implements HgScanQuery {
|
||||
|
||||
@Override
|
||||
public String getTable() {
|
||||
return table;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgScanQuery.ScanMethod getScanMethod() {
|
||||
return sanMethod;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HgOwnerKey> getPrefixList() {
|
||||
if (prefixList == null) {
|
||||
return Collections.EMPTY_LIST;
|
||||
} else {
|
||||
return Collections.unmodifiableList(prefixList);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<HgOwnerKey> getPrefixItr() {
|
||||
return prefixItr;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HgOwnerKey> getStartList() {
|
||||
if (startList == null) {
|
||||
return Collections.EMPTY_LIST;
|
||||
} else {
|
||||
return Collections.unmodifiableList(startList);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HgOwnerKey> getEndList() {
|
||||
if (endList == null) {
|
||||
return Collections.EMPTY_LIST;
|
||||
} else {
|
||||
return Collections.unmodifiableList(endList);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getLimit() {
|
||||
return limit;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getPerKeyLimit() {
|
||||
return perKeyLimit;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getPerKeyMax() {
|
||||
return perKeyMax;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSkipDegree() {
|
||||
return skipDegree;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getScanType() {
|
||||
return scanType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScanOrderType getOrderType() {
|
||||
return orderType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isOnlyKey() {
|
||||
return onlyKey;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getQuery() {
|
||||
return query;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScanBuilder builder() {
|
||||
return ScanBuilder.this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final StringBuffer sb = new StringBuffer("HgScanQuery{");
|
||||
sb.append("table='").append(getTable()).append('\'');
|
||||
sb.append(", scanMethod=").append(getScanMethod());
|
||||
sb.append(", prefixList=").append(getPrefixList());
|
||||
sb.append(", startList=").append(getStartList());
|
||||
sb.append(", endList=").append(getEndList());
|
||||
sb.append(", limit=").append(getLimit());
|
||||
sb.append(", perKeyLimit=").append(getPerKeyLimit());
|
||||
sb.append(", perKeyMax=").append(getPerKeyMax());
|
||||
sb.append(", skipDegree=").append(getSkipDegree());
|
||||
sb.append(", scanType=").append(getScanType());
|
||||
sb.append(", orderType=").append(getOrderType());
|
||||
sb.append(", onlyKey=").append(isOnlyKey());
|
||||
sb.append(", query=");
|
||||
if (query == null) sb.append("null");
|
||||
else {
|
||||
sb.append('[');
|
||||
for (int i = 0; i < query.length; ++i)
|
||||
sb.append(i == 0 ? "" : ", ").append(query[i]);
|
||||
sb.append(']');
|
||||
}
|
||||
sb.append('}');
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2022/03/11
|
||||
*/
|
||||
public interface HgSeekAble {
|
||||
byte[] position();
|
||||
|
||||
void seek(byte[] position);
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
import com.baidu.hugegraph.store.client.HgStoreSessionProvider;
|
||||
|
||||
import javax.annotation.concurrent.ThreadSafe;
|
||||
|
||||
/**
|
||||
* Maintain HgStoreSession instances.
|
||||
* // TODO: Holding more than one HgSessionManager is available,if you want to connect multi HgStore-clusters.
|
||||
*
|
||||
* @author lynn.bond@hotmail.com
|
||||
*/
|
||||
|
||||
@ThreadSafe
|
||||
public final class HgSessionManager {
|
||||
private HgSessionProvider sessionProvider;
|
||||
private final static HgSessionManager instance =new HgSessionManager();
|
||||
|
||||
private HgSessionManager(){
|
||||
// TODO: constructed by SPI
|
||||
this.sessionProvider=new HgStoreSessionProvider();
|
||||
}
|
||||
|
||||
|
||||
public static HgSessionManager getInstance(){
|
||||
return instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve or create a HgStoreSession.
|
||||
* @param graphName
|
||||
* @return
|
||||
*/
|
||||
public HgStoreSession openSession(String graphName){
|
||||
return this.sessionProvider.createSession(graphName);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
import javax.annotation.concurrent.ThreadSafe;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/12
|
||||
*/
|
||||
@ThreadSafe
|
||||
public interface HgSessionProvider {
|
||||
HgStoreSession createSession(String graphName);
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
import com.baidu.hugegraph.pd.client.PDClient;
|
||||
import com.baidu.hugegraph.pd.client.PDConfig;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeManager;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodePartitionerImpl;
|
||||
import com.baidu.hugegraph.store.client.HgStoreSessionProvider;
|
||||
|
||||
import javax.annotation.concurrent.ThreadSafe;
|
||||
|
||||
/**
|
||||
* Maintain HgStoreSession instances.
|
||||
* // TODO: Holding more than one HgSessionManager is available,if you want to connect multi HgStore-clusters.
|
||||
*
|
||||
* @author lynn.bond@hotmail.com
|
||||
*/
|
||||
|
||||
@ThreadSafe
|
||||
public final class HgStoreClient {
|
||||
private HgSessionProvider sessionProvider;
|
||||
private PDClient pdClient;
|
||||
|
||||
public HgStoreClient() {
|
||||
this.sessionProvider = new HgStoreSessionProvider();
|
||||
}
|
||||
|
||||
public HgStoreClient(PDConfig config) {
|
||||
this.sessionProvider = new HgStoreSessionProvider();
|
||||
pdClient = PDClient.create(config);
|
||||
setPdClient(pdClient);
|
||||
}
|
||||
public HgStoreClient(PDClient pdClient) {
|
||||
this.sessionProvider = new HgStoreSessionProvider();
|
||||
setPdClient(pdClient);
|
||||
}
|
||||
public static HgStoreClient create(PDConfig config) {
|
||||
return new HgStoreClient(config);
|
||||
}
|
||||
|
||||
public static HgStoreClient create(PDClient pdClient) {
|
||||
return new HgStoreClient(pdClient);
|
||||
}
|
||||
|
||||
public static HgStoreClient create(){
|
||||
return new HgStoreClient();
|
||||
}
|
||||
|
||||
public void setPDConfig(PDConfig config) {
|
||||
pdClient = PDClient.create(config);
|
||||
setPdClient(pdClient);
|
||||
}
|
||||
|
||||
public void setPdClient(PDClient client) {
|
||||
this.pdClient = client;
|
||||
HgStoreNodeManager nodeManager =
|
||||
HgStoreNodeManager.getInstance();
|
||||
|
||||
HgStoreNodePartitionerImpl p = new HgStoreNodePartitionerImpl(pdClient, nodeManager);
|
||||
nodeManager.setNodeProvider(p);
|
||||
nodeManager.setNodePartitioner(p);
|
||||
nodeManager.setNodeNotifier(p);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve or create a HgStoreSession.
|
||||
*
|
||||
* @param graphName
|
||||
* @return
|
||||
*/
|
||||
public HgStoreSession openSession(String graphName) {
|
||||
return this.sessionProvider.createSession(graphName);
|
||||
}
|
||||
|
||||
public PDClient getPdClient(){
|
||||
return pdClient;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
import com.baidu.hugegraph.store.client.type.HgStoreClientException;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com
|
||||
*/
|
||||
public interface HgStoreSession extends HgKvStore {
|
||||
|
||||
void beginTx();
|
||||
|
||||
/**
|
||||
* @throws IllegalStateException when the tx hasn't been beginning.
|
||||
* @throws HgStoreClientException when failed to commit .
|
||||
*/
|
||||
void commit();
|
||||
|
||||
/**
|
||||
* @throws IllegalStateException when the tx hasn't been beginning.
|
||||
* @throws HgStoreClientException when failed to rollback.
|
||||
*/
|
||||
void rollback();
|
||||
|
||||
boolean isTx();
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com
|
||||
*/
|
||||
public interface HgTkvEntry {
|
||||
String table();
|
||||
byte[] key();
|
||||
byte[] value();
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
package com.baidu.hugegraph.store;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com
|
||||
*/
|
||||
public interface HgTokvEntry {
|
||||
String table();
|
||||
HgOwnerKey ownerKey();
|
||||
byte[] value();
|
||||
}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Immutable Object Pattern
|
||||
*
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/26
|
||||
*/
|
||||
public final class HgNodePartition {
|
||||
private Long nodeId;
|
||||
//当前key的hashcode
|
||||
private Integer keyCode;
|
||||
|
||||
//分区的开始结束范围
|
||||
private Integer startKey;
|
||||
private Integer endKey;
|
||||
private int hash = -1;
|
||||
|
||||
public static HgNodePartition of(Long nodeId, Integer keyCode) {
|
||||
return new HgNodePartition(nodeId, keyCode);
|
||||
}
|
||||
public static HgNodePartition of(Long nodeId, Integer keyCode, Integer startKey, Integer endKey) {
|
||||
return new HgNodePartition(nodeId, keyCode, startKey, endKey);
|
||||
}
|
||||
|
||||
HgNodePartition(Long nodeId, Integer keyCode) {
|
||||
this.nodeId = nodeId;
|
||||
this.keyCode = keyCode;
|
||||
this.startKey = this.endKey = keyCode;
|
||||
}
|
||||
|
||||
HgNodePartition(Long nodeId, Integer keyCode, Integer startKey, Integer endKey) {
|
||||
this.nodeId = nodeId;
|
||||
this.keyCode = keyCode;
|
||||
this.startKey = startKey;
|
||||
this.endKey = endKey;
|
||||
}
|
||||
public Long getNodeId() {
|
||||
return nodeId;
|
||||
}
|
||||
|
||||
public Integer getKeyCode() {
|
||||
return keyCode;
|
||||
}
|
||||
|
||||
public Integer getStartKey(){
|
||||
return startKey;
|
||||
}
|
||||
|
||||
public Integer getEndKey(){
|
||||
return endKey;
|
||||
}
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
HgNodePartition that = (HgNodePartition) o;
|
||||
return Objects.equals(nodeId, that.nodeId) && Objects.equals(keyCode, that.keyCode);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
if(this.hash==-1)this.hash=Objects.hash(nodeId, keyCode);
|
||||
return this.hash;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HgNodePartition{" +
|
||||
"nodeId=" + nodeId +
|
||||
", partitionId=" + keyCode +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
import java.net.CookieHandler;
|
||||
import java.util.*;
|
||||
|
||||
import static com.baidu.hugegraph.store.client.util.HgAssert.isFalse;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/26
|
||||
* @version 1.0.0
|
||||
*/
|
||||
@NotThreadSafe
|
||||
public final class HgNodePartitionerBuilder {
|
||||
|
||||
private Set<HgNodePartition> partitions = null;
|
||||
/**
|
||||
*
|
||||
* @param nodeId
|
||||
* @param keyCode
|
||||
* @return
|
||||
* @see HgNodePartitionerBuilder:setPartitions(Set<HgNodePartition> partitions)
|
||||
*/
|
||||
@Deprecated
|
||||
public HgNodePartitionerBuilder add(Long nodeId, Integer keyCode) {
|
||||
isFalse(nodeId == null, "The argument is invalid: nodeId");
|
||||
isFalse(keyCode == null, "The argument is invalid: keyCode");
|
||||
|
||||
if(this.partitions==null){
|
||||
this.partitions=new HashSet<>(16,1);
|
||||
}
|
||||
|
||||
this.partitions.add(HgNodePartition.of(nodeId, keyCode));
|
||||
return this;
|
||||
}
|
||||
|
||||
public void setPartitions(Set<HgNodePartition> partitions) {
|
||||
isFalse(partitions == null, "The argument is invalid: partitions");
|
||||
this.partitions = partitions;
|
||||
}
|
||||
|
||||
Collection<HgNodePartition> getPartitions() {
|
||||
return this.partitions;
|
||||
}
|
||||
|
||||
static HgNodePartitionerBuilder resetAndGet() {
|
||||
return new HgNodePartitionerBuilder();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/26
|
||||
*/
|
||||
public class HgPrivate {
|
||||
private final static HgPrivate instance = new HgPrivate();
|
||||
|
||||
static HgPrivate getInstance() {
|
||||
return instance;
|
||||
}
|
||||
|
||||
private HgPrivate() {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import com.baidu.hugegraph.store.HgStoreSession;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/11
|
||||
* @version 0.2.0
|
||||
*/
|
||||
public interface HgStoreNode {
|
||||
|
||||
/**
|
||||
* Return boolean value of being online or not
|
||||
* @return
|
||||
*/
|
||||
default boolean isHealthy(){
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the unique ID of store-node.
|
||||
* @return
|
||||
*/
|
||||
Long getNodeId();
|
||||
|
||||
/**
|
||||
* A string value concatenated by host and port: "host:port"
|
||||
* @return
|
||||
*/
|
||||
String getAddress();
|
||||
|
||||
/**
|
||||
* Return a new HgStoreSession instance, that is not Thread safe.
|
||||
* Return null when the node is not in charge of the graph that was passed from argument.
|
||||
* @return
|
||||
*/
|
||||
HgStoreSession openSession(String graphName);
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/11
|
||||
*/
|
||||
public interface HgStoreNodeBuilder {
|
||||
|
||||
HgStoreNodeBuilder setNodeId(Long nodeId);
|
||||
|
||||
HgStoreNodeBuilder setAddress(String address);
|
||||
|
||||
/**
|
||||
* To build a HgStoreNode instance.
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
HgStoreNode build();
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/12
|
||||
*/
|
||||
public final class HgStoreNodeCandidates {
|
||||
|
||||
List<HgStoreNode> nodeList;
|
||||
|
||||
HgStoreNodeCandidates(List<HgStoreNode> nodeList){
|
||||
this.nodeList=nodeList;
|
||||
}
|
||||
|
||||
public int size(){
|
||||
return this.nodeList.size();
|
||||
}
|
||||
|
||||
public HgStoreNode getNode(int index){
|
||||
return this.nodeList.get(index);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,237 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import com.baidu.hugegraph.store.client.grpc.GrpcStoreNodeBuilder;
|
||||
import com.baidu.hugegraph.store.client.type.HgNodeStatus;
|
||||
import com.baidu.hugegraph.store.client.type.HgStoreClientException;
|
||||
import com.baidu.hugegraph.store.client.util.HgAssert;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConst;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import javax.annotation.concurrent.ThreadSafe;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* // TODO: Mapping to Store-Node-Cluster, one to one.
|
||||
*
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/11
|
||||
* @version 0.2.0
|
||||
*/
|
||||
@ThreadSafe
|
||||
@Slf4j
|
||||
public final class HgStoreNodeManager {
|
||||
private final static Set<String> CLUSTER_ID_SET = new HashSet<>();
|
||||
private final static HgStoreNodeManager instance = new HgStoreNodeManager();
|
||||
|
||||
private final String clusterId;
|
||||
private final Map<String, HgStoreNode> addressMap = new ConcurrentHashMap<>();
|
||||
private final Map<Long, HgStoreNode> nodeIdMap = new ConcurrentHashMap<>();
|
||||
private final Map<String, List<HgStoreNode>> graphNodesMap = new ConcurrentHashMap<>();
|
||||
|
||||
private HgStoreNodeProvider nodeProvider;
|
||||
private HgStoreNodePartitioner nodePartitioner;
|
||||
private HgStoreNodeNotifier nodeNotifier;
|
||||
|
||||
private HgStoreNodeManager() {
|
||||
this.clusterId = HgStoreClientConst.DEFAULT_NODE_CLUSTER_ID;
|
||||
}
|
||||
|
||||
private HgStoreNodeManager(String clusterId) {
|
||||
synchronized (CLUSTER_ID_SET) {
|
||||
if (CLUSTER_ID_SET.contains(clusterId)) {
|
||||
throw new RuntimeException("The cluster [" + clusterId + "] has been existing.");
|
||||
}
|
||||
CLUSTER_ID_SET.add(clusterId);
|
||||
this.clusterId = clusterId;
|
||||
}
|
||||
}
|
||||
|
||||
public static HgStoreNodeManager getInstance() {
|
||||
return instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the HgStoreNodeBuilder
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public HgStoreNodeBuilder getNodeBuilder() {
|
||||
// TODO: Constructed by a provider that retrieved by SPI
|
||||
return new GrpcStoreNodeBuilder(this, HgPrivate.getInstance());
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an instance of HgStoreNode whose ID is matched to the argument.
|
||||
*
|
||||
* @param nodeId
|
||||
* @return null when none of instance is matched to the argument,or argument is invalid.
|
||||
*/
|
||||
public HgStoreNode getStoreNode(Long nodeId) {
|
||||
if (nodeId == null) {
|
||||
return null;
|
||||
}
|
||||
return this.nodeIdMap.get(nodeId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a HgStoreNode instance with graph-name and node-id.
|
||||
* <b>CAUTION:</b>
|
||||
* <b>It won't work when user haven't set a HgStoreNodeProvider via setNodeProvider method.</b>
|
||||
*
|
||||
* @param graphName
|
||||
* @param nodeId
|
||||
* @return
|
||||
*/
|
||||
HgStoreNode applyNode(String graphName, Long nodeId) {
|
||||
HgStoreNode node = this.nodeIdMap.get(nodeId);
|
||||
|
||||
if (node != null) {
|
||||
return node;
|
||||
}
|
||||
|
||||
if (this.nodeProvider == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
node = this.nodeProvider.apply(graphName, nodeId);
|
||||
|
||||
if (node == null) {
|
||||
|
||||
log.warn("Failed to apply a HgStoreNode instance form the nodeProvider [ "
|
||||
+ this.nodeProvider.getClass().getName() + " ].");
|
||||
notifying(graphName, nodeId, HgNodeStatus.NOT_EXIST);
|
||||
return null;
|
||||
}
|
||||
|
||||
this.addNode(graphName, node);
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
private void notifying(String graphName, Long nodeId, HgNodeStatus status) {
|
||||
if (this.nodeNotifier != null) {
|
||||
try {
|
||||
this.nodeNotifier.notice(graphName, HgStoreNotice.of(nodeId, status));
|
||||
} catch (Throwable t) {
|
||||
log.error("Failed to invoke " + this.nodeNotifier.getClass().getSimpleName() + ":notice(" + nodeId + "," + status + ")", t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param graphName
|
||||
* @param notice
|
||||
* @return null: when there is no HgStoreNodeNotifier in the nodeManager;
|
||||
* @throws HgStoreClientException
|
||||
*/
|
||||
public Integer notifying(String graphName, HgStoreNotice notice) {
|
||||
|
||||
if (this.nodeNotifier != null) {
|
||||
|
||||
synchronized (Thread.currentThread()) {
|
||||
try {
|
||||
return this.nodeNotifier.notice(graphName, notice);
|
||||
} catch (Throwable t) {
|
||||
String msg = "Failed to invoke " + this.nodeNotifier.getClass().getSimpleName() + ", notice: [ " + notice + " ]";
|
||||
log.error(msg, t);
|
||||
throw new HgStoreClientException(msg);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a collection of HgStoreNode who is in charge of the graph passed in the argument.
|
||||
*
|
||||
* @param graphName
|
||||
* @return null when none matched to argument or any argument is invalid.
|
||||
*/
|
||||
public List<HgStoreNode> getStoreNodes(String graphName) {
|
||||
if (HgAssert.isInvalid(graphName)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return this.graphNodesMap.get(graphName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adding a new Store-Node, return the argument's value if the host+port was not existing,
|
||||
* otherwise return the HgStoreNode-instance added early.
|
||||
*
|
||||
* @param storeNode
|
||||
* @return
|
||||
* @throws IllegalArgumentException when any argument is invalid.
|
||||
*/
|
||||
public HgStoreNode addNode(HgStoreNode storeNode) {
|
||||
HgAssert.isFalse(storeNode == null, "the argument: storeNode is null.");
|
||||
|
||||
Long nodeId = storeNode.getNodeId();
|
||||
|
||||
HgStoreNode node = null;
|
||||
|
||||
synchronized (this.nodeIdMap) {
|
||||
node = this.addressMap.get(nodeId);
|
||||
if (node == null) {
|
||||
node = storeNode;
|
||||
this.nodeIdMap.put(nodeId, node);
|
||||
this.addressMap.put(storeNode.getAddress(), node);
|
||||
}
|
||||
}
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param graphName
|
||||
* @param storeNode
|
||||
* @return
|
||||
* @throws IllegalArgumentException when any argument is invalid.
|
||||
*/
|
||||
public HgStoreNode addNode(String graphName, HgStoreNode storeNode) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(graphName), "the argument is invalid: graphName");
|
||||
HgStoreNode node = this.addNode(storeNode);
|
||||
|
||||
List<HgStoreNode> nodes = null;
|
||||
|
||||
synchronized (this.graphNodesMap) {
|
||||
nodes = this.graphNodesMap.get(graphName);
|
||||
if (nodes == null) {
|
||||
nodes = new ArrayList<>();
|
||||
this.graphNodesMap.put(graphName, nodes);
|
||||
}
|
||||
nodes.add(node);
|
||||
}
|
||||
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
public HgStoreNodePartitioner getNodePartitioner() {
|
||||
return nodePartitioner;
|
||||
}
|
||||
|
||||
public HgStoreNodeManager setNodePartitioner(HgStoreNodePartitioner nodePartitioner) {
|
||||
HgAssert.isFalse(nodePartitioner == null, "the argument is invalid: nodePartitioner");
|
||||
this.nodePartitioner = nodePartitioner;
|
||||
return this;
|
||||
}
|
||||
|
||||
public HgStoreNodeNotifier getNodeNotifier() {
|
||||
return nodeNotifier;
|
||||
}
|
||||
|
||||
public HgStoreNodeManager setNodeNotifier(HgStoreNodeNotifier nodeNotifier) {
|
||||
HgAssert.isFalse(nodeNotifier == null, "the argument is invalid: nodeNotifier");
|
||||
this.nodeNotifier = nodeNotifier;
|
||||
return this;
|
||||
}
|
||||
|
||||
public HgStoreNodeManager setNodeProvider(HgStoreNodeProvider nodeProvider) {
|
||||
this.nodeProvider = nodeProvider;
|
||||
return this;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import com.baidu.hugegraph.store.client.type.HgNodeStatus;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/12
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public interface HgStoreNodeNotifier {
|
||||
|
||||
/**
|
||||
* It will be invoked by NodeManager, when some exception or issue was happened.
|
||||
* @param graphName
|
||||
* @param storeNotice
|
||||
* @return return 0 please, for no matter what.
|
||||
*/
|
||||
int notice(String graphName, HgStoreNotice storeNotice);
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConst;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/12
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public interface HgStoreNodePartitioner {
|
||||
|
||||
/**
|
||||
* The partition algorithm implementation, that specialized by user.
|
||||
*
|
||||
* @param builder The builder of HgNodePartitionerBuilder. It's supposed to be invoked directly by user.
|
||||
* <b>e.g. builder.add(nodeId,address,partitionId);</b>
|
||||
* @param graphName
|
||||
* @param startKey
|
||||
* @param endKey
|
||||
* @return status:
|
||||
* <ul>
|
||||
* <li>0: The partitioner is OK.</li>
|
||||
* <li>10: The partitioner is not work.</li>
|
||||
* </ul>
|
||||
*/
|
||||
int partition(HgNodePartitionerBuilder builder, String graphName, byte[] startKey, byte[] endKey);
|
||||
|
||||
/**
|
||||
* @param builder
|
||||
* @param graphName
|
||||
* @param startCode hash code
|
||||
* @param endCode hash code
|
||||
* @return
|
||||
*/
|
||||
default int partition(HgNodePartitionerBuilder builder, String graphName, int startCode, int endCode) {
|
||||
return this.partition(builder, graphName
|
||||
, HgStoreClientConst.ALL_PARTITION_OWNER
|
||||
, HgStoreClientConst.ALL_PARTITION_OWNER);
|
||||
}
|
||||
|
||||
default int partition(HgNodePartitionerBuilder builder, String graphName, int partitionId) {
|
||||
return this.partition(builder, graphName
|
||||
, HgStoreClientConst.ALL_PARTITION_OWNER
|
||||
, HgStoreClientConst.ALL_PARTITION_OWNER);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,177 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import com.baidu.hugegraph.pd.client.PDClient;
|
||||
import com.baidu.hugegraph.pd.common.KVPair;
|
||||
import com.baidu.hugegraph.pd.common.PDException;
|
||||
import com.baidu.hugegraph.pd.common.PartitionUtils;
|
||||
import com.baidu.hugegraph.pd.grpc.Metapb;
|
||||
import com.baidu.hugegraph.store.client.type.HgNodeStatus;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConst;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
public class HgStoreNodePartitionerImpl implements HgStoreNodePartitioner,
|
||||
HgStoreNodeProvider,
|
||||
HgStoreNodeNotifier {
|
||||
private PDClient pdClient;
|
||||
private HgStoreNodeManager nodeManager;
|
||||
|
||||
protected HgStoreNodePartitionerImpl() {
|
||||
}
|
||||
|
||||
public HgStoreNodePartitionerImpl(PDClient pdClient, HgStoreNodeManager nodeManager) {
|
||||
this.pdClient = pdClient;
|
||||
this.nodeManager = nodeManager;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询分区信息，结果通过HgNodePartitionerBuilder返回
|
||||
*/
|
||||
@Override
|
||||
public int partition(HgNodePartitionerBuilder builder, String graphName,
|
||||
byte[] startKey, byte[] endKey) {
|
||||
try {
|
||||
HashSet<HgNodePartition> partitions = null;
|
||||
if (HgStoreClientConst.ALL_PARTITION_OWNER == startKey) {
|
||||
List<Metapb.Store> stores = pdClient.getActiveStores(graphName);
|
||||
partitions = new HashSet<>(stores.size());
|
||||
for (Metapb.Store store : stores) {
|
||||
partitions.add(HgNodePartition.of(store.getId(), -1));
|
||||
}
|
||||
|
||||
} else if (endKey == HgStoreClientConst.EMPTY_BYTES
|
||||
|| startKey == endKey || Arrays.equals(startKey, endKey)) {
|
||||
KVPair<Metapb.Partition, Metapb.Shard> partShard =
|
||||
pdClient.getPartition(graphName, startKey);
|
||||
Metapb.Shard leader = partShard.getValue();
|
||||
partitions = new HashSet<>();
|
||||
partitions.add(HgNodePartition.of(leader.getStoreId(),
|
||||
pdClient.keyToCode(graphName, startKey)));
|
||||
} else {
|
||||
log.warn("StartOwnerkey is not equal to endOwnerkey, which is meaningless!!, It is a error!!");
|
||||
List<Metapb.Store> stores = pdClient.getActiveStores(graphName);
|
||||
partitions = new HashSet<>(stores.size());
|
||||
for (Metapb.Store store : stores) {
|
||||
partitions.add(HgNodePartition.of(store.getId(), -1));
|
||||
}
|
||||
}
|
||||
builder.setPartitions(partitions);
|
||||
} catch (PDException e) {
|
||||
log.error("An error occurred while getting partition information :{}", e.getMessage());
|
||||
throw new RuntimeException(e.getMessage(), e);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int partition(HgNodePartitionerBuilder builder, String graphName,
|
||||
int startKey, int endKey) {
|
||||
try {
|
||||
HashSet<HgNodePartition> partitions = new HashSet<>();
|
||||
Metapb.Partition partition = null;
|
||||
while ((partition == null || partition.getEndKey() < endKey)
|
||||
&& startKey < PartitionUtils.MAX_VALUE) {
|
||||
KVPair<Metapb.Partition, Metapb.Shard> partShard =
|
||||
pdClient.getPartitionByCode(graphName, startKey);
|
||||
if (partShard != null) {
|
||||
partition = partShard.getKey();
|
||||
Metapb.Shard leader = partShard.getValue();
|
||||
partitions.add(HgNodePartition.of(leader.getStoreId(), startKey,
|
||||
(int) partition.getStartKey(), (int) partition.getEndKey()));
|
||||
startKey = (int) partition.getEndKey();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
builder.setPartitions(partitions);
|
||||
} catch (PDException e) {
|
||||
log.error("An error occurred while getting partition information :{}", e.getMessage());
|
||||
throw new RuntimeException(e.getMessage(), e);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int partition(HgNodePartitionerBuilder builder, String graphName,
|
||||
int partitionId) {
|
||||
try {
|
||||
HashSet<HgNodePartition> partitions = new HashSet<>();
|
||||
Metapb.Partition partition = null;
|
||||
|
||||
KVPair<Metapb.Partition, Metapb.Shard> partShard =
|
||||
pdClient.getPartitionById(graphName, partitionId);
|
||||
if (partShard != null) {
|
||||
partition = partShard.getKey();
|
||||
Metapb.Shard leader = partShard.getValue();
|
||||
partitions.add(HgNodePartition.of(leader.getStoreId(), (int) partition.getStartKey()));
|
||||
}
|
||||
builder.setPartitions(partitions);
|
||||
} catch (PDException e) {
|
||||
log.error("An error occurred while getting partition information :{}", e.getMessage());
|
||||
throw new RuntimeException(e.getMessage(), e);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
/**
|
||||
* 查询hgstore信息
|
||||
*
|
||||
* @return hgstore
|
||||
*/
|
||||
@Override
|
||||
public HgStoreNode apply(String graphName, Long nodeId) {
|
||||
try {
|
||||
Metapb.Store store = pdClient.getStore(nodeId);
|
||||
return nodeManager.getNodeBuilder().setNodeId(store.getId())
|
||||
.setAddress(store.getAddress()).build();
|
||||
} catch (PDException e) {
|
||||
throw new RuntimeException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 通知更新缓存
|
||||
*/
|
||||
@Override
|
||||
public int notice(String graphName, HgStoreNotice storeNotice) {
|
||||
log.warn(storeNotice.toString());
|
||||
if (storeNotice.getPartitionLeaders() != null) {
|
||||
storeNotice.getPartitionLeaders().forEach((partId, leader) -> {
|
||||
pdClient.updatePartitionLeader(graphName, partId, leader);
|
||||
log.warn("updatePartitionLeader:{}-{}-{}",
|
||||
graphName, partId, leader);
|
||||
});
|
||||
}
|
||||
if (storeNotice.getPartitionIds() != null) {
|
||||
storeNotice.getPartitionIds().forEach(partId -> {
|
||||
pdClient.invalidPartitionCache(graphName, partId);
|
||||
});
|
||||
}
|
||||
if (!storeNotice.getNodeStatus().equals(
|
||||
HgNodeStatus.PARTITION_COMMON_FAULT)
|
||||
&& !storeNotice.getNodeStatus().equals(
|
||||
HgNodeStatus.NOT_PARTITION_LEADER)) {
|
||||
pdClient.invalidPartitionCache();
|
||||
log.warn("invalidPartitionCache:{} ", storeNotice.getNodeStatus());
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
public Metapb.Graph delGraph(String graphName) {
|
||||
try {
|
||||
return pdClient.delGraph(graphName);
|
||||
} catch (PDException e) {
|
||||
log.error("delGraph {} exception, {}", graphName, e.getMessage());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public void setNodeManager(HgStoreNodeManager nodeManager) {
|
||||
this.nodeManager = nodeManager;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/27
|
||||
*/
|
||||
public interface HgStoreNodeProvider {
|
||||
|
||||
/**
|
||||
* Applying a new HgStoreNode instance
|
||||
* @param graphName
|
||||
* @param nodeId
|
||||
* @return
|
||||
*/
|
||||
HgStoreNode apply(String graphName, Long nodeId);
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import com.baidu.hugegraph.store.HgStoreSession;
|
||||
|
||||
/**
|
||||
* @version 0.1.0
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/11
|
||||
*/
|
||||
public interface HgStoreNodeSession extends HgStoreSession {
|
||||
|
||||
/**
|
||||
* Return the name of graph.
|
||||
* @return
|
||||
*/
|
||||
String getGraphName();
|
||||
|
||||
/**
|
||||
* Return an instance of HgStoreNode, which provided the connection of Store-Node machine.
|
||||
* @return
|
||||
*/
|
||||
HgStoreNode getStoreNode();
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import com.baidu.hugegraph.store.client.type.HgNodeStatus;
|
||||
import com.baidu.hugegraph.store.client.util.HgAssert;
|
||||
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2021/11/16
|
||||
*/
|
||||
@NotThreadSafe
|
||||
public class HgStoreNotice {
|
||||
private Long nodeId;
|
||||
private HgNodeStatus nodeStatus;
|
||||
private String message;
|
||||
private Map<Integer, Long> partitionLeaders;
|
||||
private List<Integer> partitionIds;
|
||||
|
||||
public static HgStoreNotice of(Long nodeId, HgNodeStatus nodeStatus) {
|
||||
HgAssert.isArgumentNotNull(nodeId, "nodeId");
|
||||
HgAssert.isArgumentNotNull(nodeStatus, "nodeStatus");
|
||||
return new HgStoreNotice(nodeId, nodeStatus, "");
|
||||
}
|
||||
|
||||
public static HgStoreNotice of(Long nodeId, HgNodeStatus nodeStatus, String message) {
|
||||
HgAssert.isArgumentNotNull(nodeId, "nodeId");
|
||||
HgAssert.isArgumentNotNull(nodeStatus, "nodeStatus");
|
||||
HgAssert.isArgumentNotNull(message, "message");
|
||||
|
||||
return new HgStoreNotice(nodeId, nodeStatus, message);
|
||||
}
|
||||
|
||||
private HgStoreNotice(Long nodeId, HgNodeStatus nodeStatus, String message) {
|
||||
this.nodeId = nodeId;
|
||||
this.nodeStatus = nodeStatus;
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
public Long getNodeId() {
|
||||
return nodeId;
|
||||
}
|
||||
|
||||
public HgNodeStatus getNodeStatus() {
|
||||
return nodeStatus;
|
||||
}
|
||||
|
||||
public String getMessage() {
|
||||
return message;
|
||||
}
|
||||
|
||||
public Map<Integer, Long> getPartitionLeaders() {
|
||||
return partitionLeaders;
|
||||
}
|
||||
|
||||
public HgStoreNotice setPartitionLeaders(Map<Integer, Long> partitionLeaders) {
|
||||
this.partitionLeaders = partitionLeaders;
|
||||
return this;
|
||||
}
|
||||
|
||||
public List<Integer> getPartitionIds() {
|
||||
return partitionIds;
|
||||
}
|
||||
|
||||
public HgStoreNotice setPartitionIds(List<Integer> partitionIds) {
|
||||
this.partitionIds = partitionIds;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HgStoreNotice{" +
|
||||
"nodeId=" + nodeId +
|
||||
", nodeStatus=" + nodeStatus +
|
||||
", message='" + message + '\'' +
|
||||
", partitionLeaders=" + partitionLeaders +
|
||||
", partitionIds=" + partitionIds +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
|
||||
public class HgStoreService {
|
||||
private static HgStoreService instance=new HgStoreService();
|
||||
private HgStoreService(){}
|
||||
|
||||
static HgStoreService of(){
|
||||
return instance;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import com.baidu.hugegraph.store.HgSessionProvider;
|
||||
import com.baidu.hugegraph.store.HgStoreSession;
|
||||
|
||||
import javax.annotation.concurrent.ThreadSafe;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/12
|
||||
*/
|
||||
@ThreadSafe
|
||||
public class HgStoreSessionProvider implements HgSessionProvider {
|
||||
private MultiNodeSessionFactory sessionFactory=MultiNodeSessionFactory.getInstance();
|
||||
|
||||
@Override
|
||||
public HgStoreSession createSession(String graphName) {
|
||||
return this.sessionFactory.createStoreSession(graphName);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.HgTkvEntry;
|
||||
import com.baidu.hugegraph.store.HgTokvEntry;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/14
|
||||
*/
|
||||
class HgTkvEntryImpl implements HgTkvEntry {
|
||||
private String table;
|
||||
private byte[] key;
|
||||
private byte[] value;
|
||||
|
||||
HgTkvEntryImpl(String table, byte[] key, byte[] value) {
|
||||
this.table = table;
|
||||
this.key = key;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String table() {
|
||||
return this.table;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] key() {
|
||||
return this.key;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] value() {
|
||||
return this.value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
HgTkvEntryImpl that = (HgTkvEntryImpl) o;
|
||||
return Objects.equals(table, that.table) && Arrays.equals(key, that.key) && Arrays.equals(value, that.value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = Objects.hash(table);
|
||||
result = 31 * result + Arrays.hashCode(key);
|
||||
result = 31 * result + Arrays.hashCode(value);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HgTkvEntryImpl{" +
|
||||
"table='" + table + '\'' +
|
||||
", key=" + Arrays.toString(key) +
|
||||
", value=" + Arrays.toString(value) +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.HgTokvEntry;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/14
|
||||
*/
|
||||
class HgTokvEntryImpl implements HgTokvEntry {
|
||||
|
||||
private String table;
|
||||
private HgOwnerKey ownerKey;
|
||||
private byte[] value;
|
||||
|
||||
HgTokvEntryImpl(String table, HgOwnerKey ownerKey, byte[] value) {
|
||||
this.table = table;
|
||||
this.ownerKey = ownerKey;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String table() {
|
||||
return this.table;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgOwnerKey ownerKey() {
|
||||
return this.ownerKey;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] value() {
|
||||
return this.value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
HgTokvEntryImpl that = (HgTokvEntryImpl) o;
|
||||
return Objects.equals(table, that.table) && Objects.equals(ownerKey, that.ownerKey) && Arrays.equals(value, that.value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = Objects.hash(table, ownerKey);
|
||||
result = 31 * result + Arrays.hashCode(value);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HgTokvEntryImpl{" +
|
||||
"table='" + table + '\'' +
|
||||
", okv=" + ownerKey +
|
||||
", value=" + Arrays.toString(value) +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import com.baidu.hugegraph.store.HgStoreSession;
|
||||
|
||||
import javax.annotation.concurrent.ThreadSafe;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/12
|
||||
*/
|
||||
@ThreadSafe
|
||||
public final class MultiNodeSessionFactory {
|
||||
// TODO multi-instance ?
|
||||
private final static MultiNodeSessionFactory instance = new MultiNodeSessionFactory();
|
||||
// TODO multi-instance ?
|
||||
private HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance();
|
||||
// TODO: to be a chain assigned to each graph
|
||||
//private HgStoreNodeDispatcher storeNodeDispatcher;
|
||||
|
||||
private MultiNodeSessionFactory() {
|
||||
|
||||
//this.storeNodeDispatcher = new KeyHashStoreNodeDispatcher();
|
||||
}
|
||||
|
||||
static MultiNodeSessionFactory getInstance() {
|
||||
return instance;
|
||||
}
|
||||
|
||||
HgStoreSession createStoreSession(String graphName) {
|
||||
return buildProxy(graphName);
|
||||
}
|
||||
|
||||
private HgStoreSession buildProxy(String graphName) {
|
||||
//return new MultiNodeSessionProxy(graphName, nodeManager, storeNodeDispatcher);
|
||||
//return new NodePartitionSessionProxy(graphName,nodeManager);
|
||||
//return new NodeRetrySessionProxy(graphName,nodeManager);
|
||||
return new NodeTxSessionProxy(graphName, nodeManager);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.HgStoreSession;
|
||||
|
||||
import javax.annotation.concurrent.ThreadSafe;
|
||||
import java.util.Arrays;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/26
|
||||
*/
|
||||
@ThreadSafe
|
||||
class NodeTkv {
|
||||
private final HgNodePartition nodePartition;
|
||||
private final String table;
|
||||
private final HgOwnerKey key;
|
||||
private final HgOwnerKey endKey;
|
||||
private HgStoreSession session;
|
||||
NodeTkv(HgNodePartition nodePartition, String table, HgOwnerKey key) {
|
||||
this.nodePartition = nodePartition;
|
||||
this.table = table;
|
||||
this.key = key;
|
||||
this.endKey = key;
|
||||
this.key.setKeyCode(this.nodePartition.getKeyCode());
|
||||
}
|
||||
NodeTkv(HgNodePartition nodePartition, String table, HgOwnerKey key, int keyCode) {
|
||||
this.nodePartition = nodePartition;
|
||||
this.table = table;
|
||||
this.key = key;
|
||||
this.endKey = key;
|
||||
|
||||
this.key.setKeyCode(keyCode);
|
||||
}
|
||||
NodeTkv(HgNodePartition nodePartition, String table, HgOwnerKey startKey,
|
||||
HgOwnerKey endKey) {
|
||||
this.nodePartition = nodePartition;
|
||||
this.table = table;
|
||||
this.key = startKey;
|
||||
this.endKey = endKey;
|
||||
this.key.setKeyCode(nodePartition.getStartKey());
|
||||
this.endKey.setKeyCode(nodePartition.getEndKey());
|
||||
}
|
||||
|
||||
public Long getNodeId() {
|
||||
return this.nodePartition.getNodeId();
|
||||
}
|
||||
|
||||
public String getTable() {
|
||||
return table;
|
||||
}
|
||||
|
||||
public HgOwnerKey getKey() {
|
||||
return key;
|
||||
}
|
||||
|
||||
public HgOwnerKey getEndKey() {
|
||||
return endKey;
|
||||
}
|
||||
|
||||
public NodeTkv setKeyCode(int code){
|
||||
this.key.setKeyCode(code);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
NodeTkv nptKv = (NodeTkv) o;
|
||||
return Objects.equals(nodePartition, nptKv.nodePartition) && Objects.equals(table, nptKv.table)
|
||||
&& Objects.equals(key, nptKv.key)
|
||||
&& Objects.equals(endKey, nptKv.endKey);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = Objects.hash(nodePartition, table, key, endKey);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "NptKv{" +
|
||||
"nodePartition=" + nodePartition +
|
||||
", table='" + table + '\'' +
|
||||
", key=" + key +
|
||||
", endKey=" + endKey +
|
||||
'}';
|
||||
}
|
||||
|
||||
public HgStoreSession getSession() {
|
||||
return session;
|
||||
}
|
||||
|
||||
public void setSession(HgStoreSession session) {
|
||||
this.session = session;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,401 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_LIST;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.NODE_MAX_RETRYING_TIMES;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.TX_SESSIONS_MAP_CAPACITY;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collector;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.HgStoreSession;
|
||||
import com.baidu.hugegraph.store.client.type.HgStoreClientException;
|
||||
import com.baidu.hugegraph.store.client.util.HgAssert;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConst;
|
||||
import com.baidu.hugegraph.store.term.HgPair;
|
||||
import com.baidu.hugegraph.store.term.HgTriple;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2021/11/18
|
||||
*/
|
||||
@Slf4j
|
||||
@NotThreadSafe
|
||||
final class NodeTxExecutor {
|
||||
private String graphName;
|
||||
private Map<Long, HgStoreSession> sessions = new HashMap<>(TX_SESSIONS_MAP_CAPACITY, 1);
|
||||
private boolean isTx;
|
||||
NodeTxSessionProxy proxy;
|
||||
private List<HgPair<HgTriple<String, HgOwnerKey, Object>,
|
||||
Function<NodeTkv, Boolean>>> entries = new LinkedList<>();
|
||||
Collector<NodeTkv, ?, Map<Long, List<HgOwnerKey>>> collector = Collectors.groupingBy(
|
||||
nkv -> nkv.getNodeId(), Collectors.mapping(NodeTkv::getKey, Collectors.toList()));
|
||||
private static String maxTryMsg = "the number of retries reached the upper limit : " + NODE_MAX_RETRYING_TIMES +
|
||||
",caused by:";
|
||||
private static String msg = "Not all tx-data delivered to real-node-session successfully.";
|
||||
static {
|
||||
System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism",
|
||||
String.valueOf(Runtime.getRuntime().availableProcessors() * 2));
|
||||
}
|
||||
|
||||
|
||||
|
||||
static NodeTxExecutor graphOf(String graphName, NodeTxSessionProxy proxy) {
|
||||
return new NodeTxExecutor(graphName, proxy);
|
||||
}
|
||||
|
||||
private NodeTxExecutor(String graphName, NodeTxSessionProxy proxy) {
|
||||
this.graphName = graphName;
|
||||
this.proxy = proxy;
|
||||
}
|
||||
|
||||
public boolean isTx() {
|
||||
return isTx;
|
||||
}
|
||||
|
||||
void setTx(boolean tx) {
|
||||
isTx = tx;
|
||||
}
|
||||
|
||||
void commitTx() {
|
||||
if (!this.isTx) {
|
||||
throw new IllegalStateException("It's not in tx state");
|
||||
}
|
||||
|
||||
this.doCommit();
|
||||
}
|
||||
|
||||
void rollbackTx() {
|
||||
if (!this.isTx) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.sessions.values().stream().filter(HgStoreSession::isTx)
|
||||
.forEach(HgStoreSession::rollback);
|
||||
} catch (Throwable t) {
|
||||
throw t;
|
||||
} finally {
|
||||
this.isTx = false;
|
||||
this.sessions.clear();
|
||||
}
|
||||
}
|
||||
|
||||
void doCommit() {
|
||||
try {
|
||||
this.retryingInvoke(() -> {
|
||||
if (this.entries.isEmpty()) {
|
||||
return true;
|
||||
}
|
||||
AtomicBoolean allSuccess = new AtomicBoolean(true);
|
||||
for (HgPair<HgTriple<String, HgOwnerKey, Object>, Function<NodeTkv, Boolean>> e : this.entries) {
|
||||
doAction(e.getKey(), e.getValue());
|
||||
}
|
||||
if (!allSuccess.get()) {
|
||||
throw HgStoreClientException.of(msg);
|
||||
}
|
||||
AtomicReference<Throwable> throwable = new AtomicReference<>();
|
||||
Collection<HgStoreSession> sessions = this.sessions.values();
|
||||
sessions.parallelStream().forEach(e -> {
|
||||
if (e.isTx()) {
|
||||
try {
|
||||
e.commit();
|
||||
} catch (Throwable t) {
|
||||
throwable.compareAndSet(null, t);
|
||||
allSuccess.set(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
if (!allSuccess.get()) {
|
||||
if (isTx) {
|
||||
try {
|
||||
sessions.stream().forEach(HgStoreSession::rollback);
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
}
|
||||
Throwable cause = throwable.get();
|
||||
if (cause.getCause() != null) {
|
||||
cause = cause.getCause();
|
||||
}
|
||||
if (cause instanceof HgStoreClientException) {
|
||||
throw (HgStoreClientException) cause;
|
||||
}
|
||||
throw HgStoreClientException.of(cause);
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
} catch (Throwable t) {
|
||||
throw t;
|
||||
} finally {
|
||||
this.isTx = false;
|
||||
this.entries = new LinkedList<>();
|
||||
this.sessions = new HashMap<>(TX_SESSIONS_MAP_CAPACITY, 1);
|
||||
}
|
||||
}
|
||||
|
||||
// private Function<HgTriple<String, HgOwnerKey, Object>,
|
||||
// List<HgPair<HgStoreNode, NodeTkv>>> nodeStreamWrapper = nodeParams -> {
|
||||
// if (nodeParams.getZ() == null) {
|
||||
// return this.proxy.getNode(nodeParams.getX(),
|
||||
// nodeParams.getY());
|
||||
// } else {
|
||||
// if (nodeParams.getZ() instanceof HgOwnerKey) {
|
||||
// return this.proxy.getNode(nodeParams.getX(),
|
||||
// nodeParams.getY(),
|
||||
// (HgOwnerKey) nodeParams.getZ());
|
||||
// } if ( nodeParams.getZ() instanceof Integer ){
|
||||
// return this.proxy.doPartition(nodeParams.getX(), (Integer) nodeParams.getZ())
|
||||
// .stream()
|
||||
// .map(e -> new NodeTkv(e, nodeParams.getX(), nodeParams.getY(), nodeParams.getY()
|
||||
// .getKeyCode()))
|
||||
// .map(
|
||||
// e -> new HgPair<>(this.proxy.getStoreNode(e.getNodeId()), e)
|
||||
// );
|
||||
// }else {
|
||||
// HgAssert.isTrue(nodeParams.getZ() instanceof byte[],
|
||||
// "Illegal parameter to get node id");
|
||||
// throw new NotImplementedException();
|
||||
// }
|
||||
// }
|
||||
// };
|
||||
|
||||
// private Function<HgTriple<String, HgOwnerKey, Object>,
|
||||
// List<HgPair<HgStoreNode, NodeTkv>>> nodeStreamWrapper = nodeParams -> {
|
||||
// if (nodeParams.getZ() == null) {
|
||||
// return this.proxy.getNode(nodeParams.getX(), nodeParams.getY());
|
||||
// } else {
|
||||
// if (nodeParams.getZ() instanceof HgOwnerKey) {
|
||||
// return this.proxy.getNode(nodeParams.getX(), nodeParams.getY(),
|
||||
// (HgOwnerKey) nodeParams.getZ());
|
||||
// }
|
||||
// if (nodeParams.getZ() instanceof Integer) {
|
||||
// Collection<HgNodePartition> nodePartitions = this.proxy.doPartition(nodeParams.getX(),
|
||||
// (Integer) nodeParams.getZ());
|
||||
// ArrayList<HgPair<HgStoreNode, NodeTkv>> hgPairs = new ArrayList<>(nodePartitions.size());
|
||||
// for (HgNodePartition nodePartition : nodePartitions) {
|
||||
// NodeTkv nodeTkv = new NodeTkv(nodePartition, nodeParams.getX(), nodeParams.getY(),
|
||||
// nodeParams.getY().getKeyCode());
|
||||
// hgPairs.add(new HgPair<>(this.proxy.getStoreNode(nodeTkv.getNodeId()), nodeTkv));
|
||||
//
|
||||
// }
|
||||
// return hgPairs;
|
||||
// } else {
|
||||
// HgAssert.isTrue(nodeParams.getZ() instanceof byte[], "Illegal parameter to get node id");
|
||||
// throw new RuntimeException("not implemented");
|
||||
// }
|
||||
// }
|
||||
// };
|
||||
|
||||
private boolean doAction(HgTriple<String, HgOwnerKey, Object> nodeParams,
|
||||
Function<NodeTkv, Boolean> action) {
|
||||
if (nodeParams.getZ() == null) {
|
||||
return this.proxy.doAction(nodeParams.getX(), nodeParams.getY(), nodeParams.getY(), action);
|
||||
} else {
|
||||
if (nodeParams.getZ() instanceof HgOwnerKey) {
|
||||
boolean result = this.proxy.doAction(nodeParams.getX(), nodeParams.getY(),
|
||||
(HgOwnerKey) nodeParams.getZ(), action);
|
||||
return result;
|
||||
}
|
||||
if (nodeParams.getZ() instanceof Integer) {
|
||||
return this.proxy.doAction(nodeParams.getX(), nodeParams.getY(), (Integer) nodeParams.getZ(), action);
|
||||
} else {
|
||||
HgAssert.isTrue(nodeParams.getZ() instanceof byte[], "Illegal parameter to get node id");
|
||||
throw new RuntimeException("not implemented");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
boolean prepareTx(HgTriple<String, HgOwnerKey, Object> nodeParams,
|
||||
Function<NodeTkv, Boolean> sessionMapper) {
|
||||
if (this.isTx) {
|
||||
return this.entries.add(new HgPair(nodeParams, sessionMapper));
|
||||
} else {
|
||||
return this.isAllTrue(nodeParams, sessionMapper);
|
||||
}
|
||||
}
|
||||
|
||||
public HgStoreSession openNodeSession(HgStoreNode node) {
|
||||
HgStoreSession res = this.sessions.get(node.getNodeId());
|
||||
if (res == null) {
|
||||
this.sessions.put(node.getNodeId(), (res = node.openSession(this.graphName)));
|
||||
}
|
||||
if (this.isTx) {
|
||||
res.beginTx();
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
<R> R limitOne(
|
||||
Supplier<Stream<HgPair<HgStoreNode, NodeTkv>>> nodeStreamSupplier,
|
||||
Function<SessionData<NodeTkv>, R> sessionMapper, R emptyObj) {
|
||||
|
||||
Optional<R> res = retryingInvoke(
|
||||
() -> nodeStreamSupplier.get()
|
||||
.parallel()
|
||||
.map(
|
||||
pair -> new SessionData<NodeTkv>(
|
||||
openNodeSession(pair.getKey()),
|
||||
pair.getValue())
|
||||
).map(sessionMapper)
|
||||
.filter(
|
||||
r -> isValid(r)
|
||||
)
|
||||
.findAny()
|
||||
.orElseGet(() -> emptyObj)
|
||||
);
|
||||
return res.orElse(emptyObj);
|
||||
}
|
||||
|
||||
<R> List<R> toList(Function<Long, HgStoreNode> nodeFunction
|
||||
, List<HgOwnerKey> keyList
|
||||
, Function<HgOwnerKey, Stream<NodeTkv>> flatMapper
|
||||
, Function<SessionData<List<HgOwnerKey>>, List<R>> sessionMapper) {
|
||||
Optional<List<R>> res = retryingInvoke(
|
||||
() -> keyList.stream()
|
||||
.flatMap(flatMapper)
|
||||
.collect(collector)
|
||||
.entrySet()
|
||||
.stream()
|
||||
.map(
|
||||
e -> new SessionData<List<HgOwnerKey>>
|
||||
(
|
||||
openNodeSession(nodeFunction.apply(e.getKey())), e.getValue()
|
||||
)
|
||||
)
|
||||
.parallel()
|
||||
.map(sessionMapper)
|
||||
.flatMap(
|
||||
e -> e.stream()
|
||||
)
|
||||
//.distinct()
|
||||
.collect(Collectors.toList())
|
||||
);
|
||||
|
||||
return res.orElse(EMPTY_LIST);
|
||||
}
|
||||
|
||||
private boolean isAllTrue(HgTriple<String, HgOwnerKey, Object> nodeParams,
|
||||
Function<NodeTkv, Boolean> action) {
|
||||
Optional<Boolean> res = retryingInvoke(() -> doAction(nodeParams, action));
|
||||
return res.orElse(false);
|
||||
}
|
||||
|
||||
boolean isAllTrue(Supplier<Stream<HgPair<HgStoreNode, NodeTkv>>> dataSource,
|
||||
Function<SessionData<NodeTkv>, Boolean> action) {
|
||||
Optional<Boolean> res = retryingInvoke(
|
||||
() -> dataSource.get()
|
||||
.parallel()
|
||||
.map(
|
||||
pair -> new SessionData<NodeTkv>(
|
||||
openNodeSession(pair.getKey()),
|
||||
pair.getValue())
|
||||
).map(action)
|
||||
.allMatch(Boolean::booleanValue)
|
||||
);
|
||||
|
||||
return res.orElse(false);
|
||||
}
|
||||
|
||||
boolean ifAnyTrue(Supplier<Stream<HgPair<HgStoreNode, NodeTkv>>> nodeStreamSupplier
|
||||
, Function<SessionData<NodeTkv>, Boolean> sessionMapper) {
|
||||
|
||||
Optional<Boolean> res = retryingInvoke(
|
||||
() -> nodeStreamSupplier.get()
|
||||
.parallel()
|
||||
.map(
|
||||
pair -> new SessionData<NodeTkv>(
|
||||
openNodeSession(pair.getKey()),
|
||||
pair.getValue())
|
||||
)
|
||||
.map(sessionMapper)
|
||||
.anyMatch(Boolean::booleanValue)
|
||||
);
|
||||
|
||||
return res.orElse(false);
|
||||
}
|
||||
|
||||
<T> Optional<T> retryingInvoke(Supplier<T> supplier) {
|
||||
return IntStream.rangeClosed(0, NODE_MAX_RETRYING_TIMES).boxed()
|
||||
.map(
|
||||
i -> {
|
||||
T buffer = null;
|
||||
try {
|
||||
buffer = supplier.get();
|
||||
} catch (Throwable t) {
|
||||
if (i + 1 <= NODE_MAX_RETRYING_TIMES) {
|
||||
try {
|
||||
int sleepTime;
|
||||
// 前三次每隔一秒做一次尝试
|
||||
if (i < 3) {
|
||||
sleepTime = 1;
|
||||
} else {
|
||||
// 后面逐次递增
|
||||
sleepTime = i - 1;
|
||||
}
|
||||
log.info("Waiting {} seconds " +
|
||||
"for the next try.",
|
||||
sleepTime);
|
||||
Thread.sleep(sleepTime * 1000L);
|
||||
} catch (InterruptedException e) {
|
||||
log.error("Failed to sleep", e);
|
||||
}
|
||||
} else {
|
||||
log.error(maxTryMsg, t);
|
||||
throw HgStoreClientException.of(
|
||||
t.getMessage(), t);
|
||||
}
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
)
|
||||
.filter(e -> e != null)
|
||||
.findFirst();
|
||||
|
||||
}
|
||||
|
||||
private boolean isValid(Object obj) {
|
||||
if (obj == null) return false;
|
||||
|
||||
if (HgStoreClientConst.EMPTY_BYTES.equals(obj)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (EMPTY_LIST.equals(obj)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
class SessionData<T> {
|
||||
HgStoreSession session;
|
||||
T data;
|
||||
|
||||
SessionData(HgStoreSession session, T data) {
|
||||
this.session = session;
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,837 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import static com.baidu.hugegraph.store.client.util.HgAssert.isArgumentNotNull;
|
||||
import static com.baidu.hugegraph.store.client.util.HgAssert.isArgumentValid;
|
||||
import static com.baidu.hugegraph.store.client.util.HgAssert.isFalse;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_STRING;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.err;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toStr;
|
||||
import static java.util.stream.Collectors.groupingBy;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.BiFunction;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
|
||||
import com.baidu.hugegraph.store.HgKvEntry;
|
||||
import com.baidu.hugegraph.store.HgKvIterator;
|
||||
import com.baidu.hugegraph.store.HgKvOrderedIterator;
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.HgScanQuery;
|
||||
import com.baidu.hugegraph.store.HgStoreSession;
|
||||
import com.baidu.hugegraph.store.client.grpc.KvBatchScanner;
|
||||
import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator;
|
||||
import com.baidu.hugegraph.store.client.util.HgAssert;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConst;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientUtil;
|
||||
import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq.Builder;
|
||||
import com.baidu.hugegraph.store.term.HgPair;
|
||||
import com.baidu.hugegraph.store.term.HgTriple;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2022/01/19
|
||||
* @version 0.6.0 added batch scan on 2022/03/03
|
||||
*/
|
||||
@Slf4j
|
||||
@NotThreadSafe
|
||||
class NodeTxSessionProxy implements HgStoreSession {
|
||||
private final HgStoreNodeManager nodeManager;
|
||||
private final HgStoreNodePartitioner nodePartitioner;
|
||||
private final String graphName;
|
||||
private final NodeTxExecutor txExecutor;
|
||||
|
||||
NodeTxSessionProxy(String graphName, HgStoreNodeManager nodeManager) {
|
||||
this.nodeManager = nodeManager;
|
||||
this.graphName = graphName;
|
||||
this.nodePartitioner = this.nodeManager.getNodePartitioner();
|
||||
this.txExecutor = NodeTxExecutor.graphOf(this.graphName, this);
|
||||
|
||||
isFalse(this.nodePartitioner == null,
|
||||
"Failed to retrieve the node-partitioner from node-manager.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void beginTx() {
|
||||
this.txExecutor.setTx(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void commit() {
|
||||
this.txExecutor.commitTx();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void rollback() {
|
||||
this.txExecutor.rollbackTx();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isTx() {
|
||||
return this.txExecutor.isTx();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean put(String table, HgOwnerKey ownerKey, byte[] value) {
|
||||
// isArgumentValid(table, "table");
|
||||
// isArgumentNotNull(ownerKey, "ownerKey");
|
||||
// log.info("put -> graph: {}, table: {}, key: {}, value: {}",
|
||||
// graphName, table, ownerKey, toByteStr(value));
|
||||
// return this.txExecutor.prepareTx(
|
||||
// () -> getNodeStream(table, ownerKey),
|
||||
// e -> e.session.put(table, e.data.getKey(), value)
|
||||
// );
|
||||
return this.txExecutor.prepareTx(new HgTriple(table, ownerKey, null),
|
||||
e -> e.getSession().put(table,
|
||||
e.getKey(),
|
||||
value));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean directPut(String table, int partitionId, HgOwnerKey ownerKey, byte[] value) {
|
||||
isArgumentValid(table, "table");
|
||||
isArgumentNotNull(ownerKey, "ownerKey");
|
||||
|
||||
return this.txExecutor.prepareTx(
|
||||
new HgTriple(table, ownerKey, (Integer) partitionId),
|
||||
e -> e.getSession().put(table, e.getKey(), value)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean delete(String table, HgOwnerKey ownerKey) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
HgAssert.isFalse(ownerKey == null, "The argument is invalid: ownerKey");
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("delete -> graph: {}, table: {}, key: {}"
|
||||
, graphName, table, toStr(ownerKey));
|
||||
}
|
||||
|
||||
return this.txExecutor
|
||||
.prepareTx(
|
||||
new HgTriple(table, ownerKey, null),
|
||||
e -> e.getSession().delete(table, e.getKey())
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean deleteSingle(String table, HgOwnerKey ownerKey) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
HgAssert.isFalse(ownerKey == null, "The argument is invalid: ownerKey");
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("deleteSingle -> graph: {}, table: {}, key: {}"
|
||||
, graphName, table, toStr(ownerKey));
|
||||
}
|
||||
|
||||
return this.txExecutor
|
||||
.prepareTx(
|
||||
new HgTriple(table, ownerKey, null),
|
||||
e -> e.getSession().deleteSingle(table, e.getKey())
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean deletePrefix(String table, HgOwnerKey prefix) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
HgAssert.isFalse(prefix == null, "The argument is invalid: prefix");
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("deletePrefix -> graph: {}, table: {}, prefix: {}"
|
||||
, graphName, table, toStr(prefix));
|
||||
}
|
||||
|
||||
return this.txExecutor
|
||||
.prepareTx(
|
||||
new HgTriple(table, prefix, null),
|
||||
e -> e.getSession().deletePrefix(table, e.getKey())
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean deleteRange(String table, HgOwnerKey start, HgOwnerKey end) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
HgAssert.isFalse(start == null, "The argument is invalid: start");
|
||||
HgAssert.isFalse(end == null, "The argument is invalid: end");
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("deleteRange -> graph: {}, table: {}, start: {}, end: {}"
|
||||
, graphName, table, toStr(start), toStr(end));
|
||||
}
|
||||
|
||||
return this.txExecutor
|
||||
.prepareTx(
|
||||
new HgTriple(table, start, end),
|
||||
e -> e.getSession().deleteRange(table, e.getKey(), e.getEndKey())
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean merge(String table, HgOwnerKey key, byte[] value) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
HgAssert.isFalse(key == null, "The argument is invalid: key");
|
||||
HgAssert.isFalse(value == null, "The argument is invalid: value");
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("merge -> graph: {}, table: {}, key: {}, value: {}"
|
||||
, graphName, table, toStr(key), toStr(value));
|
||||
}
|
||||
|
||||
return this.txExecutor
|
||||
.prepareTx(
|
||||
new HgTriple(table, key, value),
|
||||
e -> e.getSession().merge(table, e.getKey(), value)
|
||||
);
|
||||
}
|
||||
|
||||
/*--- tx end ---*/
|
||||
|
||||
@Override
|
||||
public byte[] get(String table, HgOwnerKey ownerKey) {
|
||||
isArgumentValid(table, "table");
|
||||
isArgumentNotNull(ownerKey, "ownerKey");
|
||||
|
||||
return this.txExecutor
|
||||
.limitOne(
|
||||
() -> this.getNodeStream(table, ownerKey),
|
||||
e -> e.session.get(table, e.data.getKey()), HgStoreClientConst.EMPTY_BYTES
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean clean(int partId) {
|
||||
Collection<HgNodePartition> nodes = this.doPartition("", partId);
|
||||
return nodes.parallelStream()
|
||||
.map(
|
||||
e -> this.getStoreNode(e.getNodeId()).openSession(this.graphName)
|
||||
.clean(partId)
|
||||
).findFirst().get();
|
||||
}
|
||||
|
||||
@Override
|
||||
@Deprecated
|
||||
public List<HgKvEntry> batchGetOwner(String table, List<HgOwnerKey> keyList) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
HgAssert.isFalse(HgAssert.isInvalid(keyList), "The argument is invalid: keyList");
|
||||
|
||||
return this.txExecutor
|
||||
.toList(
|
||||
(l) -> this.getStoreNode(l),
|
||||
keyList,
|
||||
key -> this.toNodeTkvList(table, key, key).stream(),
|
||||
e -> e.session.batchGetOwner(table, e.data)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> batchPrefix(String table, List<HgOwnerKey> keyList) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
HgAssert.isFalse(HgAssert.isInvalid(keyList), "The argument is invalid: keyList");
|
||||
return this.toHgKvIteratorProxy(
|
||||
this.txExecutor
|
||||
.toList(
|
||||
(l) -> this.getStoreNode(l),
|
||||
keyList,
|
||||
key -> this.toNodeTkvList(table, key, key).stream(),
|
||||
e -> Collections.singletonList(e.session.batchPrefix(table, e.data))
|
||||
)
|
||||
, Long.MAX_VALUE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean truncate() {
|
||||
return this.txExecutor
|
||||
.isAllTrue(
|
||||
() -> this.getNodeStream(EMPTY_STRING),
|
||||
e -> e.session.truncate()
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean existsTable(String table) {
|
||||
return this.txExecutor
|
||||
.ifAnyTrue(
|
||||
() -> this.getNodeStream(EMPTY_STRING),
|
||||
e -> e.session.existsTable(table)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean createTable(String table) {
|
||||
return this.txExecutor
|
||||
.isAllTrue(
|
||||
() -> this.getNodeStream(EMPTY_STRING),
|
||||
e -> e.session.createTable(table)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean deleteTable(String table) {
|
||||
return this.txExecutor
|
||||
.isAllTrue(
|
||||
() -> this.getNodeStream(EMPTY_STRING),
|
||||
e -> e.session.deleteTable(table)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean dropTable(String table) {
|
||||
return this.txExecutor
|
||||
.isAllTrue(
|
||||
() -> this.getNodeStream(table),
|
||||
e -> e.session.dropTable(table)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean deleteGraph(String graph) {
|
||||
return this.txExecutor
|
||||
.isAllTrue(
|
||||
() -> this.getNodeStream(EMPTY_STRING),
|
||||
e -> e.session.deleteGraph(graph)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table) {
|
||||
return scanIterator(table, 0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, byte[] query) {
|
||||
return scanIterator(table, 0, query);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, long limit) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
|
||||
return this.toHgKvIteratorProxy(
|
||||
this.toNodeTkvList(table)
|
||||
.parallelStream()
|
||||
.map(
|
||||
e -> this.getStoreNode(e.getNodeId()).openSession(this.graphName)
|
||||
.scanIterator(e.getTable(), limit)
|
||||
)
|
||||
.collect(Collectors.toList())
|
||||
, limit);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, long limit, byte[] query) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
|
||||
return this.toHgKvIteratorProxy(
|
||||
this.toNodeTkvList(table)
|
||||
.parallelStream()
|
||||
.map(
|
||||
e -> this.getStoreNode(e.getNodeId()).openSession(this.graphName)
|
||||
.scanIterator(e.getTable(), e.getKey(), limit, query)
|
||||
)
|
||||
.collect(Collectors.toList())
|
||||
, limit);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey keyPrefix) {
|
||||
return scanIterator(table, keyPrefix, 0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey keyPrefix, long limit) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
HgAssert.isFalse(keyPrefix == null, "The argument is invalid: keyPrefix");
|
||||
|
||||
return this.toHgKvIteratorProxy(
|
||||
this.toNodeTkvList(table, keyPrefix)
|
||||
.parallelStream()
|
||||
.map(
|
||||
e -> this.getStoreNode(e.getNodeId()).openSession(this.graphName)
|
||||
.scanIterator(e.getTable(), e.getKey(), limit)
|
||||
)
|
||||
.collect(Collectors.toList())
|
||||
, limit);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey keyPrefix, long limit, byte[] query) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
HgAssert.isFalse(keyPrefix == null, "The argument is invalid: keyPrefix");
|
||||
|
||||
return this.toHgKvIteratorProxy(
|
||||
this.toNodeTkvList(table, keyPrefix)
|
||||
.parallelStream()
|
||||
.map(
|
||||
e -> this.getStoreNode(e.getNodeId()).openSession(this.graphName)
|
||||
.scanIterator(e.getTable(), e.getKey(), limit, query)
|
||||
)
|
||||
.collect(Collectors.toList())
|
||||
, limit);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey) {
|
||||
return this.scanIterator(table, startKey, endKey, 0, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
HgAssert.isFalse(startKey == null, "The argument is invalid: startKey");
|
||||
HgAssert.isFalse(endKey == null, "The argument is invalid: endKey");
|
||||
|
||||
return this.toHgKvIteratorProxy(
|
||||
this.toNodeTkvList(table, startKey, endKey)
|
||||
.parallelStream()
|
||||
.map(
|
||||
e -> this.getStoreNode(e.getNodeId()).openSession(this.graphName)
|
||||
.scanIterator(e.getTable(), e.getKey(), e.getEndKey(), limit)
|
||||
)
|
||||
.collect(Collectors.toList())
|
||||
, limit);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey
|
||||
, long limit, byte[] query) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
HgAssert.isFalse(startKey == null, "The argument is invalid: startKey");
|
||||
HgAssert.isFalse(endKey == null, "The argument is invalid: endKey");
|
||||
|
||||
return this.toHgKvIteratorProxy(
|
||||
this.toNodeTkvList(table, startKey, endKey)
|
||||
.parallelStream()
|
||||
.map(
|
||||
e -> this.getStoreNode(e.getNodeId()).openSession(this.graphName)
|
||||
.scanIterator(e.getTable(), e.getKey(), e.getEndKey(), limit, query)
|
||||
)
|
||||
.collect(Collectors.toList())
|
||||
, limit);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey
|
||||
, long limit, int scanType, byte[] query) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
HgAssert.isFalse(startKey == null, "The argument is invalid: startKey");
|
||||
HgAssert.isFalse(endKey == null, "The argument is invalid: endKey");
|
||||
|
||||
return this.toHgKvIteratorProxy(
|
||||
this.toNodeTkvList(table, startKey, endKey)
|
||||
.parallelStream()
|
||||
.map(
|
||||
e -> this.getStoreNode(e.getNodeId()).openSession(this.graphName)
|
||||
.scanIterator(e.getTable(), e.getKey(), e.getEndKey(), limit, scanType, query)
|
||||
)
|
||||
.collect(Collectors.toList())
|
||||
, limit);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, int codeFrom, int codeTo, int scanType, byte[] query) {
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("graph: {}, table: {}, codeFrom: {}, codeTo: {}, scanType: {}, query: {}"
|
||||
, graphName, table, codeFrom, codeTo, scanType, HgStoreClientUtil.toStr(query));
|
||||
}
|
||||
|
||||
HgAssert.isFalse(HgAssert.isInvalid(table), "The argument is invalid: table");
|
||||
return this.toHgKvIteratorProxy(
|
||||
this.toNodeTkvList(table, codeFrom, codeTo)
|
||||
.parallelStream()
|
||||
.map(
|
||||
e -> this.getStoreNode(e.getNodeId()).openSession(this.graphName)
|
||||
.scanIterator(e.getTable()
|
||||
, e.getKey().getKeyCode()
|
||||
, e.getEndKey().getKeyCode(),
|
||||
scanType, query)
|
||||
)
|
||||
.collect(Collectors.toList())
|
||||
, 0);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(Builder scanReqBuilder) {
|
||||
List<NodeTkv> nodeTKvs = this.toNodeTkvList(scanReqBuilder);
|
||||
Function<NodeTkv, HgKvIterator<HgKvEntry>> hgKvIteratorFunction = e -> {
|
||||
HgStoreSession session = this.getStoreNode(e.getNodeId())
|
||||
.openSession(this.graphName);
|
||||
return session.scanIterator(scanReqBuilder);
|
||||
};
|
||||
List<HgKvIterator> iterators = nodeTKvs.parallelStream()
|
||||
.map(hgKvIteratorFunction)
|
||||
.collect(Collectors.toList());
|
||||
return this.toHgKvIteratorProxy(iterators, scanReqBuilder.getLimit());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HgKvIterator<HgKvEntry>> scanBatch(HgScanQuery scanQuery) {
|
||||
HgAssert.isArgumentNotNull(scanQuery, "scanQuery");
|
||||
|
||||
return this.toTkvMapFunc(scanQuery.getScanMethod())
|
||||
.apply(scanQuery)
|
||||
.entrySet()
|
||||
.stream()
|
||||
.map(e ->
|
||||
this.getStoreNode(e.getKey())
|
||||
.openSession(this.graphName)
|
||||
.scanBatch(toScanQueryFunc(scanQuery.getScanMethod())
|
||||
.apply(scanQuery.getTable(), e.getValue())
|
||||
.setQuery(scanQuery.getQuery())
|
||||
.setLimit(scanQuery.getLimit())
|
||||
.setPerKeyLimit(scanQuery.getPerKeyLimit())
|
||||
.setPerKeyMax((scanQuery.getPerKeyMax()))
|
||||
.setScanType(scanQuery.getScanType())
|
||||
.build()
|
||||
)
|
||||
)
|
||||
//.peek(e->log.info("{}",e))
|
||||
.flatMap(List::stream)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public KvCloseableIterator<HgKvIterator<HgKvEntry>> scanBatch2(HgScanQuery scanQuery) {
|
||||
return scanBatch3(scanQuery, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public KvCloseableIterator<HgKvIterator<HgKvEntry>> scanBatch3(HgScanQuery scanQuery,
|
||||
KvCloseableIterator iterator) {
|
||||
KvCloseableIterator notifierWrap = KvBatchScanner.ofMerger(scanQuery, (query, notifier) -> {
|
||||
Map<Long, List<NodeTkv>> nodeTkvs = this.toTkvMapFunc(scanQuery.getScanMethod())
|
||||
.apply(query);
|
||||
|
||||
nodeTkvs.forEach((storeId, tkvs) -> {
|
||||
this.getStoreNode(storeId)
|
||||
.openSession(this.graphName)
|
||||
.scanBatch3(toScanQueryFunc(scanQuery.getScanMethod())
|
||||
.apply(scanQuery.getTable(), tkvs)
|
||||
.setQuery(scanQuery.getQuery())
|
||||
.setLimit(scanQuery.getLimit())
|
||||
.setSkipDegree(scanQuery.getSkipDegree())
|
||||
.setPerKeyLimit(scanQuery.getPerKeyLimit())
|
||||
.setPerKeyMax((scanQuery.getPerKeyMax()))
|
||||
.setScanType(scanQuery.getScanType())
|
||||
.setOrderType(scanQuery.getOrderType())
|
||||
.build(), notifier
|
||||
);
|
||||
});
|
||||
return true;
|
||||
});
|
||||
return notifierWrap;
|
||||
}
|
||||
|
||||
private Function<HgScanQuery, Map<Long, List<NodeTkv>>> toTkvMapFunc(HgScanQuery.ScanMethod scanMethod) {
|
||||
switch (scanMethod) {
|
||||
case RANGE:
|
||||
return scanQuery -> {
|
||||
List<HgOwnerKey> starts = scanQuery.getStartList();
|
||||
List<HgOwnerKey> ends = scanQuery.getEndList();
|
||||
int size = starts.size();
|
||||
return IntStream.range(0, size)
|
||||
.boxed()
|
||||
.map(i -> this.toNodeTkvList(scanQuery.getTable(), starts.get(i), ends.get(i)))
|
||||
.flatMap(List::stream)
|
||||
.collect(groupingBy(NodeTkv::getNodeId));
|
||||
};
|
||||
case PREFIX:
|
||||
return scanQuery ->
|
||||
scanQuery.getPrefixList()
|
||||
.stream()
|
||||
.map(keyPrefix -> this.toNodeTkvList(scanQuery.getTable(), keyPrefix))
|
||||
.flatMap(List::stream)
|
||||
.collect(groupingBy(NodeTkv::getNodeId));
|
||||
|
||||
default:
|
||||
return scanQuery -> this.toNodeTkvList(scanQuery.getTable())
|
||||
.stream()
|
||||
.collect(groupingBy(NodeTkv::getNodeId));
|
||||
}
|
||||
}
|
||||
|
||||
private BiFunction<String, List<NodeTkv>, HgScanQuery.ScanBuilder> toScanQueryFunc(
|
||||
HgScanQuery.ScanMethod scanMethod) {
|
||||
switch (scanMethod) {
|
||||
case RANGE:
|
||||
return (table, tkvList) -> {
|
||||
List<HgOwnerKey> startList = new LinkedList();
|
||||
List<HgOwnerKey> endList = new LinkedList();
|
||||
|
||||
tkvList.stream().forEach(e -> {
|
||||
startList.add(e.getKey());
|
||||
endList.add(e.getEndKey());
|
||||
});
|
||||
|
||||
return HgScanQuery.ScanBuilder.rangeOf(table, startList, endList);
|
||||
};
|
||||
case PREFIX:
|
||||
return (table, tkvList) ->
|
||||
HgScanQuery.ScanBuilder.prefixOf(table,
|
||||
tkvList.stream()
|
||||
.map(e -> e.getKey())
|
||||
.collect(Collectors.toList())
|
||||
);
|
||||
default:
|
||||
return (table, tkvList) -> HgScanQuery.ScanBuilder.tableOf(table);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/*-- common --*/
|
||||
private HgKvIterator toHgKvIteratorProxy(List<HgKvIterator> iteratorList, long limit) {
|
||||
boolean isAllOrderedLimiter = iteratorList.stream()
|
||||
.allMatch(e -> HgKvOrderedIterator.class.isInstance(e));
|
||||
|
||||
HgKvIterator<HgKvEntry> iterator;
|
||||
if (isAllOrderedLimiter) {
|
||||
iterator = new SequencedIterator(iteratorList.stream()
|
||||
.map(e -> (HgKvOrderedIterator) e)
|
||||
.collect(Collectors.toList()), limit);
|
||||
} else {
|
||||
iterator = new TopWorkIteratorProxy(iteratorList, limit);
|
||||
}
|
||||
|
||||
return iterator;
|
||||
}
|
||||
|
||||
HgStoreNode getStoreNode(Long nodeId) {
|
||||
HgStoreNode res = this.nodeManager.applyNode(this.graphName, nodeId);
|
||||
|
||||
if (res == null) {
|
||||
throw err("Failed to apply for an instance of HgStoreNode from node-manager.");
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
public boolean doAction(String table, HgOwnerKey startKey, HgOwnerKey endKey, Function<NodeTkv, Boolean> action) {
|
||||
Collection<HgNodePartition> partitions = doPartition(table, startKey.getOwner(), endKey.getOwner());
|
||||
for (HgNodePartition partition : partitions) {
|
||||
HgStoreNode storeNode = this.getStoreNode(partition.getNodeId());
|
||||
HgStoreSession session = this.txExecutor.openNodeSession(storeNode);
|
||||
NodeTkv data = new NodeTkv(partition, table, startKey, endKey);
|
||||
data.setSession(session);
|
||||
if (!action.apply(data)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean doAction(String table, HgOwnerKey startKey, Integer code, Function<NodeTkv, Boolean> action) {
|
||||
Collection<HgNodePartition> partitions = this.doPartition(table, code);
|
||||
for (HgNodePartition partition : partitions) {
|
||||
HgStoreNode storeNode = this.getStoreNode(partition.getNodeId());
|
||||
HgStoreSession session = this.txExecutor.openNodeSession(storeNode);
|
||||
NodeTkv data = new NodeTkv(partition, table, startKey, code);
|
||||
data.setSession(session);
|
||||
if (!action.apply(data)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private List<NodeTkv> toNodeTkvList(Builder scanReqBuilder) {
|
||||
// TODO 使用builder获取owner
|
||||
String table = scanReqBuilder.getTable();
|
||||
HgOwnerKey ownerKey = HgStoreClientConst.ALL_PARTITION_OWNER_KEY;
|
||||
byte[] allOwner = ownerKey.getOwner();
|
||||
Collection<HgNodePartition> partitions = doPartition(table,
|
||||
allOwner,
|
||||
allOwner);
|
||||
List<NodeTkv> nodeTkvs = new ArrayList<>(partitions.size());
|
||||
for (HgNodePartition partition : partitions) {
|
||||
nodeTkvs.add(new NodeTkv(partition, table, ownerKey, ownerKey));
|
||||
}
|
||||
return nodeTkvs;
|
||||
}
|
||||
|
||||
private List<NodeTkv> toNodeTkvList(String table) {
|
||||
Collection<HgNodePartition> partitions = doPartition(table,
|
||||
HgStoreClientConst.ALL_PARTITION_OWNER_KEY.getOwner(),
|
||||
HgStoreClientConst.ALL_PARTITION_OWNER_KEY.getOwner());
|
||||
ArrayList<NodeTkv> nodeTkvs = new ArrayList<>(partitions.size());
|
||||
for (HgNodePartition partition : partitions) {
|
||||
nodeTkvs.add(new NodeTkv(partition, table, HgStoreClientConst.ALL_PARTITION_OWNER_KEY,
|
||||
HgStoreClientConst.ALL_PARTITION_OWNER_KEY));
|
||||
}
|
||||
return nodeTkvs;
|
||||
}
|
||||
|
||||
private List<NodeTkv> toNodeTkvList(String table, HgOwnerKey ownerKey) {
|
||||
Collection<HgNodePartition> partitions = doPartition(table, ownerKey.getOwner(), ownerKey.getOwner());
|
||||
ArrayList<NodeTkv> nodeTkvs = new ArrayList<>(partitions.size());
|
||||
for (HgNodePartition partition : partitions) {
|
||||
nodeTkvs.add(new NodeTkv(partition, table, ownerKey, ownerKey));
|
||||
}
|
||||
|
||||
return nodeTkvs;
|
||||
}
|
||||
|
||||
private List<NodeTkv> toNodeTkvList(String table, HgOwnerKey startKey, HgOwnerKey endKey) {
|
||||
Collection<HgNodePartition> partitions = doPartition(table, startKey.getOwner(), endKey.getOwner());
|
||||
ArrayList<NodeTkv> nodeTkvs = new ArrayList<>(partitions.size());
|
||||
for (HgNodePartition partition : partitions) {
|
||||
nodeTkvs.add(new NodeTkv(partition, table, startKey, endKey));
|
||||
}
|
||||
return nodeTkvs;
|
||||
}
|
||||
|
||||
|
||||
private List<NodeTkv> toNodeTkvList(String table, int startCode, int endCode) {
|
||||
Collection<HgNodePartition> partitions = this.doPartition(table, startCode, endCode);
|
||||
ArrayList<NodeTkv> nodeTkvs = new ArrayList<>(partitions.size());
|
||||
for (HgNodePartition partition : partitions) {
|
||||
nodeTkvs.add(
|
||||
new NodeTkv(partition, table, HgOwnerKey.codeOf(startCode), HgOwnerKey.codeOf(endCode)));
|
||||
}
|
||||
return nodeTkvs;
|
||||
}
|
||||
/**
|
||||
* @return not null
|
||||
*/
|
||||
private Collection<HgNodePartition> doPartition(String table, byte[] startKey, byte[] endKey) {
|
||||
HgNodePartitionerBuilder partitionerBuilder = HgNodePartitionerBuilder.resetAndGet();
|
||||
|
||||
int status = this.nodePartitioner.partition(partitionerBuilder, this.graphName, startKey, endKey);
|
||||
|
||||
if (status != 0) {
|
||||
throw err("The node-partitioner is not work.");
|
||||
}
|
||||
|
||||
Collection<HgNodePartition> partitions = partitionerBuilder.getPartitions();
|
||||
|
||||
if (partitions.isEmpty()) {
|
||||
throw err("Failed to get the collection of HgNodePartition from node-partitioner.");
|
||||
}
|
||||
|
||||
return partitions;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return @return not null
|
||||
*/
|
||||
private Collection<HgNodePartition> doPartition(String table, int startCode, int endCode) {
|
||||
HgNodePartitionerBuilder partitionerBuilder = HgNodePartitionerBuilder.resetAndGet();
|
||||
int status = this.nodePartitioner.partition(partitionerBuilder, this.graphName, startCode, endCode);
|
||||
|
||||
if (status != 0) {
|
||||
throw err("The node-partitioner is not work.");
|
||||
}
|
||||
|
||||
Collection<HgNodePartition> partitions = partitionerBuilder.getPartitions();
|
||||
|
||||
if (partitions.isEmpty()) {
|
||||
throw err("Failed to get the collection of HgNodePartition from node-partitioner.");
|
||||
}
|
||||
|
||||
return partitions;
|
||||
}
|
||||
|
||||
Collection<HgNodePartition> doPartition(String table, int partitionId) {
|
||||
HgNodePartitionerBuilder partitionerBuilder = HgNodePartitionerBuilder.resetAndGet();
|
||||
int status = this.nodePartitioner.partition(partitionerBuilder, this.graphName, partitionId);
|
||||
|
||||
if (status != 0) {
|
||||
throw err("The node-partitioner is not work.");
|
||||
}
|
||||
|
||||
Collection<HgNodePartition> partitions = partitionerBuilder.getPartitions();
|
||||
|
||||
if (partitions.isEmpty()) {
|
||||
throw err("Failed to get the collection of HgNodePartition from node-partitioner.");
|
||||
}
|
||||
|
||||
return partitions;
|
||||
}
|
||||
private Stream<HgPair<HgStoreNode, NodeTkv>> getNodeStream(String table) {
|
||||
return this.toNodeTkvList(table)
|
||||
.stream()
|
||||
.map(
|
||||
e -> new HgPair<>(this.getStoreNode(e.getNodeId()), e)
|
||||
);
|
||||
}
|
||||
|
||||
Stream<HgPair<HgStoreNode, NodeTkv>> getNodeStream(String table,
|
||||
HgOwnerKey ownerKey) {
|
||||
return this.toNodeTkvList(table, ownerKey)
|
||||
.stream()
|
||||
.map(
|
||||
e -> new HgPair<>(this.getStoreNode(e.getNodeId()), e)
|
||||
);
|
||||
}
|
||||
|
||||
Stream<HgPair<HgStoreNode, NodeTkv>> getNodeStream(String table, HgOwnerKey startKey, HgOwnerKey endKey) {
|
||||
return this.toNodeTkvList(table, startKey, endKey)
|
||||
.stream()
|
||||
.map(
|
||||
e -> new HgPair<>(this.getStoreNode(e.getNodeId()), e)
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
// private List<HgPair<HgStoreNode, NodeTkv>> getNode(String table) {
|
||||
// List<NodeTkv> nodeTkvList = this.toNodeTkvList(table);
|
||||
// return nodeTkv2Node(nodeTkvList);
|
||||
// }
|
||||
|
||||
List<HgPair<HgStoreNode, NodeTkv>> getNode(String table, HgOwnerKey ownerKey) {
|
||||
List<NodeTkv> nodeTkvList = this.toNodeTkvList(table, ownerKey);
|
||||
return nodeTkv2Node(nodeTkvList);
|
||||
}
|
||||
|
||||
List<HgPair<HgStoreNode, NodeTkv>> getNode(String table, HgOwnerKey startKey, HgOwnerKey endKey) {
|
||||
List<NodeTkv> nodeTkvList = this.toNodeTkvList(table, startKey, endKey);
|
||||
return nodeTkv2Node(nodeTkvList);
|
||||
|
||||
}
|
||||
//
|
||||
//boolean doAction(String table, HgOwnerKey startKey, HgOwnerKey endKey,
|
||||
// Function<NodeTkv, Boolean> action) {
|
||||
// return this.doAction(table, startKey, endKey, action);
|
||||
//
|
||||
//}
|
||||
|
||||
// List<HgPair<HgStoreNode, NodeTkv>> getNode(String table, Integer endKey) {
|
||||
// .stream()
|
||||
// .map(e -> new NodeTkv(e, nodeParams.getX(), nodeParams.getY(), nodeParams.getY().getKeyCode()))
|
||||
// .map(
|
||||
// e -> new HgPair<>(this.proxy.getStoreNode(e.getNodeId()), e)
|
||||
// );
|
||||
// Collection<HgNodePartition> nodePartitions = this.doPartition(table, endKey);
|
||||
// for (HgNodePartition nodePartition: nodePartitions) {
|
||||
//
|
||||
// }
|
||||
// return nodeTkv2Node(nodeTkvList);
|
||||
//
|
||||
// }
|
||||
|
||||
private List<HgPair<HgStoreNode, NodeTkv>> nodeTkv2Node(Collection<NodeTkv> nodeTkvList) {
|
||||
ArrayList<HgPair<HgStoreNode, NodeTkv>> hgPairs = new ArrayList<>(nodeTkvList.size());
|
||||
for (NodeTkv nodeTkv : nodeTkvList) {
|
||||
hgPairs.add(new HgPair<>(this.getStoreNode(nodeTkv.getNodeId()), nodeTkv));
|
||||
}
|
||||
return hgPairs;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Queue;
|
||||
|
||||
import com.baidu.hugegraph.store.HgKvEntry;
|
||||
import com.baidu.hugegraph.store.HgKvIterator;
|
||||
import com.baidu.hugegraph.store.HgKvOrderedIterator;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Proxy iterator orderly, to switch next one will happen when the current one is empty.
|
||||
*
|
||||
* @author lynn.bond@hotmail.com created on 2022/03/10
|
||||
* @version 0.1.0
|
||||
*/
|
||||
@Slf4j
|
||||
public class SequencedIterator implements HgKvIterator {
|
||||
private static final byte[] EMPTY_BYTES = new byte[0];
|
||||
private HgKvOrderedIterator<HgKvEntry> iterator;
|
||||
private Queue<HgKvOrderedIterator> queue;
|
||||
private HgKvEntry entry;
|
||||
private long limit;
|
||||
private int count;
|
||||
private byte[] position = EMPTY_BYTES;
|
||||
private byte[] position4Seeking = EMPTY_BYTES;
|
||||
|
||||
SequencedIterator(List<HgKvOrderedIterator> iterators, long limit) {
|
||||
Collections.sort(iterators);
|
||||
this.queue = new LinkedList(iterators);
|
||||
this.limit = limit <= 0 ? Integer.MAX_VALUE : limit;
|
||||
}
|
||||
|
||||
private HgKvOrderedIterator getIterator() {
|
||||
if (this.queue.isEmpty()) return null;
|
||||
HgKvOrderedIterator buf;
|
||||
while ((buf = this.queue.poll()) != null) {
|
||||
buf.seek(this.position4Seeking);
|
||||
if (buf.hasNext()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (buf == null) {
|
||||
return null;
|
||||
}
|
||||
return buf;
|
||||
}
|
||||
|
||||
private void closeIterators() {
|
||||
if (this.queue.isEmpty()) return;
|
||||
HgKvOrderedIterator buf;
|
||||
while ((buf = this.queue.poll()) != null) {
|
||||
buf.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] key() {
|
||||
if (this.entry != null) {
|
||||
return this.entry.key();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] value() {
|
||||
if (this.entry != null) {
|
||||
return this.entry.value();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] position() {
|
||||
return this.position;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void seek(byte[] pos) {
|
||||
if (pos != null) {
|
||||
this.position4Seeking = pos;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
if (this.count >= this.limit) {
|
||||
return false;
|
||||
}
|
||||
if (this.iterator == null) {
|
||||
this.iterator = this.getIterator();
|
||||
} else if (!this.iterator.hasNext()) {
|
||||
this.iterator.close();
|
||||
this.iterator = this.getIterator();
|
||||
}
|
||||
if (this.iterator == null) {
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object next() {
|
||||
if (this.iterator == null) {
|
||||
hasNext();
|
||||
}
|
||||
if (this.iterator == null) {
|
||||
throw new NoSuchElementException();
|
||||
}
|
||||
this.entry = this.iterator.next();
|
||||
this.position = this.iterator.position();
|
||||
if (!this.iterator.hasNext()) {
|
||||
this.iterator.close();
|
||||
this.iterator = null;
|
||||
}
|
||||
this.count++;
|
||||
return this.entry;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (this.iterator != null) {
|
||||
this.iterator.close();
|
||||
}
|
||||
this.closeIterators();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,144 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Queue;
|
||||
|
||||
import com.baidu.hugegraph.store.HgKvEntry;
|
||||
import com.baidu.hugegraph.store.HgKvIterator;
|
||||
import com.baidu.hugegraph.store.HgKvPagingIterator;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/24
|
||||
* @version 0.1.1
|
||||
*/
|
||||
@Slf4j
|
||||
public class ShiftWorkIteratorProxy implements HgKvIterator {
|
||||
private static final byte[] EMPTY_BYTES = new byte[0];
|
||||
private HgKvPagingIterator<HgKvEntry> iterator;
|
||||
private Queue<HgKvPagingIterator> queue = new LinkedList<>();
|
||||
private HgKvEntry entry;
|
||||
private int limit;
|
||||
private int count;
|
||||
private int shiftCount;
|
||||
|
||||
ShiftWorkIteratorProxy(List<HgKvPagingIterator> iterators, int limit) {
|
||||
this.queue = new LinkedList<>(iterators);
|
||||
this.limit = limit <= 0 ? Integer.MAX_VALUE : limit;
|
||||
}
|
||||
|
||||
private HgKvPagingIterator getIterator() {
|
||||
if (this.queue.isEmpty()) return null;
|
||||
|
||||
HgKvPagingIterator buf = null;
|
||||
|
||||
while ((buf = this.queue.poll()) != null) {
|
||||
if (buf.hasNext()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (buf == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
this.queue.add(buf);
|
||||
|
||||
return buf;
|
||||
}
|
||||
|
||||
private void closeIterators() {
|
||||
if (this.queue.isEmpty()) return;
|
||||
|
||||
HgKvPagingIterator buf;
|
||||
|
||||
while ((buf = this.queue.poll()) != null) {
|
||||
buf.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void setIterator() {
|
||||
|
||||
// if (++this.shiftCount >= this.iterator.getPageSize() / 2) {
|
||||
if (++this.shiftCount >= this.iterator.getPageSize()) {
|
||||
this.iterator = null;
|
||||
this.shiftCount = 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void doNext() {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] key() {
|
||||
if (this.entry != null) {
|
||||
return this.entry.key();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] value() {
|
||||
if (this.entry != null) {
|
||||
return this.entry.value();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] position() {
|
||||
return this.iterator != null ? this.iterator.position() : EMPTY_BYTES;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void seek(byte[] position) {
|
||||
if (this.iterator != null) {
|
||||
this.iterator.seek(position);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
if (this.count >= this.limit) {
|
||||
return false;
|
||||
}
|
||||
if (this.iterator == null
|
||||
|| !this.iterator.hasNext()) {
|
||||
this.iterator = this.getIterator();
|
||||
}
|
||||
if (this.iterator == null) {
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object next() {
|
||||
if (this.iterator == null) {
|
||||
hasNext();
|
||||
}
|
||||
if (this.iterator == null) {
|
||||
throw new NoSuchElementException();
|
||||
}
|
||||
this.entry = this.iterator.next();
|
||||
this.setIterator();
|
||||
this.count++;
|
||||
//log.info("next - > {}",this.entry);
|
||||
return this.entry;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (this.iterator != null) {
|
||||
this.iterator.close();
|
||||
}
|
||||
this.closeIterators();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
package com.baidu.hugegraph.store.client;
|
||||
|
||||
import com.baidu.hugegraph.store.HgKvEntry;
|
||||
import com.baidu.hugegraph.store.HgKvIterator;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/21
|
||||
* @version 0.1.0
|
||||
*/
|
||||
class TopWorkIteratorProxy implements HgKvIterator {
|
||||
private static final byte[] EMPTY_BYTES = new byte[0];
|
||||
private HgKvIterator<HgKvEntry> iterator;
|
||||
private Queue<HgKvIterator> queue;
|
||||
private HgKvEntry entry;
|
||||
private long limit;
|
||||
private int count;
|
||||
|
||||
TopWorkIteratorProxy(List<HgKvIterator> iterators, long limit) {
|
||||
this.queue = new LinkedList<>(iterators);
|
||||
this.limit = limit <= 0 ? Integer.MAX_VALUE : limit;
|
||||
}
|
||||
|
||||
private HgKvIterator getIterator() {
|
||||
if (this.queue.isEmpty()) return null;
|
||||
|
||||
HgKvIterator buf = null;
|
||||
|
||||
while ((buf = this.queue.poll()) != null) {
|
||||
if (buf.hasNext()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (buf == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
this.queue.add(buf);
|
||||
|
||||
return buf;
|
||||
}
|
||||
private void closeIterators() {
|
||||
if (this.queue.isEmpty()) return;
|
||||
|
||||
HgKvIterator buf;
|
||||
|
||||
while ((buf = this.queue.poll()) != null) {
|
||||
buf.close();
|
||||
}
|
||||
|
||||
}
|
||||
private void setIterator() {
|
||||
this.iterator = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] key() {
|
||||
if (this.entry != null) {
|
||||
return this.entry.key();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] value() {
|
||||
if (this.entry != null) {
|
||||
return this.entry.value();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] position() {
|
||||
return this.iterator != null ? this.iterator.position() : EMPTY_BYTES;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void seek(byte[] position) {
|
||||
if(this.iterator!=null)this.iterator.seek(position);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
if (this.count >= this.limit) {
|
||||
return false;
|
||||
}
|
||||
if (this.iterator == null) {
|
||||
this.iterator = this.getIterator();
|
||||
}
|
||||
if (this.iterator == null) {
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object next() {
|
||||
if (this.iterator == null) {
|
||||
hasNext();
|
||||
}
|
||||
if (this.iterator == null) {
|
||||
throw new NoSuchElementException();
|
||||
}
|
||||
this.entry = this.iterator.next();
|
||||
this.setIterator();
|
||||
this.count++;
|
||||
return this.entry;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if(this.iterator!=null)this.iterator.close();
|
||||
this.closeIterators();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConfig;
|
||||
import com.baidu.hugegraph.store.term.HgPair;
|
||||
|
||||
import io.grpc.ManagedChannel;
|
||||
import io.grpc.ManagedChannelBuilder;
|
||||
import io.grpc.stub.AbstractAsyncStub;
|
||||
import io.grpc.stub.AbstractBlockingStub;
|
||||
|
||||
/**
|
||||
* @author zhangyingjie
|
||||
* @date 2023/3/28
|
||||
**/
|
||||
public abstract class AbstractGrpcClient {
|
||||
|
||||
private static Map<String, ManagedChannel[]> channels = new ConcurrentHashMap<>();
|
||||
private static int n = 5;
|
||||
private static int concurrency = 1 << n;
|
||||
private static AtomicLong counter = new AtomicLong(0);
|
||||
private static long limit = Long.MAX_VALUE >> 1;
|
||||
private Map<String, HgPair<ManagedChannel, AbstractBlockingStub>[]> blockingStubs = new ConcurrentHashMap<>();
|
||||
private Map<String, HgPair<ManagedChannel, AbstractAsyncStub>[]> asyncStubs = new ConcurrentHashMap<>();
|
||||
private static HgStoreClientConfig config = HgStoreClientConfig.of();
|
||||
|
||||
public AbstractGrpcClient() {
|
||||
|
||||
}
|
||||
|
||||
public ManagedChannel[] getChannels(String target) {
|
||||
ManagedChannel[] tc;
|
||||
if ((tc = channels.get(target)) == null) {
|
||||
synchronized (channels) {
|
||||
if ((tc = channels.get(target)) == null) {
|
||||
ManagedChannel[] value = new ManagedChannel[concurrency];
|
||||
IntStream.range(0, concurrency).parallel().forEach(i -> value[i] = getManagedChannel(target));
|
||||
channels.put(target, tc = value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return tc;
|
||||
}
|
||||
|
||||
public abstract AbstractBlockingStub getBlockingStub(ManagedChannel channel);
|
||||
|
||||
public AbstractBlockingStub getBlockingStub(String target) {
|
||||
ManagedChannel[] channels = getChannels(target);
|
||||
HgPair<ManagedChannel, AbstractBlockingStub>[] pairs = blockingStubs.get(target);
|
||||
long l = counter.getAndIncrement();
|
||||
if (l >= limit) {
|
||||
counter.set(0);
|
||||
}
|
||||
int index = (int) (l & (concurrency - 1));
|
||||
if (pairs == null) {
|
||||
synchronized (blockingStubs) {
|
||||
pairs = blockingStubs.get(target);
|
||||
if (pairs == null) {
|
||||
HgPair<ManagedChannel, AbstractBlockingStub>[] value = new HgPair[concurrency];
|
||||
IntStream.range(0, concurrency).parallel().forEach(i -> {
|
||||
ManagedChannel channel = channels[index];
|
||||
AbstractBlockingStub stub = getBlockingStub(channel);
|
||||
stub.withMaxInboundMessageSize(config.getGrpcMaxInboundMessageSize())
|
||||
.withMaxOutboundMessageSize(config.getGrpcMaxOutboundMessageSize());
|
||||
value[i] = new HgPair<>(channel, stub);
|
||||
// log.info("create channel for {}",target);
|
||||
});
|
||||
blockingStubs.put(target, value);
|
||||
AbstractBlockingStub stub = value[index].getValue();
|
||||
return (AbstractBlockingStub) stub.withDeadlineAfter(config.getGrpcTimeoutSeconds(),
|
||||
TimeUnit.SECONDS);
|
||||
}
|
||||
}
|
||||
}
|
||||
return (AbstractBlockingStub) pairs[index].getValue().withDeadlineAfter(config.getGrpcTimeoutSeconds(),
|
||||
TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
public AbstractAsyncStub getAsyncStub(ManagedChannel channel){
|
||||
return null;
|
||||
}
|
||||
|
||||
public AbstractAsyncStub getAsyncStub(String target) {
|
||||
ManagedChannel[] channels = getChannels(target);
|
||||
HgPair<ManagedChannel, AbstractAsyncStub>[] pairs = asyncStubs.get(target);
|
||||
long l = counter.getAndIncrement();
|
||||
if (l >= limit) {
|
||||
counter.set(0);
|
||||
}
|
||||
int index = (int) (l & (concurrency - 1));
|
||||
if (pairs == null) {
|
||||
synchronized (asyncStubs) {
|
||||
pairs = asyncStubs.get(target);
|
||||
if (pairs == null) {
|
||||
HgPair<ManagedChannel, AbstractAsyncStub>[] value = new HgPair[concurrency];
|
||||
IntStream.range(0, concurrency).parallel().forEach(i -> {
|
||||
ManagedChannel channel = channels[index];
|
||||
AbstractAsyncStub stub = getAsyncStub(channel);
|
||||
stub.withMaxInboundMessageSize(config.getGrpcMaxInboundMessageSize())
|
||||
.withMaxOutboundMessageSize(config.getGrpcMaxOutboundMessageSize());
|
||||
value[i] = new HgPair<>(channel, stub);
|
||||
// log.info("create channel for {}",target);
|
||||
});
|
||||
asyncStubs.put(target, value);
|
||||
AbstractAsyncStub stub = value[index].getValue();
|
||||
return stub;
|
||||
}
|
||||
}
|
||||
}
|
||||
return pairs[index].getValue();
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
private ManagedChannel getManagedChannel(String target) {
|
||||
return ManagedChannelBuilder.forTarget(target).usePlaintext().build();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import com.baidu.hugegraph.store.HgKvEntry;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com
|
||||
*/
|
||||
class GrpcKvEntryImpl implements HgKvEntry {
|
||||
private byte[] key;
|
||||
private byte[] value;
|
||||
private int code;
|
||||
|
||||
GrpcKvEntryImpl(byte[] k, byte[] v,int code) {
|
||||
this.key = k;
|
||||
this.value = v;
|
||||
this.code=code;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int code() {
|
||||
return code;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] key() {
|
||||
return key;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] value() {
|
||||
return value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
GrpcKvEntryImpl hgKvEntry = (GrpcKvEntryImpl) o;
|
||||
return Arrays.equals(key, hgKvEntry.key) && Arrays.equals(value, hgKvEntry.value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = Arrays.hashCode(key);
|
||||
result = 31 * result + Arrays.hashCode(value);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HgKvEntryImpl{" +
|
||||
"key=" + Arrays.toString(key) +
|
||||
", value=" + Arrays.toString(value) +
|
||||
", code=" + code +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,141 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toIntBytes;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.baidu.hugegraph.store.HgKvEntry;
|
||||
import com.baidu.hugegraph.store.HgKvIterator;
|
||||
import com.baidu.hugegraph.store.HgKvOrderedIterator;
|
||||
import com.baidu.hugegraph.store.HgKvPagingIterator;
|
||||
import com.baidu.hugegraph.store.HgPageSize;
|
||||
import com.baidu.hugegraph.store.HgSeekAble;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeSession;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConst;
|
||||
import com.baidu.hugegraph.store.grpc.common.Kv;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/20
|
||||
* @version 0.2.1
|
||||
*/
|
||||
@Slf4j
|
||||
class GrpcKvIteratorImpl implements HgKvPagingIterator<HgKvEntry>, HgKvOrderedIterator<HgKvEntry> {
|
||||
|
||||
private byte[] emptyBytes = HgStoreClientConst.EMPTY_BYTES;
|
||||
private KvCloseableIterator<Kv> iterator;
|
||||
private HgKvEntry element;
|
||||
private HgPageSize pageLimiter;
|
||||
private HgStoreNodeSession session;
|
||||
|
||||
public static HgKvIterator<HgKvEntry> of(HgStoreNodeSession nodeSession, KvCloseableIterator<Kv> iterator) {
|
||||
if (HgPageSize.class.isInstance(iterator)) {
|
||||
return of(nodeSession, iterator, (HgPageSize) iterator);
|
||||
}
|
||||
return new GrpcKvIteratorImpl(nodeSession, iterator, () -> 1);
|
||||
}
|
||||
|
||||
public static HgKvIterator<HgKvEntry> of(HgStoreNodeSession nodeSession, KvCloseableIterator<Kv> iterator,
|
||||
HgPageSize pageLimiter) {
|
||||
return new GrpcKvIteratorImpl(nodeSession, iterator, pageLimiter);
|
||||
}
|
||||
|
||||
public static HgKvIterator<HgKvEntry> of(HgStoreNodeSession nodeSession, List<Kv> kvList) {
|
||||
int pageSize = kvList.size();
|
||||
return new GrpcKvIteratorImpl(nodeSession, new KvListIterator<Kv>(kvList), () -> pageSize);
|
||||
}
|
||||
|
||||
private GrpcKvIteratorImpl(HgStoreNodeSession session, KvCloseableIterator<Kv> iterator, HgPageSize pageLimiter) {
|
||||
this.iterator = iterator;
|
||||
this.pageLimiter = pageLimiter;
|
||||
this.session = session;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
// if (log.isDebugEnabled()) {
|
||||
// if (!this.iterator.hasNext() && !nodeSession.getGraphName().endsWith("/s")) {
|
||||
// log.debug("[ANALYSIS GrpcKv hasNext-> FALSE] ");
|
||||
// }
|
||||
// }
|
||||
return this.iterator.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvEntry next() {
|
||||
Kv kv = this.iterator.next();
|
||||
this.element = new GrpcKvEntryImpl(kv.getKey().toByteArray(), kv.getValue().toByteArray(), kv.getCode());
|
||||
return this.element;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] key() {
|
||||
if (this.element == null) {
|
||||
return null;
|
||||
}
|
||||
return this.element.key();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] value() {
|
||||
if (this.element == null) {
|
||||
return null;
|
||||
}
|
||||
return this.element.value();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] position() {
|
||||
if (this.element == null) {
|
||||
return emptyBytes;
|
||||
}
|
||||
byte[] key = this.element.key();
|
||||
if (key == null) {
|
||||
return emptyBytes;
|
||||
}
|
||||
if (!HgSeekAble.class.isInstance(this.iterator)) {
|
||||
return emptyBytes;
|
||||
}
|
||||
byte[] upstream = ((HgSeekAble) this.iterator).position();
|
||||
byte[] code = toIntBytes(this.element.code());
|
||||
byte[] result = new byte[upstream.length + Integer.BYTES + key.length];
|
||||
System.arraycopy(upstream, 0, result, 0, upstream.length);
|
||||
System.arraycopy(code, 0, result, upstream.length, Integer.BYTES);
|
||||
System.arraycopy(key, 0, result, upstream.length + Integer.BYTES, key.length);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void seek(byte[] position) {
|
||||
if (HgSeekAble.class.isInstance(this.iterator)) {
|
||||
((HgSeekAble) this.iterator).seek(position);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getPageSize() {
|
||||
return pageLimiter.getPageSize();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isPageEmpty() {
|
||||
return !iterator.hasNext();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int compareTo(HgKvOrderedIterator o) {
|
||||
return Long.compare(this.getSequence(), o.getSequence());
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSequence() {
|
||||
return this.session.getStoreNode().getNodeId().longValue();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
this.iterator.close();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import com.baidu.hugegraph.store.grpc.HealthyGrpc;
|
||||
import com.baidu.hugegraph.store.grpc.HealthyOuterClass;
|
||||
import com.google.protobuf.Empty;
|
||||
import io.grpc.ManagedChannel;
|
||||
import io.grpc.ManagedChannelBuilder;
|
||||
|
||||
import javax.annotation.concurrent.ThreadSafe;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
@ThreadSafe
|
||||
public final class GrpcNodeHealthyClient {
|
||||
|
||||
private final static Map<String, ManagedChannel> CHANNEL_MAP = new ConcurrentHashMap<>();
|
||||
private final static Map<String, HealthyGrpc.HealthyBlockingStub> STUB_MAP = new ConcurrentHashMap<>();
|
||||
|
||||
// TODO: Forbid constructing out of the package.
|
||||
public GrpcNodeHealthyClient() {
|
||||
|
||||
}
|
||||
|
||||
private ManagedChannel getChannel(String target) {
|
||||
ManagedChannel channel = CHANNEL_MAP.get(target);
|
||||
if (channel == null) {
|
||||
channel = ManagedChannelBuilder.forTarget(target).usePlaintext().build();
|
||||
CHANNEL_MAP.put(target, channel);
|
||||
}
|
||||
return channel;
|
||||
}
|
||||
|
||||
private HealthyGrpc.HealthyBlockingStub getStub(String target) {
|
||||
HealthyGrpc.HealthyBlockingStub stub = STUB_MAP.get(target);
|
||||
if (stub == null) {
|
||||
stub = HealthyGrpc.newBlockingStub(getChannel(target));
|
||||
STUB_MAP.put(target, stub);
|
||||
}
|
||||
return stub;
|
||||
}
|
||||
|
||||
|
||||
/* boolean isHealthy(GrpcStoreNodeImpl node) {
|
||||
String target = node.getAddress();
|
||||
|
||||
HealthyOuterClass.StringReply response = getStub(target).isOk(Empty.newBuilder().build());
|
||||
String res = response.getMessage();
|
||||
|
||||
if ("ok".equals(res)) {
|
||||
return true;
|
||||
} else {
|
||||
System.out.printf("gRPC-res-msg: %s%n", res);
|
||||
return false;
|
||||
}
|
||||
}*/
|
||||
|
||||
public boolean isHealthy() {
|
||||
String target = "localhost:9080";
|
||||
ManagedChannel channel = ManagedChannelBuilder.forTarget(target).usePlaintext().build();
|
||||
HealthyGrpc.HealthyBlockingStub stub = HealthyGrpc.newBlockingStub(channel);
|
||||
HealthyOuterClass.StringReply response = stub.isOk(Empty.newBuilder().build());
|
||||
|
||||
String res = response.getMessage();
|
||||
System.out.printf("gRPC response message:%s%n", res);
|
||||
|
||||
if ("ok".equals(res)) {
|
||||
return true;
|
||||
} else {
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import com.baidu.hugegraph.store.client.HgPrivate;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNode;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeBuilder;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeManager;
|
||||
import com.baidu.hugegraph.store.client.util.HgAssert;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/12
|
||||
*/
|
||||
public class GrpcStoreNodeBuilder implements HgStoreNodeBuilder {
|
||||
|
||||
private static GrpcStoreSessionClient sessionClient = new GrpcStoreSessionClient();
|
||||
private static GrpcStoreStreamClient streamClient = new GrpcStoreStreamClient();
|
||||
private static final AtomicLong ids = new AtomicLong(0);
|
||||
|
||||
private Long nodeId;
|
||||
private String address;
|
||||
private HgStoreNodeManager nodeManager;
|
||||
|
||||
public GrpcStoreNodeBuilder(HgStoreNodeManager nodeManager, HgPrivate hgPrivate) {
|
||||
HgAssert.isArgumentNotNull(hgPrivate, "hgPrivate");
|
||||
HgAssert.isArgumentNotNull(nodeManager, "nodeManager");
|
||||
this.nodeManager = nodeManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GrpcStoreNodeBuilder setAddress(String address) {
|
||||
HgAssert.isFalse(HgAssert.isInvalid(address), "The argument is invalid: address.");
|
||||
this.address = address;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GrpcStoreNodeBuilder setNodeId(Long nodeId) {
|
||||
HgAssert.isFalse(nodeId == null, "The argument is invalid: nodeId.");
|
||||
this.nodeId = nodeId;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgStoreNode build() {
|
||||
// TODO: delete
|
||||
if (this.nodeId == null) {
|
||||
this.nodeId = ids.addAndGet(-1l);
|
||||
}
|
||||
|
||||
HgAssert.isFalse(this.nodeId == null, "nodeId can't to be null");
|
||||
HgAssert.isFalse(this.address == null, "address can't to be null");
|
||||
|
||||
GrpcStoreNodeImpl node = new GrpcStoreNodeImpl(this.nodeManager, sessionClient, streamClient);
|
||||
node.setNodeId(this.nodeId);
|
||||
node.setAddress(this.address);
|
||||
|
||||
return node;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
import com.baidu.hugegraph.store.HgStoreSession;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNode;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeManager;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/11
|
||||
*/
|
||||
class GrpcStoreNodeImpl implements HgStoreNode {
|
||||
private GrpcStoreSessionClient sessionClient;
|
||||
private GrpcStoreStreamClient streamClient;
|
||||
private HgStoreNodeManager nodeManager;
|
||||
private String address;
|
||||
private Long nodeId;
|
||||
|
||||
GrpcStoreNodeImpl(HgStoreNodeManager nodeManager, GrpcStoreSessionClient sessionClient,
|
||||
GrpcStoreStreamClient streamClient) {
|
||||
this.nodeManager = nodeManager;
|
||||
this.sessionClient = sessionClient;
|
||||
this.streamClient = streamClient;
|
||||
}
|
||||
|
||||
GrpcStoreNodeImpl setNodeId(Long nodeId) {
|
||||
this.nodeId = nodeId;
|
||||
return this;
|
||||
}
|
||||
|
||||
GrpcStoreNodeImpl setAddress(String address) {
|
||||
this.address = address;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long getNodeId() {
|
||||
return this.nodeId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getAddress() {
|
||||
return this.address;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgStoreSession openSession(String graphName) {
|
||||
// HgAssert.isFalse(HgAssert.isInvalid(graphName), "the argument: graphName is invalid.");
|
||||
// return new GrpcStoreNodeSessionImpl2(this, graphName,this.nodeManager, this.sessionClient, this
|
||||
// .streamClient);
|
||||
return new GrpcStoreNodeSessionImpl(this, graphName, this.nodeManager, this.sessionClient, this.streamClient);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
GrpcStoreNodeImpl that = (GrpcStoreNodeImpl) o;
|
||||
return Objects.equals(address, that.address) && Objects.equals(nodeId, that.nodeId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(address, nodeId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "storeNode: {" +
|
||||
"address: \"" + address + "\"" +
|
||||
", nodeId: " + nodeId +
|
||||
"}";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,493 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_LIST;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_TABLE;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.SCAN_TYPE_ANY;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.SCAN_TYPE_RANGE;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.isValid;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
|
||||
import com.baidu.hugegraph.store.HgKvEntry;
|
||||
import com.baidu.hugegraph.store.HgKvIterator;
|
||||
import com.baidu.hugegraph.store.HgKvStore;
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.HgScanQuery;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNode;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeManager;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeSession;
|
||||
import com.baidu.hugegraph.store.client.util.HgAssert;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConfig;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConst;
|
||||
import com.baidu.hugegraph.store.client.util.HgUuid;
|
||||
import com.baidu.hugegraph.store.grpc.common.GraphMethod;
|
||||
import com.baidu.hugegraph.store.grpc.common.Key;
|
||||
import com.baidu.hugegraph.store.grpc.common.OpType;
|
||||
import com.baidu.hugegraph.store.grpc.common.TableMethod;
|
||||
import com.baidu.hugegraph.store.grpc.session.BatchEntry;
|
||||
import com.baidu.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamStub;
|
||||
import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq;
|
||||
import com.google.protobuf.ByteString;
|
||||
import com.google.protobuf.UnsafeByteOperations;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2022/01/19
|
||||
* @version 0.6.0 added batch get on 2022/04/06
|
||||
*/
|
||||
@Slf4j
|
||||
@NotThreadSafe
|
||||
class GrpcStoreNodeSessionImpl implements HgStoreNodeSession {
|
||||
private static HgStoreClientConfig hgStoreClientConfig = HgStoreClientConfig.of();
|
||||
private HgStoreNode storeNode;
|
||||
private String graphName;
|
||||
private GrpcStoreSessionClient storeSessionClient;
|
||||
private GrpcStoreStreamClient storeStreamClient;
|
||||
private HgStoreNodeManager nodeManager;
|
||||
private NotifyingExecutor notifier;
|
||||
private SwitchingExecutor switcher;
|
||||
private BatchEntry.Builder batchEntryBuilder = BatchEntry.newBuilder();
|
||||
private Key.Builder builder = Key.newBuilder();
|
||||
private boolean isAutoCommit = true;
|
||||
private String batchId;
|
||||
private LinkedList<BatchEntry> batchEntries = new LinkedList<>();
|
||||
private static ConcurrentHashMap<String, Integer> tables = new ConcurrentHashMap() {{
|
||||
put("unknown", 0);
|
||||
put("vertex", 1);
|
||||
put("out_edge", 2);
|
||||
put("in_edge", 3);
|
||||
put("index", 4);
|
||||
put("task", 5);
|
||||
put("olap", 6);
|
||||
}};
|
||||
|
||||
GrpcStoreNodeSessionImpl(HgStoreNode storeNode, String graphName, HgStoreNodeManager nodeManager,
|
||||
GrpcStoreSessionClient sessionClient , GrpcStoreStreamClient streamClient) {
|
||||
|
||||
HgAssert.isFalse(HgAssert.isInvalid(graphName), "the argument: graphName is invalid.");
|
||||
HgAssert.isFalse(nodeManager == null, "the argument: nodeManager is null.");
|
||||
HgAssert.isFalse(storeNode == null, "the argument: storeNode is null.");
|
||||
HgAssert.isFalse(sessionClient == null, "the argument: sessionClient is null.");
|
||||
HgAssert.isFalse(streamClient == null, "the argument: streamClient is null.");
|
||||
|
||||
this.graphName = graphName;
|
||||
this.storeNode = storeNode;
|
||||
this.storeSessionClient = sessionClient;
|
||||
this.storeStreamClient = streamClient;
|
||||
this.nodeManager = nodeManager;
|
||||
|
||||
this.notifier = new NotifyingExecutor(this.graphName, this.nodeManager, this);
|
||||
this.switcher = SwitchingExecutor.of();
|
||||
|
||||
}
|
||||
|
||||
public String getGraphName() {
|
||||
return graphName;
|
||||
}
|
||||
|
||||
public HgStoreNode getStoreNode() {
|
||||
return storeNode;
|
||||
}
|
||||
|
||||
public Key toKey(HgOwnerKey ownerKey) {
|
||||
if (ownerKey == null) {
|
||||
return null;
|
||||
}
|
||||
return builder
|
||||
.setKey(UnsafeByteOperations.unsafeWrap(ownerKey.getKey()))
|
||||
.setCode(ownerKey.getKeyCode())
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void beginTx() {
|
||||
this.isAutoCommit = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void commit() {
|
||||
try {
|
||||
if (this.isAutoCommit) {
|
||||
throw new IllegalStateException("It's not in tx state");
|
||||
}
|
||||
if (this.batchEntries.isEmpty()) {
|
||||
this.resetTx();
|
||||
return;
|
||||
}
|
||||
if (!this.doCommit(this.batchEntries)) {
|
||||
throw new Exception("Failed to invoke doCommit");
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
throw new RuntimeException(t);
|
||||
} finally {
|
||||
this.resetTx();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void rollback() {
|
||||
if (this.isAutoCommit) throw new IllegalStateException("It's not in tx state");
|
||||
this.resetTx();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isTx() {
|
||||
return !this.isAutoCommit;
|
||||
}
|
||||
|
||||
private void resetTx() {
|
||||
this.isAutoCommit = true;
|
||||
this.batchId = null;
|
||||
this.batchEntries = new LinkedList<>();
|
||||
}
|
||||
|
||||
//TODO: not support distributed tx yet.
|
||||
private String getBatchId() {
|
||||
if (this.isAutoCommit) {
|
||||
this.batchId = HgUuid.newUUID();
|
||||
} else {
|
||||
if (this.batchId == null) {
|
||||
this.batchId = HgUuid.newUUID();
|
||||
}
|
||||
}
|
||||
return this.batchId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean put(String table, HgOwnerKey ownerKey, byte[] value) {
|
||||
return this.prepareBatchEntry(OpType.OP_TYPE_PUT, table, ownerKey, null, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean directPut(String table, int partitionId, HgOwnerKey key, byte[] value) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean delete(String table, HgOwnerKey ownerKey) {
|
||||
return this.prepareBatchEntry(OpType.OP_TYPE_DEL, table, ownerKey, null, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean deleteSingle(String table, HgOwnerKey ownerKey) {
|
||||
return this.prepareBatchEntry(OpType.OP_TYPE_DEL_SINGLE, table, ownerKey, null, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean deletePrefix(String table, HgOwnerKey prefix) {
|
||||
return this.prepareBatchEntry(OpType.OP_TYPE_DEL_PREFIX, table, prefix, null, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean deleteRange(String table, HgOwnerKey start, HgOwnerKey end) {
|
||||
return this.prepareBatchEntry(OpType.OP_TYPE_DEL_RANGE, table, start, end, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean merge(String table, HgOwnerKey key, byte[] value) {
|
||||
return this.prepareBatchEntry(OpType.OP_TYPE_MERGE, table, key, null, value);
|
||||
}
|
||||
|
||||
private boolean prepareBatchEntry(OpType opType, String table
|
||||
, HgOwnerKey startKey, HgOwnerKey endKey, byte[] value) {
|
||||
this.batchEntryBuilder.clear().setOpType(opType);
|
||||
this.batchEntryBuilder.setTable(tables.get(table));
|
||||
if (startKey != null) {
|
||||
this.batchEntryBuilder.setStartKey(toKey(startKey));
|
||||
}
|
||||
if (endKey != null) {
|
||||
this.batchEntryBuilder.setEndKey(toKey(endKey));
|
||||
}
|
||||
if (value != null) {
|
||||
this.batchEntryBuilder.setValue(ByteString.copyFrom(value));
|
||||
}
|
||||
if (this.isAutoCommit) {
|
||||
return this.doCommit(Collections.singletonList(this.batchEntryBuilder.build()));
|
||||
} else {
|
||||
return this.batchEntries.add(this.batchEntryBuilder.build());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private boolean doCommit(List<BatchEntry> entries) {
|
||||
return this.notifier.invoke(
|
||||
() -> this.storeSessionClient.doBatch(this, this.getBatchId(), entries),
|
||||
e -> true
|
||||
).orElse(false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] get(String table, HgOwnerKey ownerKey) {
|
||||
return this.notifier.invoke(
|
||||
() -> this.storeSessionClient.doGet(this, table, ownerKey)
|
||||
,
|
||||
e -> e.getValueResponse().getValue().toByteArray()
|
||||
).orElse(EMPTY_BYTES);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean clean(int partId) {
|
||||
return this.notifier.invoke(
|
||||
() -> this.storeSessionClient.doClean(this, partId)
|
||||
,
|
||||
e -> true
|
||||
|
||||
).orElse(false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HgKvEntry> batchGetOwner(String table, List<HgOwnerKey> keyList) {
|
||||
return this.notifier.invoke(
|
||||
() -> this.storeSessionClient.doBatchGet(this, table, keyList),
|
||||
e -> e.getKeyValueResponse().getKvList()
|
||||
.stream()
|
||||
.map(kv -> (HgKvEntry) new GrpcKvEntryImpl(kv.getKey().toByteArray()
|
||||
, kv.getValue().toByteArray(), kv.getCode())
|
||||
)
|
||||
.collect(Collectors.toList()))
|
||||
.orElse((List<HgKvEntry>) EMPTY_LIST);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> batchPrefix(String table, List<HgOwnerKey> keyList) {
|
||||
return GrpcKvIteratorImpl.of(this,
|
||||
this.storeStreamClient.doBatchScanOneShot(this,
|
||||
HgScanQuery.prefixOf(table,
|
||||
keyList))
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean existsTable(String table) {
|
||||
return this.notifier.invoke(
|
||||
() -> this.storeSessionClient.doTable(this, table, TableMethod.TABLE_METHOD_EXISTS),
|
||||
e -> true)
|
||||
.orElse(false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean createTable(String table) {
|
||||
return this.notifier.invoke(
|
||||
() -> this.storeSessionClient.doTable(this, table, TableMethod.TABLE_METHOD_CREATE),
|
||||
e -> true)
|
||||
.orElse(false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean deleteTable(String table) {
|
||||
return this.notifier.invoke(
|
||||
() -> this.storeSessionClient.doTable(this, table, TableMethod.TABLE_METHOD_DELETE),
|
||||
e -> true)
|
||||
.orElse(false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean dropTable(String table) {
|
||||
return this.notifier.invoke(
|
||||
() -> this.storeSessionClient.doTable(this, table, TableMethod.TABLE_METHOD_DROP),
|
||||
e -> true)
|
||||
.orElse(false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean deleteGraph(String graph) {
|
||||
return this.notifier.invoke(
|
||||
() -> this.storeSessionClient.doGraph(this, graph, GraphMethod.GRAPH_METHOD_DELETE),
|
||||
e -> true)
|
||||
.orElse(false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean truncate() {
|
||||
return this.notifier.invoke(
|
||||
() -> this.storeSessionClient.doTable(this, EMPTY_TABLE
|
||||
, TableMethod.TABLE_METHOD_TRUNCATE),
|
||||
e -> true)
|
||||
.orElse(false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table) {
|
||||
return GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScan(this, table, 0));
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, long limit) {
|
||||
return this.switcher.invoke(getSwitcherSupplier(limit)
|
||||
, () -> GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScan
|
||||
(this, table, limit))
|
||||
, () -> GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScanOneShot
|
||||
(this, table, limit))
|
||||
).get();
|
||||
}
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(ScanStreamReq.Builder builder) {
|
||||
HgStoreStreamStub stub = getStub();
|
||||
KvPageScanner scanner = new KvPageScanner(this,
|
||||
stub,
|
||||
builder);
|
||||
return GrpcKvIteratorImpl.of(this, scanner);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, byte[] query) {
|
||||
return GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScan(this, table, 0, query));
|
||||
}
|
||||
|
||||
private HgStoreStreamStub getStub() {
|
||||
return this.storeStreamClient.getStub(this);
|
||||
}
|
||||
|
||||
// @Override
|
||||
// public HgKvIterator<HgKvEntry> scanIterator(ScanStreamReq scanReq) {
|
||||
// KvPageScanner6 scanner = new KvPageScanner6(this,
|
||||
// getStub(),
|
||||
// scanReq.toBuilder());
|
||||
// return GrpcKvIteratorImpl.of(this, scanner);
|
||||
// }
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, long limit, byte[] query) {
|
||||
return this.switcher.invoke(getSwitcherSupplier(limit)
|
||||
, () -> GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScan
|
||||
(this, table, limit, query))
|
||||
, () -> GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScanOneShot
|
||||
(this, table, limit, query))
|
||||
).get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey keyPrefix) {
|
||||
return GrpcKvIteratorImpl.of(this, this.storeStreamClient.doScan(this, table, keyPrefix, 0));
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey keyPrefix, long limit) {
|
||||
return this.switcher.invoke(getSwitcherSupplier(limit),
|
||||
() -> GrpcKvIteratorImpl.of(this,
|
||||
this.storeStreamClient.doScan(this,
|
||||
table,
|
||||
keyPrefix,
|
||||
limit)),
|
||||
() -> GrpcKvIteratorImpl.of(this,
|
||||
this.storeStreamClient.doScanOneShot(this,
|
||||
table,
|
||||
keyPrefix,
|
||||
limit)))
|
||||
.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey keyPrefix, long limit, byte[] query) {
|
||||
return this.switcher.invoke(getSwitcherSupplier(limit),
|
||||
() -> GrpcKvIteratorImpl.of(this,
|
||||
this.storeStreamClient.doScan(
|
||||
this,
|
||||
table,
|
||||
keyPrefix,
|
||||
limit,
|
||||
query)),
|
||||
() -> GrpcKvIteratorImpl.of(this,
|
||||
this.storeStreamClient.doScanOneShot(this,
|
||||
table,
|
||||
keyPrefix,
|
||||
limit,
|
||||
query)))
|
||||
.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey) {
|
||||
return scanIterator(table, startKey, endKey, 0, HgKvStore.SCAN_ANY, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit) {
|
||||
return scanIterator(table, startKey, endKey, limit, isValid(endKey) ? SCAN_TYPE_RANGE : SCAN_TYPE_ANY, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit, byte[] query) {
|
||||
return scanIterator(table, startKey, endKey, limit, isValid(endKey) ? SCAN_TYPE_RANGE : SCAN_TYPE_ANY, query);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, HgOwnerKey startKey, HgOwnerKey endKey,
|
||||
long limit, int scanType, byte[] query) {
|
||||
|
||||
return this.switcher.invoke(getSwitcherSupplier(limit),
|
||||
() -> GrpcKvIteratorImpl.of(this,
|
||||
this.storeStreamClient.doScan(
|
||||
this,
|
||||
table,
|
||||
startKey,
|
||||
endKey,
|
||||
limit,
|
||||
scanType,
|
||||
query)),
|
||||
() -> GrpcKvIteratorImpl.of(this,
|
||||
this.storeStreamClient.doScanOneShot(this,
|
||||
table,
|
||||
startKey,
|
||||
endKey,
|
||||
limit,
|
||||
scanType,
|
||||
query)))
|
||||
.get();
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> scanIterator(String table, int codeFrom, int codeTo, int scanType, byte[] query) {
|
||||
//TODO: Should be changed when start using hashcode as partitionId.
|
||||
if(log.isDebugEnabled())log.debug("scanIterator-scanType: {}", scanType);
|
||||
return GrpcKvIteratorImpl.of(this,
|
||||
this.storeStreamClient.doScan(this, table
|
||||
, HgOwnerKey.newEmpty().codeToKey(codeFrom)
|
||||
, HgOwnerKey.newEmpty().codeToKey(codeTo)
|
||||
, HgStoreClientConst.NO_LIMIT
|
||||
, SCAN_PREFIX_BEGIN | SCAN_HASHCODE | scanType
|
||||
, query
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<HgKvIterator<HgKvEntry>> scanBatch(HgScanQuery scanQuery) {
|
||||
return Collections.singletonList(GrpcKvIteratorImpl.of(this,
|
||||
this.storeStreamClient.doBatchScan(this, scanQuery)
|
||||
));
|
||||
}
|
||||
|
||||
@Override
|
||||
public KvCloseableIterator<HgKvIterator<HgKvEntry>> scanBatch2(HgScanQuery scanQuery) {
|
||||
throw new RuntimeException("not implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public KvCloseableIterator<HgKvIterator<HgKvEntry>> scanBatch3(HgScanQuery scanQuery, KvCloseableIterator iterator) {
|
||||
return this.storeStreamClient.doBatchScan3(this, scanQuery, iterator);
|
||||
}
|
||||
|
||||
private Supplier<Boolean> getSwitcherSupplier(long limit) {
|
||||
return () -> limit <= 0 || limit > hgStoreClientConfig.getNetKvScannerPageSize();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "storeNodeSession: {" + storeNode + ", graphName: \"" +
|
||||
graphName + "\"}";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import static com.baidu.hugegraph.store.client.grpc.GrpcUtil.getHeader;
|
||||
import static com.baidu.hugegraph.store.client.grpc.GrpcUtil.toKey;
|
||||
import static com.baidu.hugegraph.store.client.grpc.GrpcUtil.toTk;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import javax.annotation.concurrent.ThreadSafe;
|
||||
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeSession;
|
||||
import com.baidu.hugegraph.store.grpc.common.GraphMethod;
|
||||
import com.baidu.hugegraph.store.grpc.common.TableMethod;
|
||||
import com.baidu.hugegraph.store.grpc.session.BatchEntry;
|
||||
import com.baidu.hugegraph.store.grpc.session.BatchGetReq;
|
||||
import com.baidu.hugegraph.store.grpc.session.BatchReq;
|
||||
import com.baidu.hugegraph.store.grpc.session.BatchWriteReq;
|
||||
import com.baidu.hugegraph.store.grpc.session.CleanReq;
|
||||
import com.baidu.hugegraph.store.grpc.session.FeedbackRes;
|
||||
import com.baidu.hugegraph.store.grpc.session.GetReq;
|
||||
import com.baidu.hugegraph.store.grpc.session.GraphReq;
|
||||
import com.baidu.hugegraph.store.grpc.session.HgStoreSessionGrpc;
|
||||
import com.baidu.hugegraph.store.grpc.session.HgStoreSessionGrpc.HgStoreSessionBlockingStub;
|
||||
import com.baidu.hugegraph.store.grpc.session.TableReq;
|
||||
|
||||
import io.grpc.ManagedChannel;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/11/18
|
||||
* @version 0.5.0
|
||||
*/
|
||||
@Slf4j
|
||||
@ThreadSafe
|
||||
class GrpcStoreSessionClient extends AbstractGrpcClient {
|
||||
|
||||
@Override
|
||||
public HgStoreSessionBlockingStub getBlockingStub(ManagedChannel channel) {
|
||||
HgStoreSessionBlockingStub stub;
|
||||
stub = HgStoreSessionGrpc.newBlockingStub(channel);
|
||||
return stub;
|
||||
}
|
||||
|
||||
private HgStoreSessionBlockingStub getBlockingStub(HgStoreNodeSession nodeSession) {
|
||||
HgStoreSessionBlockingStub stub =
|
||||
(HgStoreSessionBlockingStub) getBlockingStub(nodeSession.getStoreNode().getAddress());
|
||||
return stub;
|
||||
}
|
||||
|
||||
FeedbackRes doGet(HgStoreNodeSession nodeSession, String table, HgOwnerKey ownerKey) {
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("doGet: {}-{}-{}-{}", nodeSession, table, ownerKey, GetReq.newBuilder()
|
||||
.setHeader(getHeader(nodeSession))
|
||||
.setTk(toTk(table, ownerKey))
|
||||
.build());
|
||||
}
|
||||
return this.getBlockingStub(nodeSession)
|
||||
.get2(GetReq.newBuilder()
|
||||
.setHeader(getHeader(nodeSession))
|
||||
.setTk(toTk(table, ownerKey))
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
FeedbackRes doClean(HgStoreNodeSession nodeSession, int partId) {
|
||||
return this.getBlockingStub(nodeSession)
|
||||
.clean(CleanReq.newBuilder()
|
||||
.setHeader(getHeader(nodeSession))
|
||||
.setPartition(partId)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
FeedbackRes doBatchGet(HgStoreNodeSession nodeSession, String table, List<HgOwnerKey> keyList) {
|
||||
BatchGetReq.Builder builder = BatchGetReq.newBuilder();
|
||||
builder.setHeader(getHeader(nodeSession)).setTable(table);
|
||||
|
||||
for (HgOwnerKey key : keyList) {
|
||||
builder.addKey(toKey(key));
|
||||
}
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("batchGet2: {}-{}-{}-{}", nodeSession, table, keyList, builder.build());
|
||||
}
|
||||
return this.getBlockingStub(nodeSession).batchGet2(builder.build());
|
||||
|
||||
}
|
||||
|
||||
FeedbackRes doBatch(HgStoreNodeSession nodeSession, String batchId, List<BatchEntry> entries) {
|
||||
BatchWriteReq.Builder writeReq = BatchWriteReq.newBuilder();
|
||||
writeReq.addAllEntry(entries);
|
||||
return this.getBlockingStub(nodeSession)
|
||||
.batch(BatchReq.newBuilder()
|
||||
.setHeader(getHeader(nodeSession))
|
||||
.setWriteReq(writeReq)
|
||||
.setBatchId(batchId)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
FeedbackRes doTable(HgStoreNodeSession nodeSession, String table, TableMethod method) {
|
||||
return this.getBlockingStub(nodeSession)
|
||||
.table(TableReq.newBuilder()
|
||||
.setHeader(getHeader(nodeSession))
|
||||
.setTableName(table)
|
||||
.setMethod(method)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
FeedbackRes doGraph(HgStoreNodeSession nodeSession, String graph, GraphMethod method) {
|
||||
return this.getBlockingStub(nodeSession)
|
||||
.graph(GraphReq.newBuilder()
|
||||
.setHeader(getHeader(nodeSession))
|
||||
.setGraphName(graph)
|
||||
.setMethod(method)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.annotation.concurrent.ThreadSafe;
|
||||
|
||||
import com.baidu.hugegraph.pd.client.PDClient;
|
||||
import com.baidu.hugegraph.pd.client.PDConfig;
|
||||
import com.baidu.hugegraph.pd.grpc.Metapb;
|
||||
import com.baidu.hugegraph.store.grpc.state.HgStoreStateGrpc;
|
||||
import com.baidu.hugegraph.store.grpc.state.HgStoreStateGrpc.HgStoreStateBlockingStub;
|
||||
import com.baidu.hugegraph.store.grpc.state.ScanState;
|
||||
import com.baidu.hugegraph.store.grpc.state.SubStateReq;
|
||||
|
||||
import io.grpc.ManagedChannel;
|
||||
import io.grpc.stub.AbstractBlockingStub;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
@Slf4j
|
||||
@ThreadSafe
|
||||
public class GrpcStoreStateClient extends AbstractGrpcClient {
|
||||
|
||||
private final PDConfig pdConfig;
|
||||
private final PDClient pdClient;
|
||||
|
||||
public GrpcStoreStateClient(PDConfig pdConfig) {
|
||||
this.pdConfig = pdConfig;
|
||||
pdClient = PDClient.create(this.pdConfig);
|
||||
}
|
||||
|
||||
|
||||
public Set<ScanState> getScanState() throws Exception {
|
||||
try {
|
||||
List<Metapb.Store> activeStores = pdClient.getActiveStores();
|
||||
Set<ScanState> states = activeStores.parallelStream().map(node -> {
|
||||
String address = node.getAddress();
|
||||
HgStoreStateBlockingStub stub = (HgStoreStateBlockingStub) getBlockingStub(address);
|
||||
SubStateReq req = SubStateReq.newBuilder().build();
|
||||
return stub.getScanState(req);
|
||||
}).collect(Collectors.toSet());
|
||||
return states;
|
||||
} catch (Exception e) {
|
||||
throw e;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public AbstractBlockingStub getBlockingStub(ManagedChannel channel) {
|
||||
HgStoreStateBlockingStub stub;
|
||||
stub = HgStoreStateGrpc.newBlockingStub(channel);
|
||||
return stub;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,192 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import javax.annotation.concurrent.ThreadSafe;
|
||||
|
||||
import com.baidu.hugegraph.store.HgKvEntry;
|
||||
import com.baidu.hugegraph.store.HgKvIterator;
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.HgScanQuery;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeSession;
|
||||
import com.baidu.hugegraph.store.grpc.common.Kv;
|
||||
import com.baidu.hugegraph.store.grpc.stream.HgStoreStreamGrpc;
|
||||
import com.baidu.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamBlockingStub;
|
||||
import com.baidu.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamStub;
|
||||
|
||||
import io.grpc.ManagedChannel;
|
||||
import io.grpc.stub.AbstractAsyncStub;
|
||||
import io.grpc.stub.AbstractBlockingStub;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/19
|
||||
* @version 1.1.1 added synchronized in getChannel.
|
||||
*/
|
||||
@Slf4j
|
||||
@ThreadSafe
|
||||
public class GrpcStoreStreamClient extends AbstractGrpcClient {
|
||||
|
||||
public HgStoreStreamStub getStub(HgStoreNodeSession nodeSession) {
|
||||
return (HgStoreStreamStub) getAsyncStub(nodeSession.getStoreNode().getAddress());
|
||||
}
|
||||
|
||||
@Override
|
||||
public AbstractAsyncStub getAsyncStub(ManagedChannel channel){
|
||||
return HgStoreStreamGrpc.newStub(channel);
|
||||
}
|
||||
|
||||
private HgStoreStreamBlockingStub getBlockingStub(HgStoreNodeSession nodeSession) {
|
||||
return (HgStoreStreamBlockingStub) getBlockingStub(nodeSession.getStoreNode().getAddress());
|
||||
}
|
||||
|
||||
@Override
|
||||
public AbstractBlockingStub getBlockingStub(ManagedChannel channel) {
|
||||
return HgStoreStreamGrpc.newBlockingStub(channel);
|
||||
}
|
||||
|
||||
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table, long limit, byte[] query) {
|
||||
return KvOneShotScanner.scanAll(nodeSession
|
||||
, this.getBlockingStub(nodeSession)
|
||||
, table
|
||||
, limit
|
||||
, query
|
||||
);
|
||||
}
|
||||
|
||||
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table, long limit) {
|
||||
return KvOneShotScanner.scanAll(nodeSession
|
||||
, this.getBlockingStub(nodeSession)
|
||||
, table
|
||||
, limit
|
||||
, null
|
||||
);
|
||||
}
|
||||
|
||||
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table, HgOwnerKey prefix, long limit) {
|
||||
return KvOneShotScanner.scanPrefix(nodeSession
|
||||
, this.getBlockingStub(nodeSession)
|
||||
, table
|
||||
, prefix
|
||||
, limit
|
||||
, null
|
||||
);
|
||||
}
|
||||
|
||||
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table, HgOwnerKey prefix, long limit,
|
||||
byte[] query) {
|
||||
return KvOneShotScanner.scanPrefix(nodeSession
|
||||
, this.getBlockingStub(nodeSession)
|
||||
, table
|
||||
, prefix
|
||||
, limit
|
||||
, query
|
||||
);
|
||||
}
|
||||
|
||||
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table, HgOwnerKey startKey,
|
||||
HgOwnerKey endKey
|
||||
, long limit
|
||||
, int scanType
|
||||
, byte[] query) {
|
||||
|
||||
return KvOneShotScanner.scanRange(nodeSession
|
||||
, this.getBlockingStub(nodeSession)
|
||||
, table
|
||||
, startKey
|
||||
, endKey
|
||||
, limit
|
||||
, scanType
|
||||
, query
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
|
||||
, String table
|
||||
, long limit
|
||||
, byte[] query) {
|
||||
|
||||
return KvPageScanner.scanAll(nodeSession
|
||||
, this.getStub(nodeSession)
|
||||
, table
|
||||
, limit
|
||||
, query
|
||||
);
|
||||
}
|
||||
|
||||
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
|
||||
, String table
|
||||
, long limit) {
|
||||
|
||||
return KvPageScanner.scanAll(nodeSession
|
||||
, this.getStub(nodeSession)
|
||||
, table
|
||||
, limit
|
||||
, null
|
||||
);
|
||||
}
|
||||
|
||||
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
|
||||
, String table
|
||||
, HgOwnerKey prefix
|
||||
, long limit) {
|
||||
|
||||
return KvPageScanner.scanPrefix(nodeSession
|
||||
, this.getStub(nodeSession)
|
||||
, table
|
||||
, prefix
|
||||
, limit
|
||||
, null
|
||||
);
|
||||
}
|
||||
|
||||
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
|
||||
, String table
|
||||
, HgOwnerKey prefix
|
||||
, long limit
|
||||
, byte[] query) {
|
||||
|
||||
return KvPageScanner.scanPrefix(nodeSession
|
||||
, this.getStub(nodeSession)
|
||||
, table
|
||||
, prefix
|
||||
, limit
|
||||
, query
|
||||
);
|
||||
}
|
||||
|
||||
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
|
||||
, String table
|
||||
, HgOwnerKey startKey
|
||||
, HgOwnerKey endKey
|
||||
, long limit
|
||||
, int scanType
|
||||
, byte[] query) {
|
||||
|
||||
return KvPageScanner.scanRange(nodeSession
|
||||
, this.getStub(nodeSession)
|
||||
, table
|
||||
, startKey
|
||||
, endKey
|
||||
, limit
|
||||
, scanType
|
||||
, query
|
||||
);
|
||||
}
|
||||
|
||||
KvCloseableIterator<Kv> doBatchScan(HgStoreNodeSession nodeSession, HgScanQuery scanQuery) {
|
||||
return KvBatchScanner5.scan(nodeSession, this.getStub(nodeSession), scanQuery);
|
||||
}
|
||||
|
||||
|
||||
// 返回多个小的迭代器，允许上层并行处理
|
||||
KvCloseableIterator<HgKvIterator<HgKvEntry>> doBatchScan3(HgStoreNodeSession nodeSession,
|
||||
HgScanQuery scanQuery, KvCloseableIterator iterator) {
|
||||
KvBatchScanner.scan(this.getStub(nodeSession), nodeSession.getGraphName(), scanQuery, iterator);
|
||||
return iterator;
|
||||
}
|
||||
|
||||
KvCloseableIterator<Kv> doBatchScanOneShot(HgStoreNodeSession nodeSession, HgScanQuery scanQuery) {
|
||||
return KvBatchOneShotScanner.scan(nodeSession, this.getBlockingStub(nodeSession), scanQuery);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import com.baidu.hugegraph.store.HgKvEntry;
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeSession;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConst;
|
||||
import com.baidu.hugegraph.store.grpc.common.Header;
|
||||
import com.baidu.hugegraph.store.grpc.common.Key;
|
||||
import com.baidu.hugegraph.store.grpc.common.Kv;
|
||||
import com.baidu.hugegraph.store.grpc.common.Tk;
|
||||
import com.baidu.hugegraph.store.grpc.common.Tkv;
|
||||
import com.baidu.hugegraph.store.grpc.common.Tp;
|
||||
import com.baidu.hugegraph.store.grpc.common.Tse;
|
||||
import com.google.protobuf.ByteString;
|
||||
|
||||
import io.grpc.Status;
|
||||
import io.grpc.StatusRuntimeException;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2022/1/19
|
||||
*/
|
||||
final class GrpcUtil {
|
||||
|
||||
private static ThreadLocal<Key.Builder> keyBuilder = new ThreadLocal<Key.Builder>();
|
||||
|
||||
static Header getHeader(HgStoreNodeSession nodeSession) {
|
||||
return Header.newBuilder()
|
||||
.setGraph(nodeSession.getGraphName())
|
||||
.build();
|
||||
}
|
||||
|
||||
static Tk toTk(String table, HgOwnerKey ownerKey) {
|
||||
return Tk.newBuilder()
|
||||
.setTable(table)
|
||||
.setKey(ByteString.copyFrom(ownerKey.getKey()))
|
||||
.setCode(ownerKey.getKeyCode())
|
||||
.build();
|
||||
}
|
||||
|
||||
static Key.Builder getOwnerKeyBuilder() {
|
||||
Key.Builder builder = keyBuilder.get();
|
||||
if (builder == null) {
|
||||
builder = Key.newBuilder();
|
||||
// TODO 线程级变量，寻找删除时机
|
||||
keyBuilder.set(builder);
|
||||
}
|
||||
return builder;
|
||||
}
|
||||
|
||||
static Key toKey(HgOwnerKey ownerKey, Key.Builder builder) {
|
||||
if (ownerKey == null) {
|
||||
return null;
|
||||
}
|
||||
return builder
|
||||
.setKey(ByteString.copyFrom(ownerKey.getKey()))
|
||||
.setCode(ownerKey.getKeyCode())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
static Key toKey(HgOwnerKey ownerKey) {
|
||||
if (ownerKey == null) {
|
||||
return null;
|
||||
}
|
||||
Key.Builder builder = keyBuilder.get();
|
||||
if (builder == null) {
|
||||
builder = Key.newBuilder();
|
||||
// TODO 线程级变量，寻找删除时机
|
||||
keyBuilder.set(builder);
|
||||
}
|
||||
return builder
|
||||
.setKey(ByteString.copyFrom(ownerKey.getKey()))
|
||||
.setCode(ownerKey.getKeyCode())
|
||||
.build();
|
||||
}
|
||||
|
||||
static Tkv toTkv(String table, HgOwnerKey ownerKey, byte[] value) {
|
||||
return Tkv.newBuilder()
|
||||
.setTable(table)
|
||||
.setKey(ByteString.copyFrom(ownerKey.getKey()))
|
||||
.setValue(ByteString.copyFrom(value))
|
||||
.setCode(ownerKey.getKeyCode())
|
||||
.build();
|
||||
}
|
||||
|
||||
static Tp toTp(String table, HgOwnerKey ownerKey) {
|
||||
return Tp.newBuilder()
|
||||
.setTable(table)
|
||||
.setPrefix(ByteString.copyFrom(ownerKey.getKey()))
|
||||
.setCode(ownerKey.getKeyCode())
|
||||
.build();
|
||||
}
|
||||
|
||||
static Tse toTse(String table, HgOwnerKey startKey, HgOwnerKey endKey) {
|
||||
return Tse.newBuilder()
|
||||
.setTable(table)
|
||||
.setStart(toKey(startKey))
|
||||
.setEnd(toKey(endKey))
|
||||
.build();
|
||||
|
||||
}
|
||||
|
||||
static List<HgKvEntry> toList(List<Kv> kvList) {
|
||||
if (kvList == null || kvList.isEmpty()) {
|
||||
return HgStoreClientConst.EMPTY_LIST;
|
||||
}
|
||||
|
||||
Iterator<Kv> iter = kvList.iterator();
|
||||
List<HgKvEntry> resList = new ArrayList<>(kvList.size());
|
||||
|
||||
while (iter.hasNext()) {
|
||||
Kv entry = iter.next();
|
||||
resList.add(new GrpcKvEntryImpl(entry.getKey().toByteArray(), entry.getValue().toByteArray(),entry.getCode()));
|
||||
}
|
||||
|
||||
return resList;
|
||||
}
|
||||
|
||||
static StatusRuntimeException toErr(String msg){
|
||||
return new StatusRuntimeException(Status.UNKNOWN.withDescription(msg));
|
||||
}
|
||||
|
||||
static ByteString toBs(byte[] bytes){
|
||||
return ByteString.copyFrom((bytes != null) ? bytes : EMPTY_BYTES);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import com.baidu.hugegraph.store.HgPageSize;
|
||||
import com.baidu.hugegraph.store.HgScanQuery;
|
||||
import com.baidu.hugegraph.store.HgSeekAble;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeSession;
|
||||
import com.baidu.hugegraph.store.grpc.common.Kv;
|
||||
import com.baidu.hugegraph.store.grpc.stream.HgStoreStreamGrpc;
|
||||
import com.baidu.hugegraph.store.grpc.stream.ScanStreamBatchReq;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import static com.baidu.hugegraph.store.client.grpc.KvBatchUtil.*;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2022/04/08
|
||||
*/
|
||||
@Slf4j
|
||||
@NotThreadSafe
|
||||
class KvBatchOneShotScanner implements KvCloseableIterator<Kv>, HgPageSize, HgSeekAble {
|
||||
private final HgStoreNodeSession nodeSession;
|
||||
private final HgStoreStreamGrpc.HgStoreStreamBlockingStub stub;
|
||||
private final HgScanQuery scanQuery;
|
||||
|
||||
private Iterator<Kv> iterator;
|
||||
private List<Kv> list = null;
|
||||
|
||||
public static KvCloseableIterator scan(HgStoreNodeSession nodeSession,
|
||||
HgStoreStreamGrpc.HgStoreStreamBlockingStub stub,
|
||||
HgScanQuery scanQuery) {
|
||||
|
||||
return new KvBatchOneShotScanner(nodeSession, stub, scanQuery);
|
||||
}
|
||||
|
||||
private KvBatchOneShotScanner(HgStoreNodeSession nodeSession,
|
||||
HgStoreStreamGrpc.HgStoreStreamBlockingStub stub,
|
||||
HgScanQuery scanQuery) {
|
||||
|
||||
this.nodeSession = nodeSession;
|
||||
this.stub = stub;
|
||||
this.scanQuery = scanQuery;
|
||||
}
|
||||
|
||||
private ScanStreamBatchReq createReq() {
|
||||
return ScanStreamBatchReq.newBuilder()
|
||||
.setHeader(getHeader(this.nodeSession))
|
||||
.setQueryRequest(createQueryReq(this.scanQuery, Integer.MAX_VALUE))
|
||||
.build();
|
||||
}
|
||||
|
||||
private Iterator<Kv> createIterator() {
|
||||
this.list = this.stub.scanBatchOneShot(this.createReq()).getDataList();
|
||||
return this.list.iterator();
|
||||
}
|
||||
|
||||
/*** Iterator ***/
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
if (this.iterator == null) {
|
||||
this.iterator =this.createIterator();
|
||||
}
|
||||
return this.iterator.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Kv next() {
|
||||
if (this.iterator == null) {
|
||||
this.iterator =this.createIterator();
|
||||
}
|
||||
return this.iterator.next();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getPageSize() {
|
||||
return Integer.MAX_VALUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isPageEmpty() {
|
||||
return !this.iterator.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] position() {
|
||||
//TODO: to implement
|
||||
return EMPTY_POSITION;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void seek(byte[] position) {
|
||||
//TODO: to implement
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
//Nothing to do
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,378 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import static com.baidu.hugegraph.store.client.grpc.KvBatchUtil.createQueryReq;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.function.BiFunction;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
|
||||
import com.baidu.hugegraph.store.HgKvEntry;
|
||||
import com.baidu.hugegraph.store.HgKvIterator;
|
||||
import com.baidu.hugegraph.store.HgKvOrderedIterator;
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.HgScanQuery;
|
||||
import com.baidu.hugegraph.store.buffer.KVByteBuffer;
|
||||
import com.baidu.hugegraph.store.client.util.PropertyUtil;
|
||||
import com.baidu.hugegraph.store.grpc.common.Header;
|
||||
import com.baidu.hugegraph.store.grpc.common.ScanOrderType;
|
||||
import com.baidu.hugegraph.store.grpc.stream.HgStoreStreamGrpc;
|
||||
import com.baidu.hugegraph.store.grpc.stream.KvStream;
|
||||
import com.baidu.hugegraph.store.grpc.stream.ScanReceiptRequest;
|
||||
import com.baidu.hugegraph.store.grpc.stream.ScanStreamBatchReq;
|
||||
|
||||
import io.grpc.stub.StreamObserver;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* 批量流式查询客户端实现类
|
||||
*
|
||||
* @author YanJinbing created on 2022/07/23
|
||||
* @version 3.0.0
|
||||
*/
|
||||
@Slf4j
|
||||
@NotThreadSafe
|
||||
public class KvBatchScanner implements Closeable {
|
||||
|
||||
static int maxTaskSizePerStore = PropertyUtil.getInt("net.kv.scanner.task.size", 8);
|
||||
volatile int currentSeqNo = 0;
|
||||
private final StreamObserver<ScanStreamBatchReq> sender; // 命令发送器
|
||||
private final KvBatchScannerMerger notifier; // 数据通知
|
||||
private final String graphName; // 图名
|
||||
private volatile boolean running;
|
||||
private final HgScanQuery scanQuery;
|
||||
static final Supplier<HgKvIterator<HgKvEntry>> NO_DATA = () -> null;
|
||||
private final ScanReceiptRequest.Builder responseBuilder = ScanReceiptRequest.newBuilder();
|
||||
private KvBatchReceiver receiver;
|
||||
|
||||
public KvBatchScanner(
|
||||
HgStoreStreamGrpc.HgStoreStreamStub stub,
|
||||
String graphName,
|
||||
HgScanQuery scanQuery,
|
||||
KvCloseableIterator iterator) {
|
||||
|
||||
this.graphName = graphName;
|
||||
this.notifier = (KvBatchScannerMerger) iterator;
|
||||
this.notifier.registerScanner(this);
|
||||
this.running = true;
|
||||
this.scanQuery = scanQuery;
|
||||
receiver = new KvBatchReceiver(this, scanQuery.getOrderType() == ScanOrderType.ORDER_STRICT);
|
||||
sender = stub.scanBatch2(receiver);
|
||||
sendQuery(this.scanQuery); // 发送查询请求
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建流式查询迭代器
|
||||
* scanQuery进行拆分，启动多个流式请求，提升store的并发性
|
||||
*
|
||||
* @param scanQuery scanQuery
|
||||
* @param handler task handler
|
||||
* @return data merger iterator
|
||||
*/
|
||||
public static KvCloseableIterator ofMerger(
|
||||
HgScanQuery scanQuery, BiFunction<HgScanQuery, KvCloseableIterator, Boolean> handler) {
|
||||
KvBatchScannerMerger merger;
|
||||
if (scanQuery.getOrderType() == ScanOrderType.ORDER_STRICT) {
|
||||
merger = new KvBatchScannerMerger.SortedScannerMerger(new TaskSplitter(scanQuery, handler));
|
||||
} else {
|
||||
merger = new KvBatchScannerMerger(new TaskSplitter(scanQuery, handler));
|
||||
}
|
||||
merger.startTask();
|
||||
return merger;
|
||||
}
|
||||
|
||||
public static void scan(
|
||||
HgStoreStreamGrpc.HgStoreStreamStub stub,
|
||||
String graphName,
|
||||
HgScanQuery scanQuery,
|
||||
KvCloseableIterator iterator) {
|
||||
new KvBatchScanner(stub, graphName, scanQuery, iterator);
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送查询请求
|
||||
*
|
||||
* @param query scan query
|
||||
*/
|
||||
public void sendQuery(HgScanQuery query) {
|
||||
synchronized (this.sender) {
|
||||
if (running) {
|
||||
this.sender.onNext(
|
||||
ScanStreamBatchReq.newBuilder()
|
||||
.setHeader(Header.newBuilder().setGraph(graphName).build())
|
||||
.setQueryRequest(createQueryReq(query, 0))
|
||||
.build());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送应答
|
||||
*/
|
||||
public void sendResponse() {
|
||||
try {
|
||||
sendResponse(currentSeqNo);
|
||||
} catch (Exception e) {
|
||||
log.error("exception", e);
|
||||
}
|
||||
}
|
||||
|
||||
public void sendResponse(int seqNo) {
|
||||
currentSeqNo = seqNo;
|
||||
synchronized (this.sender) {
|
||||
if (running) {
|
||||
this.sender.onNext(
|
||||
ScanStreamBatchReq.newBuilder()
|
||||
.setHeader(Header.newBuilder().setGraph(graphName).build())
|
||||
.setReceiptRequest(responseBuilder.setTimes(seqNo).build())
|
||||
.build());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void dataArrived(Supplier<HgKvIterator<HgKvEntry>> supplier) throws InterruptedException {
|
||||
notifier.dataArrived(this, supplier);
|
||||
}
|
||||
|
||||
/**
|
||||
* 数据接收结束
|
||||
*/
|
||||
public void dataComplete() {
|
||||
close();
|
||||
}
|
||||
|
||||
// 流被关闭
|
||||
@Override
|
||||
public void close() {
|
||||
try {
|
||||
if (notifier.unregisterScanner(this) < 0) {
|
||||
notifier.dataArrived(this, NO_DATA); // 任务结束，唤醒队列
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
log.error("exception ", e);
|
||||
}
|
||||
synchronized (this.sender) {
|
||||
try {
|
||||
if (running) {
|
||||
sender.onCompleted();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
}
|
||||
running = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 任务拆分器
|
||||
*/
|
||||
static class TaskSplitter {
|
||||
final HgScanQuery scanQuery;
|
||||
final BiFunction<HgScanQuery, KvCloseableIterator, Boolean> taskHandler;
|
||||
private KvBatchScannerMerger notifier;
|
||||
private Iterator<HgOwnerKey> prefixItr;
|
||||
private int maxTaskSize = 0; // 最大并行任务数
|
||||
private int maxBatchSize = PropertyUtil.getInt("net.kv.scanner.batch.size", 1000); // 每批次最大点数量
|
||||
private volatile boolean finished = false;
|
||||
private volatile boolean splitting = false;
|
||||
private volatile int nextKeySerialNo = 1;
|
||||
|
||||
public TaskSplitter(HgScanQuery scanQuery, BiFunction<HgScanQuery, KvCloseableIterator, Boolean> handler) {
|
||||
this.scanQuery = scanQuery;
|
||||
this.taskHandler = handler;
|
||||
if (scanQuery.getScanMethod() == HgScanQuery.ScanMethod.PREFIX) {
|
||||
if (scanQuery.getPrefixItr() != null) {
|
||||
prefixItr = scanQuery.getPrefixItr();
|
||||
} else {
|
||||
prefixItr = scanQuery.getPrefixList().listIterator();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void setNotifier(KvBatchScannerMerger notifier) {
|
||||
this.notifier = notifier;
|
||||
}
|
||||
|
||||
public boolean isFinished() {
|
||||
return finished;
|
||||
}
|
||||
|
||||
/**
|
||||
* 评估最大任务数
|
||||
*/
|
||||
private void evaluateMaxTaskSize() {
|
||||
if (maxTaskSize == 0) { // 根据第一批次任务，得到store数量，然后计算最大任务数
|
||||
if (scanQuery.getOrderType() == ScanOrderType.ORDER_STRICT) {
|
||||
maxTaskSize = 1; // 点排序，每台机器一个流, 所有store流结束后才能启动其他流
|
||||
} else {
|
||||
maxTaskSize = this.notifier.getScannerCount() * maxTaskSizePerStore;
|
||||
}
|
||||
maxBatchSize = this.notifier.getScannerCount() * maxBatchSize; // 每台机器最多1000条
|
||||
|
||||
/*
|
||||
* Limit少于10000时启动一个流，节省网络带宽
|
||||
*/
|
||||
if (scanQuery.getLimit() < maxBatchSize * 30L) {
|
||||
maxTaskSize = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 拆分任务，任务拆分为多个grpc请求
|
||||
*/
|
||||
public void splitTask() {
|
||||
if (this.finished || this.splitting) {
|
||||
return;
|
||||
}
|
||||
synchronized (this) {
|
||||
if (this.finished) {
|
||||
return;
|
||||
}
|
||||
this.splitting = true;
|
||||
if (scanQuery.getScanMethod() == HgScanQuery.ScanMethod.PREFIX) {
|
||||
if (prefixItr.hasNext() && (maxTaskSize == 0 || notifier.getScannerCount() < maxTaskSize)) {
|
||||
List<HgOwnerKey> keys = new ArrayList<>(maxBatchSize);
|
||||
for (int i = 0; i < maxBatchSize && prefixItr.hasNext(); i++) {
|
||||
keys.add(prefixItr.next().setSerialNo(nextKeySerialNo++));
|
||||
}
|
||||
taskHandler.apply(
|
||||
HgScanQuery.prefixOf(scanQuery.getTable(), keys, scanQuery.getOrderType()), this.notifier);
|
||||
// 评估最大任务数
|
||||
evaluateMaxTaskSize();
|
||||
if (this.notifier.getScannerCount() < this.maxTaskSize) {
|
||||
splitTask(); // 未达到最大任务数，继续拆分
|
||||
}
|
||||
}
|
||||
this.finished = !prefixItr.hasNext();
|
||||
} else {
|
||||
taskHandler.apply(scanQuery, this.notifier);
|
||||
this.finished = true;
|
||||
}
|
||||
this.splitting = false;
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void close() {
|
||||
finished = true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询结果接收器
|
||||
*/
|
||||
static class KvBatchReceiver implements StreamObserver<KvStream> {
|
||||
KvBatchScanner scanner;
|
||||
boolean sortByVertex;
|
||||
|
||||
KvBatchReceiver(KvBatchScanner scanner, boolean sortByVertex) {
|
||||
this.scanner = scanner;
|
||||
this.sortByVertex = sortByVertex;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onNext(KvStream value) {
|
||||
try {
|
||||
ByteBuffer buffer = value.getStream();
|
||||
int seqNo = value.getSeqNo();
|
||||
boolean isOver = value.getOver();
|
||||
scanner.dataArrived(
|
||||
() -> {
|
||||
scanner.sendResponse(seqNo);
|
||||
if (isOver) {
|
||||
scanner.dataComplete();
|
||||
}
|
||||
return new KVBytesIterator(buffer, sortByVertex, scanner);
|
||||
});
|
||||
} catch (InterruptedException e) {
|
||||
close();
|
||||
log.error("exception ", e);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(Throwable t) {
|
||||
log.error("exception ", t);
|
||||
close();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onCompleted() {
|
||||
close();
|
||||
}
|
||||
|
||||
private void close() {
|
||||
if (scanner != null) {
|
||||
scanner.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static class KVBytesIterator implements HgKvOrderedIterator<HgKvEntry> {
|
||||
private final KvBatchScanner scanner;
|
||||
KVByteBuffer buffer;
|
||||
HgKvEntry entry;
|
||||
int sn;
|
||||
boolean hasSN;
|
||||
|
||||
public KVBytesIterator(ByteBuffer buffer, boolean hasNo, KvBatchScanner scanner) {
|
||||
this.buffer = new KVByteBuffer(buffer);
|
||||
this.hasSN = hasNo;
|
||||
this.scanner = scanner;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
// this.scanner.close();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] key() {
|
||||
return entry.key();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] value() {
|
||||
return entry.value();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] position() {
|
||||
return new byte[0];
|
||||
}
|
||||
|
||||
@Override
|
||||
public void seek(byte[] position) {
|
||||
throw new RuntimeException("not implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return buffer.hasRemaining();
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvEntry next() {
|
||||
if (hasSN) {
|
||||
sn = buffer.getInt();
|
||||
}
|
||||
entry = new GrpcKvEntryImpl(buffer.getBytes(), buffer.getBytes(), 0);
|
||||
return entry;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSequence() {
|
||||
return sn;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(HgKvOrderedIterator o) {
|
||||
return Long.compare(this.getSequence(), o.getSequence());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,413 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import com.baidu.hugegraph.store.HgPageSize;
|
||||
import com.baidu.hugegraph.store.HgScanQuery;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeSession;
|
||||
import com.baidu.hugegraph.store.client.type.HgStoreClientException;
|
||||
import com.baidu.hugegraph.store.client.util.Base58;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConfig;
|
||||
import com.baidu.hugegraph.store.grpc.common.Kv;
|
||||
import com.baidu.hugegraph.store.grpc.stream.*;
|
||||
import io.grpc.stub.StreamObserver;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static com.baidu.hugegraph.store.client.grpc.KvBatchUtil.createQueryReq;
|
||||
import static com.baidu.hugegraph.store.client.grpc.KvBatchUtil.getRequestBuilder;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2022/03/23
|
||||
* @version 5.0.0
|
||||
*/
|
||||
@Slf4j
|
||||
@NotThreadSafe
|
||||
class KvBatchScanner5 {
|
||||
private final static HgStoreClientConfig storeClientConfig = HgStoreClientConfig.of();
|
||||
//private final static int HAVE_NEXT_TIMEOUT_SECONDS = storeClientConfig.getNetKvScannerHaveNextTimeout();
|
||||
private final static int HAVE_NEXT_TIMEOUT_SECONDS = 60;
|
||||
private final static long PAGE_SIZE = storeClientConfig.getNetKvScannerPageSize();
|
||||
|
||||
private enum OrderState {
|
||||
NEW(0),
|
||||
WORKING(1),
|
||||
COMPLETED(10);
|
||||
|
||||
int value;
|
||||
|
||||
OrderState(int value) {
|
||||
this.value = value;
|
||||
}
|
||||
}
|
||||
|
||||
public static KvCloseableIterator scan(HgStoreNodeSession nodeSession,
|
||||
HgStoreStreamGrpc.HgStoreStreamStub stub,
|
||||
HgScanQuery scanQuery) {
|
||||
return new OrderConsumer(new OrderBroker(stub, scanQuery, nodeSession));
|
||||
}
|
||||
|
||||
/*** Broker ***/
|
||||
private static class OrderBroker {
|
||||
private final HgScanQuery scanQuery;
|
||||
private final StreamObserver<ScanStreamBatchReq> requestObserver;
|
||||
private final ScanStreamBatchReq.Builder reqBuilder;
|
||||
private final ReentrantLock senderLock = new ReentrantLock();
|
||||
private final AtomicBoolean serverFinished = new AtomicBoolean();
|
||||
private final AtomicBoolean clientFinished = new AtomicBoolean();
|
||||
|
||||
private final ScanReceiptRequest.Builder receiptReqBuilder = ScanReceiptRequest.newBuilder();
|
||||
private final ScanCancelRequest cancelReq = ScanCancelRequest.newBuilder().build();
|
||||
private final HgStoreNodeSession nodeSession;
|
||||
private final OrderAgent agent;
|
||||
|
||||
public final OrderKeeper keeper = new OrderKeeper();
|
||||
|
||||
private OrderState state = OrderState.NEW;
|
||||
|
||||
private final AtomicLong receivedCount = new AtomicLong();
|
||||
private final AtomicInteger receivedLastTimes = new AtomicInteger();
|
||||
private final BlockingQueue<Integer> timesQueue = new LinkedBlockingQueue();
|
||||
String brokerId = "";
|
||||
|
||||
OrderBroker(HgStoreStreamGrpc.HgStoreStreamStub stub,
|
||||
HgScanQuery scanQuery,
|
||||
HgStoreNodeSession nodeSession) {
|
||||
|
||||
if (log.isDebugEnabled())
|
||||
if (scanQuery.getPrefixList() != null && scanQuery.getPrefixList().size() > 0) {
|
||||
brokerId = Base58.encode(scanQuery.getPrefixList().get(0).getKey());
|
||||
|
||||
log.debug("[ANALYSIS START] [{}] firstKey: {}, keyLength: {}, table: {}, node: {}"
|
||||
, brokerId
|
||||
, scanQuery.getPrefixList().get(0)
|
||||
, scanQuery.getPrefixList().size()
|
||||
, scanQuery.getTable()
|
||||
, nodeSession.getStoreNode().getAddress());
|
||||
}
|
||||
|
||||
this.scanQuery = scanQuery;
|
||||
this.reqBuilder = getRequestBuilder(nodeSession);
|
||||
this.nodeSession = nodeSession;
|
||||
this.agent = new OrderAgent(brokerId);
|
||||
this.requestObserver = stub.scanBatch(agent);
|
||||
|
||||
}
|
||||
|
||||
List<Kv> oneMore() {
|
||||
|
||||
if (this.state == OrderState.NEW) {
|
||||
synchronized (this.state) {
|
||||
if (this.state == OrderState.NEW) {
|
||||
this.makeADeal();
|
||||
this.state = OrderState.WORKING;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
this.sendReceipt();
|
||||
}
|
||||
|
||||
return this.keeper.pickUp();
|
||||
}
|
||||
|
||||
void receipt(int times) {
|
||||
this.timesQueue.offer(times);
|
||||
receivedLastTimes.set(times);
|
||||
}
|
||||
|
||||
void sendReceipt() {
|
||||
Integer buf = this.timesQueue.poll();
|
||||
|
||||
if (buf == null) {
|
||||
buf = this.receivedLastTimes.get();
|
||||
}
|
||||
|
||||
AtomicInteger timesBuf = new AtomicInteger(buf);
|
||||
|
||||
if (!this.clientFinished.get()) {
|
||||
this.send(() ->
|
||||
getReqBuilder().setReceiptRequest(this.receiptReqBuilder.setTimes(timesBuf.get()).build()).build()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private void makeADeal() {
|
||||
this.send(() -> getReqBuilder()
|
||||
.setQueryRequest(createQueryReq(scanQuery, PAGE_SIZE)).build()
|
||||
);
|
||||
}
|
||||
|
||||
private void finish(long tookAmt) {
|
||||
this.clientFinished.set(true);
|
||||
if (log.isDebugEnabled()) log.debug("[ANALYSIS END] [{}] times: {}, received: {}, took: {}"
|
||||
, this.brokerId
|
||||
, this.receivedLastTimes.get()
|
||||
, this.receivedCount.get()
|
||||
, tookAmt
|
||||
);
|
||||
if (this.receivedCount.get() != tookAmt) {
|
||||
if (log.isDebugEnabled()) log.debug("[ANALYSIS END] [{}] times: {}, received: {}, took: {}"
|
||||
, this.brokerId
|
||||
, this.receivedLastTimes.get()
|
||||
, this.receivedCount.get()
|
||||
, tookAmt
|
||||
);
|
||||
}
|
||||
synchronized (this.state) {
|
||||
if (this.state.value < OrderState.COMPLETED.value) {
|
||||
this.send(() -> getReqBuilder().setCancelRequest(this.cancelReq).build());
|
||||
this.state = OrderState.COMPLETED;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private ScanStreamBatchReq.Builder getReqBuilder() {
|
||||
return this.reqBuilder.clearQueryRequest();
|
||||
}
|
||||
|
||||
private void send(Supplier<ScanStreamBatchReq> supplier) {
|
||||
this.senderLock.lock();
|
||||
try {
|
||||
if (this.serverFinished.get() == false) {
|
||||
this.requestObserver.onNext(supplier.get());
|
||||
}
|
||||
Thread.yield();
|
||||
} finally {
|
||||
this.senderLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private class OrderAgent implements StreamObserver<KvPageRes> {
|
||||
private final AtomicInteger count = new AtomicInteger(0);
|
||||
private AtomicBoolean over = new AtomicBoolean(false);
|
||||
private final String agentId;
|
||||
|
||||
OrderAgent(String agentId) {
|
||||
this.agentId = agentId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onNext(KvPageRes value) {
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("Scan [ {} ] [ {} ] times, received: [ {} ]"
|
||||
, nodeSession.getStoreNode().getAddress(), value.getTimes(), value.getDataList().size());
|
||||
}
|
||||
|
||||
serverFinished.set(value.getOver());
|
||||
|
||||
List<Kv> buffer = value.getDataList();
|
||||
count.addAndGet(buffer.size());
|
||||
if (log.isDebugEnabled()) {
|
||||
if (value.getOver()) {
|
||||
log.debug("[ANALYSIS OVER] [{}] count: {}", agentId, count);
|
||||
}
|
||||
}
|
||||
keeper.receive(buffer, value.getTimes());
|
||||
this.over.set(value.getOver());
|
||||
this.checkOver(value.getTimes());
|
||||
}
|
||||
|
||||
private void checkOver(int times) {
|
||||
if (this.over.get()) {
|
||||
requestObserver.onCompleted();
|
||||
keeper.done(times);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(Throwable t) {
|
||||
log.error("received server onError event, Throwable:", t);
|
||||
keeper.shout(t);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onCompleted() {
|
||||
if (log.isDebugEnabled()) log.debug("received sever completed event.");
|
||||
serverFinished.set(true);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/*** Inventory Keeper ***/
|
||||
private class OrderKeeper {
|
||||
private final BlockingQueue<Supplier<List<Kv>>> queue = new LinkedBlockingQueue<>();
|
||||
private final ReentrantLock pickUpLock = new ReentrantLock();
|
||||
private final AtomicBoolean done = new AtomicBoolean();
|
||||
private final AtomicBoolean stop = new AtomicBoolean();
|
||||
private int timesOfOver;
|
||||
private int lastTimes;
|
||||
private Throwable serverErr;
|
||||
|
||||
void receive(List<Kv> data, int times) {
|
||||
receivedCount.addAndGet(data.size());
|
||||
this.queue.offer(() -> data);
|
||||
receipt(times);
|
||||
|
||||
this.lastTimes = times;
|
||||
}
|
||||
|
||||
private List<Kv> pickUp() {
|
||||
Supplier<List<Kv>> res;
|
||||
|
||||
pickUpLock.lock();
|
||||
try {
|
||||
|
||||
if (this.done.get()) {
|
||||
if (this.stop.get()) {
|
||||
log.warn("Invoking pickUp method after OrderKeeper has bean closing.");
|
||||
}
|
||||
res = this.queue.poll();
|
||||
if (res == null) {
|
||||
res = () -> null;
|
||||
}
|
||||
} else {
|
||||
res = this.queue.poll(HAVE_NEXT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
|
||||
if (res == null) {
|
||||
if (this.done.get()) {
|
||||
res = () -> null;
|
||||
} else {
|
||||
throw HgStoreClientException.of("Timeout, max time: " + HAVE_NEXT_TIMEOUT_SECONDS + " seconds" +
|
||||
", isOver: " + this.done.get() +
|
||||
", isStop: " + this.stop.get() +
|
||||
", last-times: " + this.lastTimes +
|
||||
", over-times: " + this.timesOfOver);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
log.error("Failed to receive List<Kv> from queue because of interruption of current thread ["
|
||||
+ Thread.currentThread().getName() + "]");
|
||||
|
||||
Thread.currentThread().interrupt();
|
||||
|
||||
throw HgStoreClientException.of("Failed to receive List<Kv> from queue, cause by:", e);
|
||||
} finally {
|
||||
pickUpLock.unlock();
|
||||
}
|
||||
|
||||
checkServerErr();
|
||||
return res.get();
|
||||
|
||||
}
|
||||
|
||||
void done(int times) {
|
||||
this.timesOfOver = times;
|
||||
this.done.set(true);
|
||||
this.queue.offer(() -> null);
|
||||
}
|
||||
|
||||
void shout(Throwable t) {
|
||||
this.serverErr = t;
|
||||
log.error("Failed to receive from sever", t);
|
||||
this.queue.offer(() -> null);
|
||||
}
|
||||
|
||||
private void checkServerErr() {
|
||||
if (this.serverErr != null) {
|
||||
throw HgStoreClientException.of(this.serverErr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
/* iterator */
|
||||
private static class OrderConsumer implements KvCloseableIterator<Kv>, HgPageSize {
|
||||
private OrderBroker broker;
|
||||
private Iterator<Kv> dataIterator;
|
||||
private long tookCount = 0;
|
||||
private final String consumerId;
|
||||
|
||||
OrderConsumer(OrderBroker broker) {
|
||||
this.broker = broker;
|
||||
consumerId = broker.brokerId;
|
||||
}
|
||||
|
||||
private Iterator<Kv> getIterator() {
|
||||
List<Kv> list = this.broker.oneMore();
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
if (list != null && list.isEmpty()) {
|
||||
log.debug("[ANALYSIS EMPTY] [{}] , tookCount: {}", consumerId, tookCount);
|
||||
}
|
||||
}
|
||||
|
||||
if (list == null || list.isEmpty()) {
|
||||
return null;
|
||||
} else {
|
||||
return list.iterator();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
this.broker.finish(this.tookCount);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getPageSize() {
|
||||
return PAGE_SIZE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
|
||||
if (this.dataIterator == null) {
|
||||
this.dataIterator = this.getIterator();
|
||||
} else {
|
||||
if (this.dataIterator.hasNext()) {
|
||||
return true;
|
||||
} else {
|
||||
this.dataIterator = this.getIterator();
|
||||
}
|
||||
}
|
||||
|
||||
if (this.dataIterator == null) {
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("[ANALYSIS NULL -> FALSE] [{}] , tookCount: {}", consumerId, tookCount);
|
||||
}
|
||||
return false;
|
||||
} else {
|
||||
if (log.isDebugEnabled()) {
|
||||
if (!this.dataIterator.hasNext()) {
|
||||
log.debug("[ANALYSIS hasNext -> FALSE] [{}] , tookCount: {}", consumerId, tookCount);
|
||||
}
|
||||
}
|
||||
return this.dataIterator.hasNext();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public Kv next() {
|
||||
if (this.dataIterator == null) {
|
||||
if (!this.hasNext()) {
|
||||
throw new NoSuchElementException();
|
||||
}
|
||||
}
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
tookCount++;
|
||||
if (tookCount % 10000 == 0) {
|
||||
log.debug("[ANALYSIS NEXT] [{}] , tookCount: {}", consumerId, tookCount);
|
||||
}
|
||||
}
|
||||
return this.dataIterator.next();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,310 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import static com.baidu.hugegraph.store.client.grpc.KvBatchScanner.NO_DATA;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import com.baidu.hugegraph.store.HgKvEntry;
|
||||
import com.baidu.hugegraph.store.HgKvIterator;
|
||||
import com.baidu.hugegraph.store.HgKvOrderedIterator;
|
||||
import com.baidu.hugegraph.store.HgPageSize;
|
||||
import com.baidu.hugegraph.store.client.util.PropertyUtil;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* 批量查询结果归并，阻塞队列工作模式
|
||||
* 对请求任务的拆分,创建多个请求队列
|
||||
*/
|
||||
@Slf4j
|
||||
public class KvBatchScannerMerger implements KvCloseableIterator<HgKvIterator<HgKvEntry>>, HgPageSize {
|
||||
|
||||
static int maxWaitCount = PropertyUtil.getInt("net.kv.scanner.wait.timeout", 60);
|
||||
protected final BlockingQueue<Supplier<HgKvIterator<HgKvEntry>>> queue = new LinkedBlockingQueue<>();
|
||||
private Supplier<HgKvIterator<HgKvEntry>> current = null;
|
||||
private final KvBatchScanner.TaskSplitter taskSplitter;
|
||||
private List<KvBatchScanner> scanners = new CopyOnWriteArrayList<>();
|
||||
|
||||
public KvBatchScannerMerger(KvBatchScanner.TaskSplitter splitter) {
|
||||
this.taskSplitter = splitter;
|
||||
splitter.setNotifier(this);
|
||||
}
|
||||
|
||||
public void startTask() {
|
||||
taskSplitter.splitTask();
|
||||
}
|
||||
|
||||
public void dataArrived(KvBatchScanner scanner, Supplier<HgKvIterator<HgKvEntry>> supplier)
|
||||
throws InterruptedException {
|
||||
queue.put(supplier);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
int waitTime = 0;
|
||||
while (current == null) {
|
||||
try {
|
||||
// 队列有数据，还有活跃的查询器，任务未分配完
|
||||
if (queue.size() != 0 || scanners.size() > 0 || !taskSplitter.isFinished()) {
|
||||
current = queue.poll(1, TimeUnit.SECONDS); //定期检查client是否被关闭了
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
if (current == null) {
|
||||
// 超时重试
|
||||
sendTimeout();
|
||||
if (++waitTime > maxWaitCount) {
|
||||
log.error("KvBatchScanner wait data timeout {}, closeables is {}, task is {}",
|
||||
waitTime, scanners.size(), taskSplitter.isFinished());
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
log.error("hasNext interrupted {}", e);
|
||||
throw new RuntimeException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
return current != null && current != NO_DATA;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public HgKvIterator<HgKvEntry> next() {
|
||||
HgKvIterator<HgKvEntry> iterator = null;
|
||||
if (current != null) {
|
||||
iterator = current.get();
|
||||
}
|
||||
current = null;
|
||||
return iterator;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
taskSplitter.close();
|
||||
scanners.forEach(c -> c.close());
|
||||
}
|
||||
|
||||
private void sendTimeout() {
|
||||
scanners.forEach(v -> {
|
||||
v.sendResponse();
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getPageSize() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
public void registerScanner(KvBatchScanner closeable) {
|
||||
this.scanners.add(closeable);
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回值<0表示任务结束
|
||||
*
|
||||
* @param closeable
|
||||
* @return
|
||||
*/
|
||||
public int unregisterScanner(KvBatchScanner closeable) {
|
||||
this.scanners.remove(closeable);
|
||||
try {
|
||||
taskSplitter.splitTask();
|
||||
} catch (Exception e) {
|
||||
log.error("exception ", e);
|
||||
}
|
||||
return taskSplitter.isFinished() && this.scanners.size() == 0 ?
|
||||
-1 : this.scanners.size();
|
||||
}
|
||||
|
||||
public int getScannerCount() {
|
||||
return this.scanners.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* 组装一个Scanner的多个有序迭代器为一个迭代器
|
||||
*/
|
||||
static class ScannerDataQueue {
|
||||
private BlockingQueue<Supplier<HgKvIterator<HgKvEntry>>> queue;
|
||||
private HgKvOrderedIterator<HgKvEntry> iterator = null;
|
||||
private int currentSN = 0;
|
||||
private HgKvEntry entry;
|
||||
|
||||
public ScannerDataQueue() {
|
||||
queue = new LinkedBlockingQueue<>();
|
||||
}
|
||||
|
||||
public int sn() {
|
||||
return currentSN;
|
||||
}
|
||||
|
||||
public void add(Supplier<HgKvIterator<HgKvEntry>> supplier) {
|
||||
if (queue != null) {
|
||||
queue.add(supplier);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 迭代器是否有效，如果没有数据，等待数据到达
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public boolean hasNext() {
|
||||
while (entry == null && queue != null) {
|
||||
try {
|
||||
int waitTime = 0;
|
||||
Supplier<HgKvIterator<HgKvEntry>> current;
|
||||
current = queue.poll(1, TimeUnit.SECONDS); //定期检查client是否被关闭了
|
||||
if (current == null) {
|
||||
if (++waitTime > maxWaitCount) {
|
||||
break;
|
||||
}
|
||||
} else if (current == NO_DATA) {
|
||||
queue = null;
|
||||
break;
|
||||
} else {
|
||||
iterator = (HgKvOrderedIterator<HgKvEntry>) current.get();
|
||||
if (iterator != null && iterator.hasNext()) {
|
||||
moveNext();
|
||||
} else {
|
||||
iterator = null;
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
log.error("hasNext interrupted {}", e);
|
||||
throw new RuntimeException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
return entry != null;
|
||||
}
|
||||
|
||||
public HgKvEntry next() {
|
||||
HgKvEntry current = entry;
|
||||
moveNext();
|
||||
return current;
|
||||
}
|
||||
|
||||
private void moveNext() {
|
||||
if (iterator.hasNext()) {
|
||||
entry = iterator.next();
|
||||
currentSN = (int) iterator.getSequence();
|
||||
} else {
|
||||
entry = null;
|
||||
iterator = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 对多个Scanner返回结果进行归并排序
|
||||
*/
|
||||
static class SortedScannerMerger extends KvBatchScannerMerger {
|
||||
// 每一个流对应一个接收队列
|
||||
private Map<KvBatchScanner, ScannerDataQueue> scannerQueues = new ConcurrentHashMap<>();
|
||||
|
||||
public SortedScannerMerger(KvBatchScanner.TaskSplitter splitter) {
|
||||
super(splitter);
|
||||
queue.add(() -> {
|
||||
// 对store返回结果进行归并排序
|
||||
return new HgKvIterator<HgKvEntry>() {
|
||||
private ScannerDataQueue iterator;
|
||||
private int currentSN = 0;
|
||||
private HgKvEntry entry;
|
||||
|
||||
@Override
|
||||
public byte[] key() {
|
||||
return entry.key();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] value() {
|
||||
return entry.value();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] position() {
|
||||
return new byte[0];
|
||||
}
|
||||
|
||||
@Override
|
||||
public void seek(byte[] position) {
|
||||
throw new RuntimeException("not implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
//
|
||||
if (iterator == null || !iterator.hasNext() || currentSN != iterator.sn()) {
|
||||
iterator = selectIterator();
|
||||
}
|
||||
|
||||
if (iterator != null) {
|
||||
currentSN = iterator.sn();
|
||||
}
|
||||
return iterator != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HgKvEntry next() {
|
||||
entry = iterator.next();
|
||||
return entry;
|
||||
}
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 从多个Scanner中挑选一个sn最小的迭代器
|
||||
* 如果Scanner没有数据，等待数据到达。
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
private ScannerDataQueue selectIterator() {
|
||||
int sn = Integer.MAX_VALUE;
|
||||
ScannerDataQueue current = null;
|
||||
while (current == null && !scannerQueues.isEmpty()) {
|
||||
Iterator<KvBatchScanner> itr = scannerQueues.keySet().iterator();
|
||||
while (itr.hasNext()) {
|
||||
KvBatchScanner key = itr.next();
|
||||
ScannerDataQueue kvItr = scannerQueues.get(key);
|
||||
if (!kvItr.hasNext()) {
|
||||
scannerQueues.remove(key);
|
||||
continue;
|
||||
}
|
||||
if (kvItr.sn() <= sn) {
|
||||
sn = kvItr.sn();
|
||||
current = kvItr;
|
||||
}
|
||||
}
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
public void registerScanner(KvBatchScanner scanner) {
|
||||
super.registerScanner(scanner);
|
||||
scannerQueues.putIfAbsent(scanner, new ScannerDataQueue());
|
||||
}
|
||||
|
||||
public int unregisterScanner(KvBatchScanner scanner) {
|
||||
dataArrived(scanner, NO_DATA);
|
||||
return super.unregisterScanner(scanner);
|
||||
}
|
||||
|
||||
public void dataArrived(KvBatchScanner scanner, Supplier<HgKvIterator<HgKvEntry>> supplier) {
|
||||
scannerQueues.putIfAbsent(scanner, new ScannerDataQueue());
|
||||
scannerQueues.get(scanner).add(supplier);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.HgScanQuery;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeSession;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConst;
|
||||
import com.baidu.hugegraph.store.grpc.common.Header;
|
||||
import com.baidu.hugegraph.store.grpc.common.ScanMethod;
|
||||
import com.baidu.hugegraph.store.grpc.stream.*;
|
||||
import com.google.protobuf.ByteString;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
import java.util.List;
|
||||
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2022/04/23
|
||||
*/
|
||||
@Slf4j
|
||||
@NotThreadSafe
|
||||
class KvBatchUtil {
|
||||
static final byte[] EMPTY_POSITION = HgStoreClientConst.EMPTY_BYTES;
|
||||
|
||||
static ScanStreamBatchReq.Builder getRequestBuilder(HgStoreNodeSession nodeSession) {
|
||||
return ScanStreamBatchReq.newBuilder().setHeader(getHeader(nodeSession));
|
||||
}
|
||||
|
||||
static ScanQueryRequest createQueryReq(HgScanQuery scanQuery, long pageSize) {
|
||||
|
||||
ScanQueryRequest.Builder qb = ScanQueryRequest.newBuilder();
|
||||
ScanCondition.Builder cb = ScanCondition.newBuilder();
|
||||
|
||||
qb.setLimit(getLimit(scanQuery.getLimit()));
|
||||
qb.setPerKeyLimit(getLimit(scanQuery.getPerKeyLimit()));
|
||||
qb.setPerKeyMax(getLimit(scanQuery.getPerKeyMax()));
|
||||
|
||||
switch (scanQuery.getScanMethod()) {
|
||||
case ALL:
|
||||
qb.setMethod(ScanMethod.ALL);
|
||||
break;
|
||||
case PREFIX:
|
||||
qb.setMethod(ScanMethod.PREFIX);
|
||||
addPrefixCondition(scanQuery, qb, cb);
|
||||
break;
|
||||
case RANGE:
|
||||
qb.setMethod(ScanMethod.RANGE);
|
||||
addRangeCondition(scanQuery, qb, cb);
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException("Unsupported ScanType: " + scanQuery.getScanMethod());
|
||||
}
|
||||
|
||||
qb.setTable(scanQuery.getTable());
|
||||
qb.setPageSize(pageSize);
|
||||
qb.setQuery(toBs(scanQuery.getQuery()));
|
||||
qb.setScanType(scanQuery.getScanType());
|
||||
qb.setOrderType(scanQuery.getOrderType());
|
||||
qb.setSkipDegree(scanQuery.getSkipDegree());
|
||||
|
||||
return qb.build();
|
||||
}
|
||||
|
||||
static long getLimit(long limit) {
|
||||
return limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE : limit;
|
||||
}
|
||||
|
||||
static Header getHeader(HgStoreNodeSession nodeSession) {
|
||||
return Header.newBuilder().setGraph(nodeSession.getGraphName()).build();
|
||||
}
|
||||
|
||||
static void addPrefixCondition(HgScanQuery scanQuery, ScanQueryRequest.Builder qb, ScanCondition.Builder cb) {
|
||||
List<HgOwnerKey> prefixList = scanQuery.getPrefixList();
|
||||
|
||||
if (prefixList == null || prefixList.isEmpty()) {
|
||||
throw new RuntimeException("The start-list of ScanQuery shouldn't to be invalid in ScanMethod.PREFIX mode.");
|
||||
}
|
||||
|
||||
prefixList.forEach((e) -> {
|
||||
qb.addCondition(cb.clear()
|
||||
.setPrefix(toBs(e.getKey()))
|
||||
.setCode(e.getKeyCode())
|
||||
.setSerialNo(e.getSerialNo())
|
||||
.build()
|
||||
);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
static void addRangeCondition(HgScanQuery scanQuery, ScanQueryRequest.Builder qb, ScanCondition.Builder cb) {
|
||||
List<HgOwnerKey> startList = scanQuery.getStartList();
|
||||
List<HgOwnerKey> endList = scanQuery.getEndList();
|
||||
|
||||
if (startList == null || startList.isEmpty()) {
|
||||
throw new RuntimeException("The start-list of ScanQuery shouldn't to be invalid in ScanMethod.RANGE mode.");
|
||||
}
|
||||
|
||||
if (endList == null || endList.isEmpty()) {
|
||||
throw new RuntimeException("The end-list of ScanQuery shouldn't to be invalid in ScanMethod.RANGE mode.");
|
||||
}
|
||||
|
||||
if (startList.size() != endList.size()) {
|
||||
throw new RuntimeException("The size of start-list not equals end-list's.");
|
||||
}
|
||||
|
||||
for (int i = 0, s = startList.size(); i < s; i++) {
|
||||
HgOwnerKey start = startList.get(i);
|
||||
HgOwnerKey end = endList.get(i);
|
||||
qb.addCondition(cb.clear().setCode(start.getKeyCode())
|
||||
.setStart(toBs(start.getKey()))
|
||||
.setEnd(toBs(end.getKey()))
|
||||
.setSerialNo(start.getSerialNo())
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static HgOwnerKey toOk(HgOwnerKey key) {
|
||||
return key == null ? HgStoreClientConst.EMPTY_OWNER_KEY : key;
|
||||
}
|
||||
|
||||
static ByteString toBs(byte[] bytes) {
|
||||
return ByteString.copyFrom((bytes != null) ? bytes : EMPTY_BYTES);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.util.Iterator;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2022/3/16
|
||||
*/
|
||||
public interface KvCloseableIterator<T> extends Iterator<T>, Closeable {
|
||||
public void close();
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2022/4/6
|
||||
*/
|
||||
class KvListIterator<T> implements KvCloseableIterator{
|
||||
|
||||
private final Iterator<T> iterator;
|
||||
|
||||
KvListIterator(List<T> list){
|
||||
this.iterator=list.iterator();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
/*Nothing to do.*/
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return this.iterator.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public T next() {
|
||||
return this.iterator.next();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,188 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import static com.baidu.hugegraph.store.HgKvStore.SCAN_ANY;
|
||||
import static com.baidu.hugegraph.store.HgKvStore.SCAN_PREFIX_BEGIN;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toBytes;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toLong;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.HgPageSize;
|
||||
import com.baidu.hugegraph.store.HgSeekAble;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeSession;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConfig;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConst;
|
||||
import com.baidu.hugegraph.store.grpc.common.Header;
|
||||
import com.baidu.hugegraph.store.grpc.common.Kv;
|
||||
import com.baidu.hugegraph.store.grpc.common.ScanMethod;
|
||||
import com.baidu.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamBlockingStub;
|
||||
import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq;
|
||||
import com.google.protobuf.ByteString;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/12/1
|
||||
*/
|
||||
@Slf4j
|
||||
@NotThreadSafe
|
||||
class KvOneShotScanner implements KvCloseableIterator<Kv>, HgPageSize, HgSeekAble {
|
||||
private static final HgStoreClientConfig storeClientConfig = HgStoreClientConfig.of();
|
||||
private final HgStoreNodeSession session;
|
||||
private final HgStoreStreamBlockingStub stub;
|
||||
private ScanStreamReq.Builder reqBuilder = ScanStreamReq.newBuilder();
|
||||
private ScanStreamReq req;
|
||||
private Iterator<Kv> iterator;
|
||||
private List<Kv> list = null;
|
||||
private final String table;
|
||||
private final HgOwnerKey startKey;
|
||||
private final HgOwnerKey endKey;
|
||||
private final HgOwnerKey prefix;
|
||||
private final ScanMethod scanMethod;
|
||||
private final long limit;
|
||||
private final int partition;
|
||||
private final int scanType;
|
||||
private final byte[] query;
|
||||
private final int pageSize;
|
||||
private boolean in = true;
|
||||
private byte[] nodePosition = HgStoreClientConst.EMPTY_BYTES;
|
||||
|
||||
public static KvCloseableIterator<Kv> scanAll(HgStoreNodeSession session, HgStoreStreamBlockingStub stub,
|
||||
String table, long limit, byte[] query) {
|
||||
return new KvOneShotScanner(ScanMethod.ALL, session, stub, table, null, null, null, limit, -1, SCAN_ANY,
|
||||
query);
|
||||
}
|
||||
|
||||
public static KvCloseableIterator<Kv> scanPrefix(HgStoreNodeSession session, HgStoreStreamBlockingStub stub,
|
||||
String table, HgOwnerKey prefix, long limit, byte[] query) {
|
||||
return new KvOneShotScanner(ScanMethod.PREFIX, session, stub, table, prefix, null, null, limit,
|
||||
prefix.getKeyCode(), SCAN_PREFIX_BEGIN, query);
|
||||
}
|
||||
|
||||
public static KvCloseableIterator<Kv> scanRange(HgStoreNodeSession nodeSession, HgStoreStreamBlockingStub stub,
|
||||
String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit,
|
||||
int scanType, byte[] query) {
|
||||
return new KvOneShotScanner(ScanMethod.RANGE, nodeSession, stub, table, null, startKey, endKey, limit,
|
||||
startKey.getKeyCode(), scanType, query);
|
||||
}
|
||||
|
||||
|
||||
private KvOneShotScanner(ScanMethod scanMethod, HgStoreNodeSession session, HgStoreStreamBlockingStub stub,
|
||||
String table, HgOwnerKey prefix, HgOwnerKey startKey, HgOwnerKey endKey, long limit,
|
||||
int partition, int scanType, byte[] query) {
|
||||
this.scanMethod = scanMethod;
|
||||
this.session = session;
|
||||
this.stub = stub;
|
||||
this.table = table;
|
||||
this.startKey = toOk(startKey);
|
||||
this.endKey = toOk(endKey);
|
||||
this.prefix = toOk(prefix);
|
||||
this.partition = partition;
|
||||
this.scanType = scanType;
|
||||
this.query = query != null ? query : HgStoreClientConst.EMPTY_BYTES;
|
||||
this.limit = limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE : limit; // <=0 means no limit
|
||||
this.pageSize = storeClientConfig.getNetKvScannerPageSize();
|
||||
|
||||
}
|
||||
|
||||
private Header getHeader(HgStoreNodeSession nodeSession) {
|
||||
return Header.newBuilder().setGraph(nodeSession.getGraphName()).build();
|
||||
}
|
||||
|
||||
private void createReq() {
|
||||
this.req = this.reqBuilder
|
||||
.setHeader(this.getHeader(this.session))
|
||||
.setMethod(this.scanMethod)
|
||||
.setTable(this.table)
|
||||
.setStart(toBs(this.startKey.getKey()))
|
||||
.setEnd(toBs(this.endKey.getKey()))
|
||||
.setLimit(this.limit)
|
||||
.setPrefix(toBs(this.prefix.getKey()))
|
||||
.setCode(this.partition)
|
||||
.setScanType(this.scanType)
|
||||
.setQuery(toBs(this.query))
|
||||
.setPageSize(pageSize)
|
||||
.setPosition(toBs(this.nodePosition))
|
||||
.build();
|
||||
}
|
||||
|
||||
private void init() {
|
||||
|
||||
if (this.iterator == null) {
|
||||
this.createReq();
|
||||
this.list = this.stub.scanOneShot(this.req).getDataList();
|
||||
this.iterator = this.list.iterator();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
if (!this.in) {
|
||||
return false;
|
||||
}
|
||||
if (this.iterator == null) {
|
||||
this.init();
|
||||
}
|
||||
return this.iterator.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Kv next() {
|
||||
if (this.iterator == null) {
|
||||
this.init();
|
||||
}
|
||||
return this.iterator.next();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getPageSize() {
|
||||
return this.limit;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isPageEmpty() {
|
||||
return !this.iterator.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] position() {
|
||||
return toBytes(this.session.getStoreNode().getNodeId().longValue());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void seek(byte[] position) {
|
||||
if (position == null || position.length < Long.BYTES) {
|
||||
return;
|
||||
}
|
||||
byte[] nodeIdBytes = new byte[Long.BYTES];
|
||||
System.arraycopy(position, 0, nodeIdBytes, 0, Long.BYTES);
|
||||
long nodeId = this.session.getStoreNode().getNodeId().longValue();
|
||||
long pId = toLong(nodeIdBytes);
|
||||
this.in = nodeId >= pId;
|
||||
if (this.in && nodeId == pId) {
|
||||
this.nodePosition = new byte[position.length - Long.BYTES];
|
||||
System.arraycopy(position, Long.BYTES, this.nodePosition, 0, this.nodePosition.length);
|
||||
} else {
|
||||
this.nodePosition = HgStoreClientConst.EMPTY_BYTES;
|
||||
}
|
||||
}
|
||||
|
||||
static HgOwnerKey toOk(HgOwnerKey key) {
|
||||
return key == null ? HgStoreClientConst.EMPTY_OWNER_KEY : key;
|
||||
}
|
||||
|
||||
static ByteString toBs(byte[] bytes) {
|
||||
return ByteString.copyFrom((bytes != null) ? bytes : EMPTY_BYTES);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
//TODO: implements
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,284 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import static com.baidu.hugegraph.store.HgKvStore.SCAN_ANY;
|
||||
import static com.baidu.hugegraph.store.HgKvStore.SCAN_PREFIX_BEGIN;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toBytes;
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientUtil.toLong;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.HgPageSize;
|
||||
import com.baidu.hugegraph.store.HgSeekAble;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeSession;
|
||||
import com.baidu.hugegraph.store.client.util.HgBufferProxy;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConfig;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConst;
|
||||
import com.baidu.hugegraph.store.client.util.MetricX;
|
||||
import com.baidu.hugegraph.store.grpc.common.Header;
|
||||
import com.baidu.hugegraph.store.grpc.common.Kv;
|
||||
import com.baidu.hugegraph.store.grpc.common.ScanMethod;
|
||||
import com.baidu.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamStub;
|
||||
import com.baidu.hugegraph.store.grpc.stream.KvPageRes;
|
||||
import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq;
|
||||
import com.baidu.hugegraph.store.grpc.stream.SelectParam;
|
||||
import com.google.protobuf.ByteString;
|
||||
|
||||
import io.grpc.stub.StreamObserver;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/21
|
||||
* @version 0.6.1 lynn.bond@hotamil.com on 2022/04/05
|
||||
*/
|
||||
@Slf4j
|
||||
@NotThreadSafe
|
||||
class KvPageScanner implements KvCloseableIterator<Kv>, HgPageSize, HgSeekAble {
|
||||
|
||||
private static HgStoreClientConfig clientConfig = HgStoreClientConfig.of();
|
||||
private static int nextTimeout = clientConfig.getNetKvScannerHaveNextTimeout();
|
||||
private final HgStoreNodeSession session;
|
||||
private final HgStoreStreamStub stub;
|
||||
private int pageSize = clientConfig.getNetKvScannerPageSize();
|
||||
private final AtomicBoolean completed = new AtomicBoolean(false);
|
||||
private HgBufferProxy<List<Kv>> proxy;
|
||||
private Iterator<Kv> iterator;
|
||||
private StreamObserver<ScanStreamReq> observer;
|
||||
private ScanStreamReq.Builder reqBuilder = ScanStreamReq.newBuilder();
|
||||
private SelectParam.Builder selectBuilder = SelectParam.newBuilder();
|
||||
private final BlockingQueue<ScanStreamReq> reqQueue = new LinkedBlockingQueue<>();
|
||||
private boolean in = true;
|
||||
private byte[] nodePosition = HgStoreClientConst.EMPTY_BYTES;
|
||||
|
||||
private KvPageScanner(ScanMethod scanMethod, HgStoreNodeSession session, HgStoreStreamStub stub, String table,
|
||||
HgOwnerKey prefix, HgOwnerKey startKey, HgOwnerKey endKey, long limit, int partition,
|
||||
int scanType, byte[] query) {
|
||||
this.session = session;
|
||||
this.stub = stub;
|
||||
this.pageSize = clientConfig.getNetKvScannerPageSize();
|
||||
this.reqBuilder.setHeader(this.getHeader(this.session))
|
||||
.setMethod(scanMethod)
|
||||
.setTable(table)
|
||||
.setStart(toBs(toOk(startKey).getKey()))
|
||||
.setEnd(toBs(toOk(endKey).getKey()))
|
||||
.setLimit(limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE : limit)
|
||||
.setPrefix(toBs(toOk(prefix).getKey()))
|
||||
.setCode(partition)
|
||||
.setScanType(scanType)
|
||||
.setQuery(toBs(query != null ? query : HgStoreClientConst.EMPTY_BYTES))
|
||||
.setPageSize(pageSize)
|
||||
.setPosition(toBs(this.nodePosition));
|
||||
this.init();
|
||||
}
|
||||
|
||||
public KvPageScanner(HgStoreNodeSession session, HgStoreStreamStub stub, ScanStreamReq.Builder reqBuilder) {
|
||||
this.session = session;
|
||||
this.stub = stub;
|
||||
reqBuilder.setPageSize(pageSize);
|
||||
reqBuilder.setPosition(toBs(this.nodePosition));
|
||||
this.reqBuilder = reqBuilder;
|
||||
this.init();
|
||||
}
|
||||
|
||||
public static KvCloseableIterator<Kv> scanAll(HgStoreNodeSession nodeSession, HgStoreStreamStub stub, String table,
|
||||
long limit, byte[] query) {
|
||||
return new KvPageScanner(ScanMethod.ALL, nodeSession, stub, table, null, null, null, limit,
|
||||
-1, SCAN_ANY, query);
|
||||
}
|
||||
|
||||
public static KvCloseableIterator<Kv> scanPrefix(HgStoreNodeSession nodeSession, HgStoreStreamStub stub,
|
||||
String table, HgOwnerKey prefix, long limit, byte[] query) {
|
||||
return new KvPageScanner(ScanMethod.PREFIX, nodeSession, stub, table, prefix, null, null, limit,
|
||||
prefix.getKeyCode(), SCAN_PREFIX_BEGIN, query);
|
||||
}
|
||||
|
||||
public static KvCloseableIterator<Kv> scanRange(HgStoreNodeSession nodeSession, HgStoreStreamStub stub,
|
||||
String table, HgOwnerKey startKey, HgOwnerKey endKey, long limit,
|
||||
int scanType, byte[] query) {
|
||||
return new KvPageScanner(ScanMethod.RANGE, nodeSession, stub, table, null, startKey, endKey, limit,
|
||||
startKey.getKeyCode(), scanType, query);
|
||||
}
|
||||
|
||||
private ScanStreamReq createScanReq() {
|
||||
return this.reqBuilder.setPosition(toBs(this.nodePosition)).build();
|
||||
}
|
||||
|
||||
private ScanStreamReq createStopReq() {
|
||||
return this.reqBuilder.setHeader(this.getHeader(this.session)).setCloseFlag(1).build();
|
||||
}
|
||||
|
||||
private void init() {
|
||||
this.proxy = HgBufferProxy.of(() -> this.serverScan());
|
||||
this.observer = this.stub.scan(new ServeObserverImpl());
|
||||
|
||||
}
|
||||
|
||||
/*** Server event Start ***/
|
||||
private class ServeObserverImpl implements StreamObserver<KvPageRes> {
|
||||
|
||||
@Override
|
||||
public void onNext(KvPageRes value) {
|
||||
if (value.getOver()) {
|
||||
completed.set(true);
|
||||
observer.onCompleted();
|
||||
}
|
||||
proxy.send(value.getDataList());
|
||||
if (completed.get()) {
|
||||
proxy.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(Throwable t) {
|
||||
completed.set(true);
|
||||
try {
|
||||
observer.onCompleted();
|
||||
} catch (Exception e) {
|
||||
log.warn("failed to invoke requestObserver.onCompleted(), reason:", e.getMessage());
|
||||
}
|
||||
proxy.close();
|
||||
proxy.setError(t);
|
||||
log.error("failed to complete scan of session: " + session, t);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onCompleted() {
|
||||
completed.set(true);
|
||||
proxy.close();
|
||||
}
|
||||
}
|
||||
|
||||
/*** Server Event End ***/
|
||||
|
||||
private void serverScan() {
|
||||
if (this.completed.get()) {
|
||||
this.proxy.close();
|
||||
return;
|
||||
}
|
||||
if (this.proxy.isClosed()) {
|
||||
return;
|
||||
}
|
||||
this.send(this.createScanReq());
|
||||
}
|
||||
|
||||
private void stopSever() {
|
||||
this.send(this.createStopReq());
|
||||
}
|
||||
|
||||
private void send(ScanStreamReq req) {
|
||||
if (!this.completed.get()) {
|
||||
try {
|
||||
this.observer.onNext(req);
|
||||
} catch (IllegalStateException | IllegalArgumentException e) {
|
||||
|
||||
} catch (Exception e) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void clientError(String msg) {
|
||||
this.observer.onError(GrpcUtil.toErr(msg));
|
||||
}
|
||||
|
||||
/*** Iterator ***/
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
if (!this.in) {
|
||||
return false;
|
||||
}
|
||||
if (this.iterator != null && this.iterator.hasNext()) {
|
||||
return true;
|
||||
}
|
||||
long start = 0;
|
||||
boolean debugEnabled = log.isDebugEnabled();
|
||||
if (debugEnabled) {
|
||||
start = System.nanoTime();
|
||||
}
|
||||
List<Kv> data = this.proxy.receive(nextTimeout, (sec) -> {
|
||||
String msg = "failed to receive data from net scanning, because of timeout [ " + sec + " ] sec.";
|
||||
log.error(msg);
|
||||
this.clientError(msg);
|
||||
throw new RuntimeException(msg);
|
||||
});
|
||||
if (debugEnabled) {
|
||||
MetricX.plusIteratorWait(System.nanoTime() - start);
|
||||
}
|
||||
if (data != null) {
|
||||
this.iterator = data.iterator();
|
||||
} else {
|
||||
this.iterator = Collections.emptyIterator();
|
||||
}
|
||||
return this.iterator.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Kv next() {
|
||||
if (this.iterator == null && !this.hasNext()) {
|
||||
throw new NoSuchElementException();
|
||||
}
|
||||
return this.iterator.next();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getPageSize() {
|
||||
return this.pageSize;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isPageEmpty() {
|
||||
return !this.iterator.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] position() {
|
||||
return toBytes(this.session.getStoreNode().getNodeId().longValue());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void seek(byte[] position) {
|
||||
if (position == null || position.length < Long.BYTES) {
|
||||
return;
|
||||
}
|
||||
byte[] nodeIdBytes = new byte[Long.BYTES];
|
||||
System.arraycopy(position, 0, nodeIdBytes, 0, Long.BYTES);
|
||||
long nodeId = this.session.getStoreNode().getNodeId().longValue();
|
||||
long pId = toLong(nodeIdBytes);
|
||||
this.in = nodeId >= pId;
|
||||
if (this.in && nodeId == pId) {
|
||||
this.nodePosition = new byte[position.length - Long.BYTES];
|
||||
System.arraycopy(position, Long.BYTES, this.nodePosition, 0, this.nodePosition.length);
|
||||
} else {
|
||||
this.nodePosition = HgStoreClientConst.EMPTY_BYTES;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
this.stopSever();
|
||||
}
|
||||
|
||||
/*** commons ***/
|
||||
private Header getHeader(HgStoreNodeSession nodeSession) {
|
||||
return Header.newBuilder().setGraph(nodeSession.getGraphName()).build();
|
||||
}
|
||||
|
||||
static HgOwnerKey toOk(HgOwnerKey key) {
|
||||
return key == null ? HgStoreClientConst.EMPTY_OWNER_KEY : key;
|
||||
}
|
||||
|
||||
static ByteString toBs(byte[] bytes) {
|
||||
return ByteString.copyFrom((bytes != null) ? bytes : EMPTY_BYTES);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,229 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeManager;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeSession;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNotice;
|
||||
import com.baidu.hugegraph.store.client.type.HgNodeStatus;
|
||||
import com.baidu.hugegraph.store.client.type.HgStoreClientException;
|
||||
import com.baidu.hugegraph.store.grpc.common.ResStatus;
|
||||
import com.baidu.hugegraph.store.grpc.session.FeedbackRes;
|
||||
import com.baidu.hugegraph.store.grpc.session.PartitionFaultResponse;
|
||||
import com.baidu.hugegraph.store.grpc.session.PartitionFaultType;
|
||||
import com.baidu.hugegraph.store.grpc.session.PartitionLeader;
|
||||
import com.google.protobuf.util.JsonFormat;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2021/11/18
|
||||
* @version 0.3.0 on 2022/01/27
|
||||
*/
|
||||
@Slf4j
|
||||
final class NotifyingExecutor {
|
||||
private String graphName;
|
||||
private HgStoreNodeManager nodeManager;
|
||||
private HgStoreNodeSession nodeSession;
|
||||
|
||||
private Map<PartitionFaultType, Consumer<PartitionFaultResponse>> partitionFaultHandlers;
|
||||
|
||||
NotifyingExecutor(String graphName, HgStoreNodeManager nodeManager, HgStoreNodeSession nodeSession) {
|
||||
this.graphName = graphName;
|
||||
this.nodeManager = nodeManager;
|
||||
this.nodeSession = nodeSession;
|
||||
}
|
||||
|
||||
private void intHandler() {
|
||||
this.partitionFaultHandlers = new HashMap<>();
|
||||
|
||||
this.partitionFaultHandlers.put(
|
||||
PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LEADER, notifyPartitionLeaderConsumer()
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
<T> Optional<T> invoke(Supplier<FeedbackRes> supplier, Function<FeedbackRes, T> okFunction) {
|
||||
FeedbackRes res = null;
|
||||
|
||||
try {
|
||||
res = supplier.get();
|
||||
} catch (Throwable t) {
|
||||
log.error("Failed to invoke: " + supplier.toString() + ", caused " +
|
||||
"by:", t);
|
||||
handleErr(t);
|
||||
throw err(t);
|
||||
}
|
||||
|
||||
if (log.isDebugEnabled()) log.debug("gRPC [{}] status: {}"
|
||||
, this.nodeSession.getStoreNode().getAddress(), res.getStatus().getCode());
|
||||
|
||||
Optional<T> option = null;
|
||||
|
||||
switch (res.getStatus().getCode()) {
|
||||
case RES_CODE_OK:
|
||||
option = Optional.of(okFunction.apply(res));
|
||||
break;
|
||||
case RES_CODE_FAIL:
|
||||
handleFail(res);
|
||||
break;
|
||||
case RES_CODE_NOT_EXIST:
|
||||
break;
|
||||
case RES_CODE_EXCESS:
|
||||
normalFail(res);
|
||||
break;
|
||||
default:
|
||||
log.error("gRPC [{}] status-msg: {}"
|
||||
, nodeSession.getStoreNode().getAddress(), res.getStatus().getMsg());
|
||||
}
|
||||
|
||||
if (option == null) {
|
||||
option = Optional.empty();
|
||||
}
|
||||
|
||||
return option;
|
||||
}
|
||||
|
||||
private void handleErr(Throwable t) {
|
||||
try {
|
||||
notifyErrConsumer(HgNodeStatus.NOT_WORK).accept(t);
|
||||
} catch (Throwable tt) {
|
||||
log.error("Failed to notify error to HgStoreNodeNotifier, cause:", tt);
|
||||
}
|
||||
}
|
||||
|
||||
private void handleFail(FeedbackRes feedbackRes) {
|
||||
Supplier<HgStoreClientException> exSup;
|
||||
|
||||
if (
|
||||
(exSup = handlePartitionFault(feedbackRes)) != null
|
||||
// add more fault-handler here.
|
||||
|| (exSup = defaultExceptionSupplier(feedbackRes)) != null
|
||||
) {
|
||||
throw exSup.get();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void normalFail(FeedbackRes res) {
|
||||
ResStatus status = res.getStatus();
|
||||
HgStoreClientException ex;
|
||||
try {
|
||||
String msg = JsonFormat.printer().omittingInsignificantWhitespace()
|
||||
.print(res);
|
||||
ex = err(msg);
|
||||
} catch (Exception e) {
|
||||
ex = err(status.getCode() + ", " + status.getMsg());
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
|
||||
private Supplier<HgStoreClientException> defaultExceptionSupplier(FeedbackRes feedbackRes) {
|
||||
return () -> HgStoreClientException.of(err(feedbackRes.getStatus().getMsg()));
|
||||
}
|
||||
|
||||
private Supplier<HgStoreClientException> handlePartitionFault(
|
||||
FeedbackRes feedbackRes) {
|
||||
PartitionFaultResponse res = feedbackRes.getPartitionFaultResponse();
|
||||
if (res == null) {
|
||||
return null;
|
||||
}
|
||||
if (this.partitionFaultHandlers == null) {
|
||||
intHandler();
|
||||
}
|
||||
Consumer<PartitionFaultResponse> consumer =
|
||||
this.partitionFaultHandlers.get(res.getFaultType());
|
||||
if (consumer == null) {
|
||||
consumer = notifyPartitionConsumer();
|
||||
}
|
||||
String msg = res.toString();
|
||||
if (msg == null || msg.length() == 0) {
|
||||
msg = feedbackRes.getStatus().getMsg();
|
||||
}
|
||||
consumer.accept(res);
|
||||
String finalMsg = msg;
|
||||
return () -> HgStoreClientException.of(
|
||||
err(res.getFaultType() + ", " +
|
||||
finalMsg));
|
||||
}
|
||||
|
||||
private HgStoreClientException err(String msg) {
|
||||
return err(msg, null);
|
||||
}
|
||||
|
||||
private HgStoreClientException err(Throwable t) {
|
||||
return err(t.getMessage(), t);
|
||||
}
|
||||
|
||||
private HgStoreClientException err(String reason, Throwable t) {
|
||||
StringBuilder builder = new StringBuilder().append(
|
||||
"{sessionInfo: {" + this.nodeSession.toString() +
|
||||
"}, reason: ");
|
||||
if (reason.startsWith("{")) {
|
||||
builder.append(reason);
|
||||
} else {
|
||||
builder.append("\"").append(reason).append("\"");
|
||||
}
|
||||
String msg = builder.append("}").toString();
|
||||
if (t != null) {
|
||||
return HgStoreClientException.of(msg, t);
|
||||
}
|
||||
return HgStoreClientException.of(msg);
|
||||
}
|
||||
|
||||
private Consumer<PartitionFaultResponse> notifyPartitionLeaderConsumer() {
|
||||
return res -> {
|
||||
log.info("partitions' leader have changed: [partitionId - leaderId] ");
|
||||
nodeManager.notifying(
|
||||
this.graphName,
|
||||
HgStoreNotice.of(this.nodeSession.getStoreNode().getNodeId(), HgNodeStatus.NOT_PARTITION_LEADER)
|
||||
.setPartitionLeaders(
|
||||
res.getPartitionLeadersList()
|
||||
.stream()
|
||||
.peek((e) -> {
|
||||
log.info("[{} - {}]", e.getPartitionId(), e.getLeaderId());
|
||||
}
|
||||
)
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
PartitionLeader::getPartitionId,
|
||||
PartitionLeader::getLeaderId
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
private Consumer<PartitionFaultResponse> notifyPartitionConsumer() {
|
||||
return notifyPartitionConsumer(HgNodeStatus.PARTITION_COMMON_FAULT);
|
||||
}
|
||||
|
||||
private Consumer<PartitionFaultResponse> notifyPartitionConsumer(HgNodeStatus status) {
|
||||
return res -> {
|
||||
nodeManager.notifying(
|
||||
this.graphName,
|
||||
HgStoreNotice.of(this.nodeSession.getStoreNode().getNodeId(), status)
|
||||
.setPartitionIds(res.getPartitionIdsList())
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
private Consumer<Throwable> notifyErrConsumer(HgNodeStatus status) {
|
||||
return t -> {
|
||||
nodeManager.notifying(
|
||||
this.graphName,
|
||||
HgStoreNotice.of(this.nodeSession.getStoreNode().getNodeId(), status, t.getMessage())
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import static com.baidu.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES;
|
||||
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.client.HgStoreNodeSession;
|
||||
import com.baidu.hugegraph.store.client.util.HgStoreClientConst;
|
||||
import com.baidu.hugegraph.store.grpc.common.Header;
|
||||
import com.google.protobuf.ByteString;
|
||||
|
||||
/**
|
||||
* @author zhangyingjie
|
||||
* @date 2022/10/8
|
||||
**/
|
||||
public class ScanUtil {
|
||||
|
||||
|
||||
public static Header getHeader(HgStoreNodeSession nodeSession) {
|
||||
return Header.newBuilder().setGraph(nodeSession.getGraphName()).build();
|
||||
}
|
||||
|
||||
public static HgOwnerKey toOk(HgOwnerKey key) {
|
||||
return key == null ? HgStoreClientConst.EMPTY_OWNER_KEY : key;
|
||||
}
|
||||
|
||||
public static ByteString toBs(byte[] bytes) {
|
||||
return ByteString.copyFrom((bytes != null) ? bytes : EMPTY_BYTES);
|
||||
}
|
||||
|
||||
public static ByteString getHgOwnerKey(HgOwnerKey ownerKey) {
|
||||
return toBs(toOk(ownerKey).getKey());
|
||||
}
|
||||
|
||||
public static byte[] getQuery(byte[] query) {
|
||||
return query != null ? query : HgStoreClientConst.EMPTY_BYTES;
|
||||
}
|
||||
public static long getLimit(long limit) {
|
||||
return limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE : limit;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import com.baidu.hugegraph.store.HgSeekAble;
|
||||
import com.baidu.hugegraph.store.client.util.HgAssert;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2022/3/11
|
||||
*/
|
||||
class SeekAbleIterator<E> implements Iterator, HgSeekAble {
|
||||
private Iterator<E> iterator;
|
||||
private Consumer<byte[]> seeker;
|
||||
private Supplier<byte[]> positioner;
|
||||
|
||||
public static <E> SeekAbleIterator of(Iterator<E> iterator,Supplier<byte[]> positioner, Consumer<byte[]> seeker) {
|
||||
HgAssert.isArgumentNotNull(iterator, "iterator");
|
||||
HgAssert.isArgumentNotNull(positioner, "positioner");
|
||||
HgAssert.isArgumentNotNull(seeker, "seeker");
|
||||
return new SeekAbleIterator(iterator,positioner, seeker);
|
||||
}
|
||||
|
||||
private SeekAbleIterator(Iterator<E> iterator, Supplier<byte[]> positioner,Consumer<byte[]> seeker) {
|
||||
this.iterator = iterator;
|
||||
this.positioner=positioner;
|
||||
this.seeker = seeker;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] position() {
|
||||
return this.positioner.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void seek(byte[] position) {
|
||||
this.seeker.accept(position);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return this.iterator.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public E next() {
|
||||
return this.iterator.next();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
package com.baidu.hugegraph.store.client.grpc;
|
||||
|
||||
import javax.annotation.concurrent.ThreadSafe;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2021/12/1
|
||||
*/
|
||||
@ThreadSafe
|
||||
final class SwitchingExecutor {
|
||||
|
||||
static SwitchingExecutor of(){
|
||||
return new SwitchingExecutor();
|
||||
}
|
||||
|
||||
private SwitchingExecutor(){}
|
||||
|
||||
<T> Optional<T> invoke(Supplier<Boolean> switcher,Supplier<T> trueSupplier,Supplier<T> falseSupplier){
|
||||
Optional<T> option = null;
|
||||
|
||||
if(switcher.get()){
|
||||
option= Optional.of(trueSupplier.get());
|
||||
}else{
|
||||
option=Optional.of(falseSupplier.get());
|
||||
}
|
||||
if (option == null) {
|
||||
option = Optional.empty();
|
||||
}
|
||||
|
||||
return option;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
package com.baidu.hugegraph.store.client.type;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/26
|
||||
*/
|
||||
public enum HgNodeStatus {
|
||||
|
||||
UNKNOWN(0, "UNKNOWN"),
|
||||
NOT_EXIST(100,"NOT_EXIST"), // Failed to apply for an instance via node-id from NodeManager.
|
||||
NOT_ONLINE(105,"NOT_ONLINE"), // Failed to connect to Store-Node at the first time.
|
||||
NOT_WORK(110,"NOT_WORK"), // When a Store-Node to be not work anymore.
|
||||
|
||||
PARTITION_COMMON_FAULT(200,"PARTITION_COMMON_FAULT"), //
|
||||
NOT_PARTITION_LEADER(205,"NOT_PARTITION_LEADER") // When a Store-Node is not a specific partition leader.
|
||||
|
||||
;
|
||||
|
||||
private int status;
|
||||
private String name;
|
||||
|
||||
private HgNodeStatus(int status,String name){
|
||||
this.status=status;
|
||||
this.name=name;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
package com.baidu.hugegraph.store.client.type;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/27
|
||||
*/
|
||||
public class HgStoreClientException extends RuntimeException {
|
||||
|
||||
public static HgStoreClientException of(String msg) {
|
||||
return new HgStoreClientException(msg);
|
||||
}
|
||||
public static HgStoreClientException of(String msg,Throwable cause) {
|
||||
return new HgStoreClientException(msg,cause);
|
||||
}
|
||||
|
||||
public static HgStoreClientException of(Throwable cause) {
|
||||
return new HgStoreClientException(cause);
|
||||
}
|
||||
|
||||
public HgStoreClientException(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
|
||||
public HgStoreClientException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
|
||||
public HgStoreClientException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
package com.baidu.hugegraph.store.client.util;
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.math.BigInteger;
|
||||
|
||||
/**
|
||||
* Created with IntelliJ IDEA.
|
||||
* User: noah
|
||||
* Date: 8/2/13
|
||||
* Time: 10:36 AM
|
||||
* To change this template use File | Settings | File Templates.
|
||||
*/
|
||||
public class Base58 {
|
||||
|
||||
public static final char[] ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray();
|
||||
private static final int[] INDEXES = new int[128];
|
||||
|
||||
static {
|
||||
for (int i = 0; i < INDEXES.length; i++) {
|
||||
INDEXES[i] = -1;
|
||||
}
|
||||
for (int i = 0; i < ALPHABET.length; i++) {
|
||||
INDEXES[ALPHABET[i]] = i;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes the given bytes in base58. No checksum is appended.
|
||||
*/
|
||||
public static String encode(byte[] input) {
|
||||
if (input.length == 0) {
|
||||
return "";
|
||||
}
|
||||
input = copyOfRange(input, 0, input.length);
|
||||
// Count leading zeroes.
|
||||
int zeroCount = 0;
|
||||
while (zeroCount < input.length && input[zeroCount] == 0) {
|
||||
++zeroCount;
|
||||
}
|
||||
// The actual encoding.
|
||||
byte[] temp = new byte[input.length * 2];
|
||||
int j = temp.length;
|
||||
|
||||
int startAt = zeroCount;
|
||||
while (startAt < input.length) {
|
||||
byte mod = divmod58(input, startAt);
|
||||
if (input[startAt] == 0) {
|
||||
++startAt;
|
||||
}
|
||||
temp[--j] = (byte) ALPHABET[mod];
|
||||
}
|
||||
|
||||
// Strip extra '1' if there are some after decoding.
|
||||
while (j < temp.length && temp[j] == ALPHABET[0]) {
|
||||
++j;
|
||||
}
|
||||
// Add as many leading '1' as there were leading zeros.
|
||||
while (--zeroCount >= 0) {
|
||||
temp[--j] = (byte) ALPHABET[0];
|
||||
}
|
||||
|
||||
byte[] output = copyOfRange(temp, j, temp.length);
|
||||
try {
|
||||
return new String(output, "US-ASCII");
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new RuntimeException(e); // Cannot happen.
|
||||
}
|
||||
}
|
||||
|
||||
public static byte[] decode(String input) throws IllegalArgumentException {
|
||||
if (input.length() == 0) {
|
||||
return new byte[0];
|
||||
}
|
||||
byte[] input58 = new byte[input.length()];
|
||||
// Transform the String to a base58 byte sequence
|
||||
for (int i = 0; i < input.length(); ++i) {
|
||||
char c = input.charAt(i);
|
||||
|
||||
int digit58 = -1;
|
||||
if (c >= 0 && c < 128) {
|
||||
digit58 = INDEXES[c];
|
||||
}
|
||||
if (digit58 < 0) {
|
||||
throw new IllegalArgumentException("Illegal character " + c + " at " + i);
|
||||
}
|
||||
|
||||
input58[i] = (byte) digit58;
|
||||
}
|
||||
// Count leading zeroes
|
||||
int zeroCount = 0;
|
||||
while (zeroCount < input58.length && input58[zeroCount] == 0) {
|
||||
++zeroCount;
|
||||
}
|
||||
// The encoding
|
||||
byte[] temp = new byte[input.length()];
|
||||
int j = temp.length;
|
||||
|
||||
int startAt = zeroCount;
|
||||
while (startAt < input58.length) {
|
||||
byte mod = divmod256(input58, startAt);
|
||||
if (input58[startAt] == 0) {
|
||||
++startAt;
|
||||
}
|
||||
|
||||
temp[--j] = mod;
|
||||
}
|
||||
// Do no add extra leading zeroes, move j to first non null byte.
|
||||
while (j < temp.length && temp[j] == 0) {
|
||||
++j;
|
||||
}
|
||||
|
||||
return copyOfRange(temp, j - zeroCount, temp.length);
|
||||
}
|
||||
|
||||
public static BigInteger decodeToBigInteger(String input) throws IllegalArgumentException {
|
||||
return new BigInteger(1, decode(input));
|
||||
}
|
||||
|
||||
//
|
||||
// number -> number / 58, returns number % 58
|
||||
//
|
||||
private static byte divmod58(byte[] number, int startAt) {
|
||||
int remainder = 0;
|
||||
for (int i = startAt; i < number.length; i++) {
|
||||
int digit256 = (int) number[i] & 0xFF;
|
||||
int temp = remainder * 256 + digit256;
|
||||
|
||||
number[i] = (byte) (temp / 58);
|
||||
|
||||
remainder = temp % 58;
|
||||
}
|
||||
|
||||
return (byte) remainder;
|
||||
}
|
||||
|
||||
//
|
||||
// number -> number / 256, returns number % 256
|
||||
//
|
||||
private static byte divmod256(byte[] number58, int startAt) {
|
||||
int remainder = 0;
|
||||
for (int i = startAt; i < number58.length; i++) {
|
||||
int digit58 = (int) number58[i] & 0xFF;
|
||||
int temp = remainder * 58 + digit58;
|
||||
|
||||
number58[i] = (byte) (temp / 256);
|
||||
|
||||
remainder = temp % 256;
|
||||
}
|
||||
|
||||
return (byte) remainder;
|
||||
}
|
||||
|
||||
private static byte[] copyOfRange(byte[] source, int from, int to) {
|
||||
byte[] range = new byte[to - from];
|
||||
System.arraycopy(source, from, range, 0, range.length);
|
||||
|
||||
return range;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
package com.baidu.hugegraph.store.client.util;
|
||||
|
||||
import java.util.concurrent.SynchronousQueue;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2021/11/22
|
||||
*/
|
||||
@Slf4j
|
||||
public final class ExecutorPool {
|
||||
|
||||
public static ThreadFactory newThreadFactory(String namePrefix, int priority) {
|
||||
HgAssert.isArgumentNotNull(namePrefix, "namePrefix");
|
||||
return new HgThreadFactory(namePrefix, priority);
|
||||
}
|
||||
|
||||
public static ThreadFactory newThreadFactory(String namePrefix) {
|
||||
HgAssert.isArgumentNotNull(namePrefix, "namePrefix");
|
||||
return new HgDefaultThreadFactory(namePrefix);
|
||||
}
|
||||
|
||||
public static ThreadPoolExecutor createExecutor(String name, long keepAliveTime, int coreThreads, int maxThreads) {
|
||||
return new ThreadPoolExecutor(coreThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS,
|
||||
new SynchronousQueue<>(),
|
||||
newThreadFactory(name), new ThreadPoolExecutor.CallerRunsPolicy()
|
||||
);
|
||||
}
|
||||
|
||||
//
|
||||
// private final static ExecutorService executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE,
|
||||
// 10L, TimeUnit.SECONDS,
|
||||
// new LinkedBlockingQueue<>(),
|
||||
// new HgDefaultThreadFactory
|
||||
// ("store-common"));
|
||||
// private final static ExecutorService grpcExecutor = createExecutor("store-grpc", 60L, 600, 10240);
|
||||
//// public final static ExecutorService scannerExecutor = createExecutor("scanner", 10l, 200, 1024);
|
||||
//
|
||||
// static {
|
||||
// Thread hook = new Thread(() -> {
|
||||
// try {
|
||||
// executor.shutdown();
|
||||
// executor.awaitTermination(30, TimeUnit.SECONDS);
|
||||
// return;
|
||||
// } catch (InterruptedException e) {
|
||||
// log.error("failed to await executorService.shutdown()", e);
|
||||
// }
|
||||
// executor.shutdownNow().forEach(r -> log.error("not run task:" + r.toString()));
|
||||
// try {
|
||||
// executor.awaitTermination(30, TimeUnit.SECONDS);
|
||||
// } catch (InterruptedException e) {
|
||||
// log.error("failed to await executorService.shutdownNow()", e);
|
||||
// throw HgStoreClientException.of(e);
|
||||
// }
|
||||
// }
|
||||
// );
|
||||
// Runtime.getRuntime().addShutdownHook(hook);
|
||||
//}
|
||||
//
|
||||
//// public static void execute(Runnable command) {
|
||||
//// isArgumentNotNull(command, "command");
|
||||
//// executor.execute(command);
|
||||
//// }
|
||||
////
|
||||
// public static ExecutorService getGrpcCommonExecutor() {
|
||||
// return grpcExecutor;
|
||||
// }
|
||||
//
|
||||
|
||||
/**
|
||||
* The default thread factory
|
||||
*/
|
||||
static class HgThreadFactory implements ThreadFactory {
|
||||
private final AtomicInteger threadNumber = new AtomicInteger(1);
|
||||
private final String namePrefix;
|
||||
private final int priority;
|
||||
|
||||
HgThreadFactory(String namePrefix, int priority) {
|
||||
this.namePrefix = namePrefix;
|
||||
this.priority = priority;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Thread newThread(Runnable r) {
|
||||
Thread t = new Thread(null, r, namePrefix + "-" + threadNumber.getAndIncrement(), 0);
|
||||
if (t.isDaemon()) {
|
||||
t.setDaemon(false);
|
||||
}
|
||||
if (t.getPriority() != priority) {
|
||||
t.setPriority(priority);
|
||||
}
|
||||
return t;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The default thread factory, which added threadNamePrefix in construction method.
|
||||
*/
|
||||
static class HgDefaultThreadFactory implements ThreadFactory {
|
||||
private static AtomicInteger poolNumber = new AtomicInteger(1);
|
||||
private final AtomicInteger threadNumber = new AtomicInteger(1);
|
||||
private final String namePrefix;
|
||||
|
||||
HgDefaultThreadFactory(String threadNamePrefix) {
|
||||
this.namePrefix = threadNamePrefix + "-" + poolNumber.getAndIncrement() + "-thread-";
|
||||
}
|
||||
|
||||
@Override
|
||||
public Thread newThread(Runnable r) {
|
||||
Thread t = new Thread(null, r, namePrefix + threadNumber.getAndIncrement(), 0);
|
||||
if (t.isDaemon()) {
|
||||
t.setDaemon(false);
|
||||
}
|
||||
if (t.getPriority() != Thread.NORM_PRIORITY) {
|
||||
t.setPriority(Thread.NORM_PRIORITY);
|
||||
}
|
||||
return t;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
package com.baidu.hugegraph.store.client.util;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com
|
||||
*/
|
||||
public final class HgAssert {
|
||||
public static void isTrue(boolean expression, String message) {
|
||||
if (message == null) throw new IllegalArgumentException("message is null");
|
||||
if (!expression) {
|
||||
throw new IllegalArgumentException(message);
|
||||
}
|
||||
}
|
||||
|
||||
public static void isTrue(boolean expression, Supplier<String> msg) {
|
||||
if (msg == null) {
|
||||
throw new IllegalArgumentException("message supplier is null");
|
||||
}
|
||||
if (!expression) {
|
||||
throw new IllegalArgumentException(msg.get());
|
||||
}
|
||||
}
|
||||
|
||||
public static void isFalse(boolean expression, String message) {
|
||||
isTrue(!expression, message);
|
||||
}
|
||||
|
||||
public static void isFalse(boolean expression, Supplier<String> msg) {
|
||||
isTrue(!expression, msg);
|
||||
}
|
||||
|
||||
public static void isArgumentValid(byte[] bytes, String parameter) {
|
||||
isFalse(isInvalid(bytes), () -> "The argument is invalid: " + parameter);
|
||||
}
|
||||
|
||||
public static void isArgumentValid(String str, String parameter) {
|
||||
isFalse(isInvalid(str), () -> "The argument is invalid: " + parameter);
|
||||
}
|
||||
|
||||
public static void isArgumentValid(Collection<?> collection, String parameter) {
|
||||
isFalse(isInvalid(collection), () -> "The argument is invalid: " + parameter);
|
||||
}
|
||||
|
||||
public static void isArgumentNotNull(Object obj, String parameter) {
|
||||
isTrue(obj != null, () -> "The argument is null: " + parameter);
|
||||
}
|
||||
|
||||
public static void istValid(byte[] bytes, String msg) {
|
||||
isFalse(isInvalid(bytes), msg);
|
||||
}
|
||||
|
||||
public static void isValid(String str, String msg) {
|
||||
isFalse(isInvalid(str), msg);
|
||||
}
|
||||
|
||||
public static void isNotNull(Object obj, String msg) {
|
||||
isTrue(obj != null, msg);
|
||||
}
|
||||
|
||||
public static boolean isContains(Object[] objs, Object obj) {
|
||||
if (objs == null || objs.length == 0 || obj == null) return false;
|
||||
for (Object item : objs) {
|
||||
if (obj.equals(item)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static boolean isInvalid(String... strs) {
|
||||
if (strs == null || strs.length == 0) return true;
|
||||
for (String item : strs) {
|
||||
if (item == null || "".equals(item.trim())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static boolean isInvalid(byte[] bytes) {
|
||||
if (bytes == null || bytes.length == 0) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
public static boolean isInvalid(Map<?, ?> map) {
|
||||
if (map == null || map.isEmpty()) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
public static boolean isInvalid(Collection<?> list) {
|
||||
if (list == null || list.isEmpty()) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
public static <T> boolean isContains(Collection<T> list, T item) {
|
||||
if (list == null || item == null) return false;
|
||||
return list.contains(item);
|
||||
}
|
||||
|
||||
public static boolean isNull(Object... objs) {
|
||||
if (objs == null) return true;
|
||||
for (Object item : objs) {
|
||||
if (item == null) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
package com.baidu.hugegraph.store.client.util;
|
||||
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import javax.annotation.CheckForNull;
|
||||
|
||||
import com.baidu.hugegraph.store.client.type.HgStoreClientException;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2022/3/15
|
||||
* @version 0.1.0
|
||||
*/
|
||||
@Slf4j
|
||||
public final class HgBufferProxy<T> {
|
||||
private final BlockingQueue<Supplier<T>> queue;
|
||||
private final AtomicBoolean closed = new AtomicBoolean(false);
|
||||
private final ReentrantLock lock = new ReentrantLock();
|
||||
private final Runnable task;
|
||||
private Throwable err;
|
||||
|
||||
public static HgBufferProxy of(Runnable task) {
|
||||
HgAssert.isArgumentNotNull(task, "task");
|
||||
return new HgBufferProxy(task);
|
||||
}
|
||||
|
||||
private HgBufferProxy(Runnable task) {
|
||||
this.task = task;
|
||||
this.queue = new LinkedBlockingQueue<>();
|
||||
}
|
||||
|
||||
public void send(T t) {
|
||||
if (t == null) {
|
||||
throw new IllegalArgumentException("the argument t is null");
|
||||
}
|
||||
if (this.closed.get()) {
|
||||
return;
|
||||
}
|
||||
this.lock.lock();
|
||||
try {
|
||||
this.queue.offer(() -> t);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void apply() {
|
||||
this.lock.lock();
|
||||
try {
|
||||
if (!this.closed.get()) {
|
||||
this.task.run();
|
||||
Thread.yield();
|
||||
}
|
||||
} finally {
|
||||
this.lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* return an item from the chan
|
||||
*
|
||||
* @return null when the chan has been closed
|
||||
* @throws RuntimeException
|
||||
*/
|
||||
@CheckForNull
|
||||
public T receive(int time, Consumer<Integer> callback) {
|
||||
Supplier<T> s;
|
||||
if (this.closed.get()) {
|
||||
s = this.queue.poll();
|
||||
this.checkErr();
|
||||
return s != null ? s.get() : null;
|
||||
}
|
||||
if (this.queue.size() <= 1) {
|
||||
this.apply();
|
||||
}
|
||||
lock.lock();
|
||||
try {
|
||||
if (this.isClosed()) {
|
||||
s = this.queue.poll();
|
||||
this.checkErr();
|
||||
return s != null ? s.get() : null;
|
||||
}
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
try {
|
||||
s = this.queue.poll(time, TimeUnit.SECONDS);
|
||||
} catch (Throwable t) {
|
||||
log.error("failed to receive a item from channel, cause by: ", t);
|
||||
throw HgStoreClientException.of(t);
|
||||
}
|
||||
if (s == null) {
|
||||
if (this.closed.get()) {
|
||||
s = this.queue.poll();
|
||||
} else {
|
||||
if (callback == null) {
|
||||
throw new RuntimeException("timeout, max time: " + time + " seconds");
|
||||
} else {
|
||||
callback.accept(time);
|
||||
}
|
||||
}
|
||||
}
|
||||
this.checkErr();
|
||||
return s != null ? s.get() : null;
|
||||
}
|
||||
|
||||
public boolean isClosed() {
|
||||
return this.closed.get();
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws RuntimeException when fail to close the chan
|
||||
*/
|
||||
public void close() {
|
||||
if (this.closed.get()) {
|
||||
return;
|
||||
}
|
||||
lock.lock();
|
||||
this.closed.set(true);
|
||||
try {
|
||||
this.queue.offer(() -> null);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public void setError(Throwable streamErr) {
|
||||
this.err = streamErr;
|
||||
}
|
||||
|
||||
private void checkErr() {
|
||||
if (this.err != null) {
|
||||
throw HgStoreClientException.of(this.err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,175 @@
|
|||
package com.baidu.hugegraph.store.client.util;
|
||||
|
||||
import java.util.PropertyResourceBundle;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com on 2021/11/29
|
||||
*/
|
||||
@Slf4j
|
||||
public final class HgStoreClientConfig {
|
||||
|
||||
private static final int GRPC_DEFAULT_TIMEOUT_SECONDS = 100;
|
||||
private static final int GRPC_DEFAULT_MAX_INBOUND_MESSAGE_SIZE = 1024 * 1024 * 1024;
|
||||
private static final int GRPC_DEFAULT_MAX_OUTBOUND_MESSAGE_SIZE = 1024 * 1024 * 1024;
|
||||
|
||||
private static final int NET_KV_SCANNER_PAGE_SIZE = 10_000;
|
||||
private static final int NET_KV_SCANNER_HAVE_NEXT_TIMEOUT = 30 * 60;
|
||||
|
||||
private Integer grpcTimeoutSeconds = GRPC_DEFAULT_TIMEOUT_SECONDS;
|
||||
private Integer grpcMaxInboundMessageSize = GRPC_DEFAULT_MAX_INBOUND_MESSAGE_SIZE;
|
||||
private Integer grpcMaxOutboundMessageSize = GRPC_DEFAULT_MAX_OUTBOUND_MESSAGE_SIZE;
|
||||
|
||||
private Integer netKvScannerPageSize = NET_KV_SCANNER_PAGE_SIZE;
|
||||
private Integer netKvScannerHaveNextTimeout = NET_KV_SCANNER_HAVE_NEXT_TIMEOUT;
|
||||
|
||||
private static PropertyResourceBundle prb = null;
|
||||
private static String fileName = "hg-store-client";
|
||||
|
||||
private static HgStoreClientConfig defaultInstance;
|
||||
|
||||
|
||||
public synchronized static HgStoreClientConfig of() {
|
||||
|
||||
if (defaultInstance != null) {
|
||||
return defaultInstance;
|
||||
}
|
||||
|
||||
defaultInstance = new HgStoreClientConfig();
|
||||
|
||||
overrideViaProperties(defaultInstance);
|
||||
|
||||
return defaultInstance;
|
||||
}
|
||||
|
||||
private static void overrideViaProperties(HgStoreClientConfig config) {
|
||||
try {
|
||||
prb = (PropertyResourceBundle) PropertyResourceBundle.getBundle(fileName);
|
||||
} catch (Throwable t) {
|
||||
log.warn("Failed to load " + fileName + ".properties.");
|
||||
log.info("Default configuration was activated.");
|
||||
return;
|
||||
}
|
||||
PropertiesWrapper wrapper = new PropertiesWrapper(prb);
|
||||
|
||||
log.info("grpc.timeout.seconds = "
|
||||
+ (config.grpcTimeoutSeconds = wrapper.getInt("grpc.timeout.seconds"
|
||||
, config.grpcTimeoutSeconds))
|
||||
);
|
||||
log.info("grpc.max.inbound.message.size = "
|
||||
+ (config.grpcMaxInboundMessageSize = GRPC_DEFAULT_MAX_INBOUND_MESSAGE_SIZE)
|
||||
);
|
||||
log.info("grpc.max.outbound.message.size = "
|
||||
+ (config.grpcMaxOutboundMessageSize = GRPC_DEFAULT_MAX_OUTBOUND_MESSAGE_SIZE)
|
||||
);
|
||||
log.info("net.kv.scanner.page.size = "
|
||||
+ (config.netKvScannerPageSize = wrapper.getInt("net.kv.scanner.page.size"
|
||||
, config.netKvScannerPageSize))
|
||||
);
|
||||
log.info("net.kv.scanner.have.next.timeout = {}", config.netKvScannerHaveNextTimeout);
|
||||
}
|
||||
|
||||
private HgStoreClientConfig() {
|
||||
}
|
||||
|
||||
public Integer getGrpcTimeoutSeconds() {
|
||||
return grpcTimeoutSeconds;
|
||||
}
|
||||
|
||||
public Integer getGrpcMaxInboundMessageSize() {
|
||||
return grpcMaxInboundMessageSize;
|
||||
}
|
||||
|
||||
public Integer getGrpcMaxOutboundMessageSize() {
|
||||
return grpcMaxOutboundMessageSize;
|
||||
}
|
||||
|
||||
public Integer getNetKvScannerPageSize() {
|
||||
return netKvScannerPageSize;
|
||||
}
|
||||
|
||||
public Integer getNetKvScannerHaveNextTimeout() {
|
||||
return netKvScannerHaveNextTimeout;
|
||||
}
|
||||
|
||||
public HgStoreClientConfig setGrpcTimeoutSeconds(Integer grpcTimeoutSeconds) {
|
||||
this.grpcTimeoutSeconds = grpcTimeoutSeconds;
|
||||
return this;
|
||||
}
|
||||
|
||||
public HgStoreClientConfig setGrpcMaxInboundMessageSize(Integer grpcMaxInboundMessageSize) {
|
||||
this.grpcMaxInboundMessageSize = grpcMaxInboundMessageSize;
|
||||
return this;
|
||||
}
|
||||
|
||||
public HgStoreClientConfig setGrpcMaxOutboundMessageSize(Integer grpcMaxOutboundMessageSize) {
|
||||
this.grpcMaxOutboundMessageSize = grpcMaxOutboundMessageSize;
|
||||
return this;
|
||||
}
|
||||
|
||||
public HgStoreClientConfig setNetKvScannerPageSize(Integer netKvScannerPageSize) {
|
||||
this.netKvScannerPageSize = netKvScannerPageSize;
|
||||
return this;
|
||||
}
|
||||
|
||||
public HgStoreClientConfig setNetKvScannerHaveNextTimeout(Integer netKvScannerHaveNextTimeout) {
|
||||
this.netKvScannerHaveNextTimeout = netKvScannerHaveNextTimeout;
|
||||
return this;
|
||||
}
|
||||
|
||||
private static class PropertiesWrapper {
|
||||
private PropertyResourceBundle prb;
|
||||
|
||||
PropertiesWrapper(PropertyResourceBundle prb) {
|
||||
this.prb = prb;
|
||||
}
|
||||
|
||||
Integer getInt(String key, Integer defaultValue) {
|
||||
|
||||
String buf = this.getStr(key);
|
||||
if (buf == null || buf.isEmpty()) return defaultValue;
|
||||
|
||||
Integer res = null;
|
||||
try {
|
||||
res = Integer.valueOf(buf);
|
||||
} catch (Throwable t) {
|
||||
log.error("Failed to parse a int value[ " + buf + " ] of the key[ " + key + " ].", t);
|
||||
}
|
||||
|
||||
return res;
|
||||
|
||||
}
|
||||
|
||||
String getStr(String key, String defaultValue) {
|
||||
String res = getStr(key);
|
||||
|
||||
if (res == null && defaultValue != null) {
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
String getStr(String key) {
|
||||
String value = null;
|
||||
|
||||
if (!prb.containsKey(key)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
value = prb.getString(key);
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to get value with key: [" + key + "]");
|
||||
return null;
|
||||
}
|
||||
|
||||
if (value != null) {
|
||||
value = value.trim();
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
package com.baidu.hugegraph.store.client.util;
|
||||
|
||||
import com.baidu.hugegraph.store.HgKvStore;
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import static com.baidu.hugegraph.store.HgKvStore.*;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com
|
||||
*/
|
||||
public final class HgStoreClientConst {
|
||||
public final static String DEFAULT_NODE_CLUSTER_ID = "default-node-cluster";
|
||||
|
||||
public final static String EMPTY_STRING = "";
|
||||
public final static String EMPTY_TABLE = "";
|
||||
public final static byte[] EMPTY_BYTES = new byte[0];
|
||||
public final static byte[] MAX_BYTES=new byte[]{(byte)0b11111111};
|
||||
public final static List EMPTY_LIST = Collections.EMPTY_LIST;
|
||||
|
||||
public final static byte[] ALL_PARTITION_OWNER = new byte[0]; // means to dispatch to all partitions.
|
||||
public final static HgOwnerKey EMPTY_OWNER_KEY = HgOwnerKey.of(EMPTY_BYTES, EMPTY_BYTES);
|
||||
public final static HgOwnerKey ALL_PARTITION_OWNER_KEY = HgOwnerKey.of(ALL_PARTITION_OWNER, ALL_PARTITION_OWNER);
|
||||
|
||||
//public final static int SCAN_GTE_BEGIN_LT_END = SCAN_GTE_BEGIN | SCAN_LT_END;
|
||||
public final static int SCAN_TYPE_RANGE= SCAN_GTE_BEGIN | SCAN_LTE_END;
|
||||
public final static int SCAN_TYPE_ANY = SCAN_ANY;
|
||||
public final static int NO_LIMIT=0;
|
||||
|
||||
public final static int TX_SESSIONS_MAP_CAPACITY=32;
|
||||
public static final int NODE_MAX_RETRYING_TIMES=10;
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,141 @@
|
|||
package com.baidu.hugegraph.store.client.util;
|
||||
|
||||
import com.baidu.hugegraph.store.HgOwnerKey;
|
||||
import com.baidu.hugegraph.store.client.type.HgStoreClientException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* @author lynn.bond@hotmail.com created on 2021/10/14
|
||||
*/
|
||||
@Slf4j
|
||||
public final class HgStoreClientUtil {
|
||||
public static HgOwnerKey toOwnerKey(byte[] key) {
|
||||
return new HgOwnerKey(HgStoreClientConst.EMPTY_BYTES, key);
|
||||
}
|
||||
|
||||
public static HgOwnerKey toOwnerKey(String key) {
|
||||
return new HgOwnerKey(HgStoreClientConst.EMPTY_BYTES, toBytes(key));
|
||||
}
|
||||
|
||||
public static HgOwnerKey toAllNodeKey(String key) {
|
||||
return new HgOwnerKey(HgStoreClientConst.ALL_PARTITION_OWNER, toBytes(key));
|
||||
}
|
||||
|
||||
public static HgOwnerKey toOwnerKey(String owner, String key) {
|
||||
return new HgOwnerKey(toBytes(owner), toBytes(key));
|
||||
}
|
||||
|
||||
public static HgStoreClientException err(String msg) {
|
||||
log.error(msg);
|
||||
return HgStoreClientException.of(msg);
|
||||
}
|
||||
|
||||
public static boolean isValid(HgOwnerKey key) {
|
||||
if (key == null) return false;
|
||||
if (key.getKey() == null) return false;
|
||||
if (key.getKey().length == 0) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
public static String toStr(byte[] b) {
|
||||
if (b == null) return "";
|
||||
if (b.length == 0) return "";
|
||||
return new String(b, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public static String toByteStr(byte[] b) {
|
||||
if (b == null) return "";
|
||||
if (b.length == 0) return "";
|
||||
return Arrays.toString(b);
|
||||
}
|
||||
|
||||
public static String toStr(HgOwnerKey ownerKey) {
|
||||
if (ownerKey == null) return "";
|
||||
return new StringBuilder()
|
||||
.append("{ ")
|
||||
.append("owner: ").append(Arrays.toString(ownerKey.getOwner()))
|
||||
.append(", key: ").append(toStr(ownerKey.getKey()))
|
||||
.append(" }").toString();
|
||||
}
|
||||
|
||||
public static byte[] toBytes(String str) {
|
||||
if (str == null) return null;
|
||||
return str.getBytes(StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public static byte[] toBytes(long l) {
|
||||
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
|
||||
buffer.putLong(l);
|
||||
return buffer.array();
|
||||
}
|
||||
|
||||
public static byte[] toIntBytes(final int i) {
|
||||
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
|
||||
buffer.putInt(i);
|
||||
return buffer.array();
|
||||
}
|
||||
|
||||
public static long toLong(byte[] bytes) {
|
||||
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
|
||||
buffer.put(bytes);
|
||||
buffer.flip();//need flip
|
||||
return buffer.getLong();
|
||||
}
|
||||
|
||||
public static int toInt(byte[] bytes) {
|
||||
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
|
||||
buffer.put(bytes);
|
||||
buffer.flip();//need flip
|
||||
return buffer.getInt();
|
||||
}
|
||||
|
||||
public static String getHostAddress() {
|
||||
String res = null;
|
||||
|
||||
try {
|
||||
res = InetAddress.getLocalHost().getHostAddress();
|
||||
} catch (UnknownHostException e) {
|
||||
e.printStackTrace();
|
||||
res = "";
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
public static byte[] combine(byte[] first, byte[] second) {
|
||||
if (first == null){
|
||||
first=HgStoreClientConst.EMPTY_BYTES;
|
||||
}
|
||||
|
||||
if(second == null){
|
||||
second=HgStoreClientConst.EMPTY_BYTES;
|
||||
}
|
||||
|
||||
byte[] result = new byte[first.length + second.length];
|
||||
System.arraycopy(first, 0, result, 0, first.length);
|
||||
System.arraycopy(second, 0, result, first.length, second.length);
|
||||
return result;
|
||||
}
|
||||
|
||||
public static void printCallStack(String txt, Throwable ex) {
|
||||
StackTraceElement[] stackElements = ex.getStackTrace();
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(txt).append(":\n");
|
||||
if (stackElements != null) {
|
||||
for (int i = 0; i < stackElements.length; i++) {
|
||||
sb.append(stackElements[i].getClassName()).append(" : ")
|
||||
.append(stackElements[i].getMethodName()).append(" [ ");
|
||||
sb.append(stackElements[i].getLineNumber()).append(" ]\n");
|
||||
|
||||
}
|
||||
sb.append("--------------------------------------------------------------------------------------\n");
|
||||
}
|
||||
log.error(sb.toString());
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue