anph
/
hugegraph-sync
forked from hugegraph/hugegraph-sync
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refact: rename package for cli & dist & node & rocksdb & test modules 

TODO: some files need update manully
Change-Id: If83118e2dd123a7a88f9c5f7b33fbc7cf1b55de7
This commit is contained in:
imbajin 2023-04-26 11:08:51 +08:00
parent d62386b5a3
commit bb081bcfc2
221 changed files with 5268 additions and 2238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hg-store-cli/deploy-snapshot.sh
View File

@ -1,4 +1,21 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
readonly REPO_URL=http://10.14.139.8:8081/artifactory/star-snapshot
mvn --settings ../settings.xml -Dmaven.test.skip=true -DaltDeploymentRepository=star-local::default::${REPO_URL} clean deploy

19
hg-store-cli/pom.xml
View File

@ -1,4 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@ -51,7 +68,7 @@
</goals>
<configuration>
<mainClass>
com.baidu.hugegraph.store.cli.StoreConsoleApplication
org.apache.hugegraph.store.cli.StoreConsoleApplication
</mainClass>
</configuration>
</execution>

40
hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/AppConfig.java
View File

@ -1,40 +0,0 @@
package com.baidu.hugegraph.store.cli;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
/**
* @author lynn.bond@hotmail.com on 2022/2/15
*/
@Data
@Component
public class AppConfig {
@Value("${pd.address}")
private String pdAddress;
@Value("${net.kv.scanner.page.size}")
private int scannerPageSize;
@Value("${scanner.graph}")
private String scannerGraph;
@Value("${scanner.table}")
private String scannerTable;
@Value("${scanner.max}")
private int scannerMax;
@Value("${scanner.mod}")
private int scannerModNumber;
@Value("${committer.graph}")
private String committerGraph;
@Value("${committer.table}")
private String committerTable;
@Value("${committer.amount}")
private int committerAmount;
}

48
hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/util/HgMetricX.java
View File

@ -1,48 +0,0 @@
package com.baidu.hugegraph.store.cli.util;
import java.util.concurrent.atomic.AtomicLong;
/**
* @author lynn.bond@hotmail.com on 2022/1/29
*/
public class HgMetricX {
private long start;
private long end;
private long waitStart = System.currentTimeMillis();
private long waitTotal;
public static HgMetricX ofStart() {
return new HgMetricX(System.currentTimeMillis());
}
private HgMetricX(long start) {
this.start = start;
}
;
public long start() {
return this.start = System.currentTimeMillis();
}
public long end() {
return this.end = System.currentTimeMillis();
}
public long past() {
return this.end - this.start;
}
public long getWaitTotal() {
return this.waitTotal;
}
public void startWait() {
this.waitStart = System.currentTimeMillis();
}
public void appendWait() {
this.waitTotal += System.currentTimeMillis() - waitStart;
}
}

58
hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/AppConfig.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import lombok.Data;
/**
* @author lynn.bond@hotmail.com on 2022/2/15
*/
@Data
@Component
public class AppConfig {
@Value("${pd.address}")
private String pdAddress;
@Value("${net.kv.scanner.page.size}")
private int scannerPageSize;
@Value("${scanner.graph}")
private String scannerGraph;
@Value("${scanner.table}")
private String scannerTable;
@Value("${scanner.max}")
private int scannerMax;
@Value("${scanner.mod}")
private int scannerModNumber;
@Value("${committer.graph}")
private String committerGraph;
@Value("${committer.table}")
private String committerTable;
@Value("${committer.amount}")
private int committerAmount;
}

61
hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/StoreConsoleApplication.java → hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/StoreConsoleApplication.java
View File

@ -1,21 +1,38 @@
package com.baidu.hugegraph.store.cli;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
import com.baidu.hugegraph.pd.client.PDClient;
import com.baidu.hugegraph.pd.client.PDConfig;
import com.baidu.hugegraph.pd.common.PDException;
import com.baidu.hugegraph.store.HgStoreClient;
import com.baidu.hugegraph.store.cli.loader.HgThread2DB;
import com.baidu.hugegraph.store.cli.scan.GrpcShardScanner;
import com.baidu.hugegraph.store.cli.scan.HgStoreCommitter;
import com.baidu.hugegraph.store.cli.scan.HgStoreScanner;
package org.apache.hugegraph.store.cli;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import org.apache.hugegraph.store.cli.loader.HgThread2DB;
import org.apache.hugegraph.store.cli.scan.GrpcShardScanner;
import org.apache.hugegraph.store.cli.scan.HgStoreCommitter;
import org.apache.hugegraph.store.cli.scan.HgStoreScanner;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import java.io.IOException;
import com.baidu.hugegraph.pd.client.PDConfig;
import com.baidu.hugegraph.pd.common.PDException;
import com.baidu.hugegraph.store.HgStoreClient;
import lombok.extern.slf4j.Slf4j;
/**
@ -36,11 +53,11 @@ public class StoreConsoleApplication implements CommandLineRunner {
@Override
public void run(String... args) throws IOException, InterruptedException, PDException {
if (args.length <= 0){
if (args.length <= 0) {
// HgThread2DB hB = new HgThread2DB("localhost:8686");
// hB.startMultiprocessQuery("12", "10");
System.out.println("参数类型 cmd[-load, -query, -scan]");
}else {
} else {
switch (args[0]) {
case "-load":
HgThread2DB hgThread2DB = new HgThread2DB(args[1]);
@ -48,9 +65,9 @@ public class StoreConsoleApplication implements CommandLineRunner {
hgThread2DB.setGraphName(args[3]);
}
try {
if (args[2].equals("order")){
if (args[2].equals("order")) {
hgThread2DB.testOrder(args[4]);
}else {
} else {
hgThread2DB.startMultiprocessInsert(args[2]);
}
} catch (IOException e) {
@ -66,9 +83,9 @@ public class StoreConsoleApplication implements CommandLineRunner {
}
break;
case "-scan":
if ( args.length < 4){
if (args.length < 4) {
System.out.println("参数类型 -scan pd graphName tableName");
}else {
} else {
doScan(args[1], args[2], args[3]);
}
break;
@ -86,17 +103,17 @@ public class StoreConsoleApplication implements CommandLineRunner {
}
}
private void doCommit(){
HgStoreCommitter committer=HgStoreCommitter.of(appConfig.getCommitterGraph());
committer.put(appConfig.getScannerTable(),appConfig.getCommitterAmount());
private void doCommit() {
HgStoreCommitter committer = HgStoreCommitter.of(appConfig.getCommitterGraph());
committer.put(appConfig.getScannerTable(), appConfig.getCommitterAmount());
}
private void doScan(String pd, String graphName, String tableName) throws PDException {
HgStoreClient storeClient = HgStoreClient.create(PDConfig.of(pd)
.setEnableCache(true));
.setEnableCache(true));
HgStoreScanner storeScanner = HgStoreScanner.of(storeClient, graphName);
storeScanner.scanTable2(tableName);
// storeScanner.scanHash();
// storeScanner.scanHash();
}
}

231
hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/loader/HgThread2DB.java → hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/loader/HgThread2DB.java
View File

@ -1,29 +1,33 @@
package com.baidu.hugegraph.store.cli.loader;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
import com.baidu.hugegraph.pd.client.PDClient;
import com.baidu.hugegraph.pd.client.PDConfig;
import com.baidu.hugegraph.store.HgStoreClient;
import com.baidu.hugegraph.store.HgOwnerKey;
import com.baidu.hugegraph.store.HgStoreSession;
import com.baidu.hugegraph.store.HgKvEntry;
import com.baidu.hugegraph.store.HgKvIterator;
import com.baidu.hugegraph.store.HgScanQuery;
import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator;
import com.baidu.hugegraph.store.client.util.MetricX;
import com.baidu.hugegraph.store.cli.util.HgCliUtil;
import lombok.extern.slf4j.Slf4j;
package org.apache.hugegraph.store.cli.loader;
import java.io.IOException;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.BufferedReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
@ -32,6 +36,21 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hugegraph.store.cli.util.HgCliUtil;
import com.baidu.hugegraph.pd.client.PDClient;
import com.baidu.hugegraph.pd.client.PDConfig;
import com.baidu.hugegraph.store.HgKvEntry;
import com.baidu.hugegraph.store.HgKvIterator;
import com.baidu.hugegraph.store.HgOwnerKey;
import com.baidu.hugegraph.store.HgScanQuery;
import com.baidu.hugegraph.store.HgStoreClient;
import com.baidu.hugegraph.store.HgStoreSession;
import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator;
import com.baidu.hugegraph.store.client.util.MetricX;
import lombok.extern.slf4j.Slf4j;
/**
* 使用pd，支持raft
@ -42,35 +61,37 @@ public class HgThread2DB {
private static PDClient pdClient;
public String graphName = "hugegraphtest";
/*正在进行和在排队的任务的总数*/
private static AtomicInteger taskTotal = new AtomicInteger(0);
private static AtomicInteger queryTaskTotal = new AtomicInteger(0);
private static final AtomicInteger taskTotal = new AtomicInteger(0);
private static final AtomicInteger queryTaskTotal = new AtomicInteger(0);
private static ThreadPoolExecutor threadPool = null;
private static ThreadPoolExecutor queryThreadPool = null;
private static AtomicLong insertDataCount = new AtomicLong();
private static AtomicLong queryCount = new AtomicLong();
private static AtomicLong totalQueryCount = new AtomicLong();
volatile long startTime = System.currentTimeMillis();
private static final AtomicLong insertDataCount = new AtomicLong();
private static final AtomicLong queryCount = new AtomicLong();
private static final AtomicLong totalQueryCount = new AtomicLong();
volatile long startTime = System.currentTimeMillis();
private static int limitScanBatchCount = 100;
private static ArrayBlockingQueue listQueue = null;
private static AtomicLong longId = new AtomicLong();
private static final AtomicLong longId = new AtomicLong();
private static CountDownLatch countDownLatch = null;
private HgStoreClient storeClient;
private static final CountDownLatch countDownLatch = null;
private final HgStoreClient storeClient;
public HgThread2DB(String pdAddr) {
int threadCount = Runtime.getRuntime().availableProcessors();
listQueue = new ArrayBlockingQueue<List<HgOwnerKey>>(100000000);
queryThreadPool = new ThreadPoolExecutor(500, 1000,
200, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>( 1000));
threadPool = new ThreadPoolExecutor(threadCount*2, threadCount * 3,
200, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(threadCount + 100));
200, TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(1000));
threadPool = new ThreadPoolExecutor(threadCount * 2, threadCount * 3,
200, TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(threadCount + 100));
storeClient = HgStoreClient.create(PDConfig.of(pdAddr)
.setEnableCache(true));
.setEnableCache(true));
pdClient = storeClient.getPdClient();
}
public void setGraphName(String graphName){
public void setGraphName(String graphName) {
this.graphName = graphName;
log.info("setGraphName {}", graphName);
}
@ -89,13 +110,14 @@ public class HgThread2DB {
session.put(tableName, hgKey, value);
});
if ( insertDataCount.get() > 10000000){
synchronized (insertDataCount){
if (insertDataCount.get() > 10000000) {
synchronized (insertDataCount) {
long count = insertDataCount.get();
insertDataCount.set(0);
if (count > 10000000){
log.info("count : " + count + " qps : " + count * 1000 / (System.currentTimeMillis() - startTime)
+" threadCount : " + taskTotal);
if (count > 10000000) {
log.info("count : " + count + " qps : " +
count * 1000 / (System.currentTimeMillis() - startTime)
+ " threadCount : " + taskTotal);
startTime = System.currentTimeMillis();
}
}
@ -120,20 +142,21 @@ public class HgThread2DB {
for (int y = 0; y < maxlist; y++) {
insertDataCount.getAndIncrement();
String strLine = "" + getLong() + getLong() + getLong() + getLong();
String strLine = getLong() + getLong() + getLong() + getLong();
// log.info("========{}",strLine);
HgOwnerKey hgKey = HgCliUtil.toOwnerKey(strLine, strLine);
byte[] value = HgCliUtil.toBytes(strLine);
session.put(tableName, hgKey, value);
}
if ( insertDataCount.get() > 10000000){
synchronized (insertDataCount){
if (insertDataCount.get() > 10000000) {
synchronized (insertDataCount) {
long count = insertDataCount.get();
insertDataCount.set(0);
if (count > 10000000){
log.info("count : " + count + " qps : " + count * 1000 / (System.currentTimeMillis() - startTime)
+" threadCount : " + taskTotal);
if (count > 10000000) {
log.info("count : " + count + " qps : " +
count * 1000 / (System.currentTimeMillis() - startTime)
+ " threadCount : " + taskTotal);
startTime = System.currentTimeMillis();
}
}
@ -154,12 +177,13 @@ public class HgThread2DB {
HgStoreSession session = storeClient.openSession(graphName);
session.beginTx();
int loop = Integer.parseInt(input);
if (loop == 0){
if (loop == 0) {
loop = 2000;
}
for (int i = 0; i < loop; i++) {
long startTime = System.currentTimeMillis();
HgOwnerKey hgOwnerKey = HgCliUtil.toOwnerKey(startTime + "owner:" + i, startTime + "k:" + i);
HgOwnerKey hgOwnerKey =
HgCliUtil.toOwnerKey(startTime + "owner:" + i, startTime + "k:" + i);
session.put(tableName, hgOwnerKey, HgCliUtil.toBytes(i));
}
@ -189,6 +213,7 @@ public class HgThread2DB {
/**
* 多线程读取文件入库
*
* @throws IOException
* @throws InterruptedException
*/
@ -200,7 +225,8 @@ public class HgThread2DB {
long dataCount = 0;
if (readfile.exists()) {
// 读取文件
InputStreamReader isr = new InputStreamReader(new FileInputStream(readfile), "UTF-8");
InputStreamReader isr = new InputStreamReader(new FileInputStream(readfile),
StandardCharsets.UTF_8);
BufferedReader reader = new BufferedReader(isr);
String strLine = null;
@ -227,7 +253,7 @@ public class HgThread2DB {
e.printStackTrace();
}
taskTotal.decrementAndGet();
synchronized (taskTotal){
synchronized (taskTotal) {
taskTotal.notifyAll();
}
}
@ -235,8 +261,8 @@ public class HgThread2DB {
taskTotal.getAndIncrement();
threadPool.execute(task);
while ( taskTotal.get() > 100){
synchronized (taskTotal){
while (taskTotal.get() > 100) {
synchronized (taskTotal) {
taskTotal.wait();
}
}
@ -251,7 +277,7 @@ public class HgThread2DB {
isr.close();
reader.close();
// 把剩余的入库
if (!keys.isEmpty()){
if (!keys.isEmpty()) {
List<String> finalKeys1 = keys;
Runnable task = new Runnable() {
public void run() {
@ -261,7 +287,7 @@ public class HgThread2DB {
e.printStackTrace();
}
taskTotal.decrementAndGet();
synchronized (taskTotal){
synchronized (taskTotal) {
taskTotal.notifyAll();
}
}
@ -269,11 +295,11 @@ public class HgThread2DB {
threadPool.execute(task);
taskTotal.getAndIncrement();
}
while ( taskTotal.get() > 0){
synchronized (taskTotal){
while (taskTotal.get() > 0) {
synchronized (taskTotal) {
try {
taskTotal.wait(1000);
if ( taskTotal.get() > 0){
if (taskTotal.get() > 0) {
System.out.println("wait thread exit " + taskTotal.get());
}
} catch (InterruptedException e) {
@ -284,7 +310,7 @@ public class HgThread2DB {
threadPool.shutdown();
}else {
} else {
System.out.println("样本文件不存在：" + filepath);
}
metrics.end();
@ -297,6 +323,7 @@ public class HgThread2DB {
/**
* 多线程读取文件入库
*
* @throws IOException
* @throws InterruptedException
*/
@ -342,11 +369,11 @@ public class HgThread2DB {
}
}
while ( taskTotal.get() > 0){
synchronized (taskTotal){
while (taskTotal.get() > 0) {
synchronized (taskTotal) {
try {
taskTotal.wait(1000);
if ( taskTotal.get() > 0){
if (taskTotal.get() > 0) {
System.out.println("wait thread exit " + taskTotal.get());
}
} catch (InterruptedException e) {
@ -383,7 +410,7 @@ public class HgThread2DB {
while (!listQueue.isEmpty()) {
log.info(" ====== start scanBatch2 count:{} list:{}=============",
queryThreadPool.getActiveCount(), listQueue.size());
queryThreadPool.getActiveCount(), listQueue.size());
List<HgOwnerKey> keys = (List<HgOwnerKey>) listQueue.take();
List<HgOwnerKey> newQueryList = new ArrayList<>();
@ -413,9 +440,11 @@ public class HgThread2DB {
queryCount.set(0);
if (count > 1000000) {
log.info("count : " + count + " qps : " + count * 1000 /
(System.currentTimeMillis() - startTime)
+ " threadCount : " + queryThreadPool.getActiveCount() + " queueSize:"
+ listQueue.size());
(System.currentTimeMillis() -
startTime)
+ " threadCount : " +
queryThreadPool.getActiveCount() + " queueSize:"
+ listQueue.size());
startTime = System.currentTimeMillis();
}
}
@ -448,12 +477,14 @@ public class HgThread2DB {
/**
* 多线程查询
* @param point 起始查询点，后续根据这个点查询到的value做为下一次的查询条件进行迭代
*
* @param point 起始查询点，后续根据这个点查询到的value做为下一次的查询条件进行迭代
* @param scanCount 允许启动的线程数量
* @throws IOException
* @throws InterruptedException
*/
public void startMultiprocessQuery(String point, String scanCount) throws IOException, InterruptedException {
public void startMultiprocessQuery(String point, String scanCount) throws IOException,
InterruptedException {
log.info("--- start startMultiprocessQuery---");
startTime = System.currentTimeMillis();
MetricX metrics = MetricX.ofStart();
@ -466,69 +497,79 @@ public class HgThread2DB {
final long[] start = {System.currentTimeMillis()};
LinkedBlockingQueue[] queue = new LinkedBlockingQueue[limitScanBatchCount];
for(int i =0; i<limitScanBatchCount; i++) {
for (int i = 0; i < limitScanBatchCount; i++) {
queue[i] = new LinkedBlockingQueue();
}
List<String> strKey = Arrays.asList(new String[]{"20727483", "50329304", "26199460", "1177521", "27960125",
"30440025", "15833920", "15015183", "33153097", "21250581"});
strKey.forEach(key->{
List<String> strKey = Arrays.asList(
"20727483", "50329304", "26199460", "1177521", "27960125",
"30440025", "15833920", "15015183", "33153097", "21250581");
strKey.forEach(key -> {
log.info("newkey:{}", key);
HgOwnerKey hgKey = HgCliUtil.toOwnerKey(key, key);
queue[0].add(hgKey);
});
for(int i =0; i<limitScanBatchCount; i++) {
for (int i = 0; i < limitScanBatchCount; i++) {
int finalI = i;
KvCloseableIterator<HgKvIterator<HgKvEntry>> iterators =
session.scanBatch2(
HgScanQuery.prefixIteratorOf(HgCliUtil.TABLE_NAME, new Iterator<HgOwnerKey>() {
HgOwnerKey current = null;
HgScanQuery.prefixIteratorOf(HgCliUtil.TABLE_NAME,
new Iterator<HgOwnerKey>() {
HgOwnerKey current = null;
@Override
public boolean hasNext() {
while (current == null) {
try {
current = (HgOwnerKey) queue[finalI].poll(1, TimeUnit.SECONDS);
} catch (InterruptedException e) {
//
}
}
if (current == null){
log.info("===== current is null ==========");
}
return current != null;
}
@Override
public boolean hasNext() {
while (current == null) {
try {
current =
(HgOwnerKey) queue[finalI].poll(
1,
TimeUnit.SECONDS);
} catch (
InterruptedException e) {
//
}
}
if (current == null) {
log.info(
"===== current is " +
"null ==========");
}
return current != null;
}
@Override
public HgOwnerKey next() {
return current;
}
})
@Override
public HgOwnerKey next() {
return current;
}
})
);
new Thread(()->{
new Thread(() -> {
while (iterators.hasNext()) {
HgKvIterator<HgKvEntry> iterator = iterators.next();
long c = 0;
while (iterator.hasNext()) {
String newPoint = HgCliUtil.toStr(iterator.next().value());
HgOwnerKey newHgKey = HgCliUtil.toOwnerKey(newPoint, newPoint);
if ( queue[(int) (c % limitScanBatchCount)].size() < 1000000) {
if (queue[(int) (c % limitScanBatchCount)].size() < 1000000) {
queue[(int) (c % limitScanBatchCount)].add(newHgKey);
}
c++;
}
if ( counter[0].addAndGet(c) > 1000000){
if (counter[0].addAndGet(c) > 1000000) {
synchronized (counter) {
if ( counter[0].get() > 10000000) {
log.info("count {}, qps {}", counter[0].get(), counter[0].get() * 1000 / (System.currentTimeMillis() - start[0]));
if (counter[0].get() > 10000000) {
log.info("count {}, qps {}", counter[0].get(),
counter[0].get() * 1000 /
(System.currentTimeMillis() - start[0]));
start[0] = System.currentTimeMillis();
counter[0].set(0);
}
}
}
}
}, "client query thread:" + i ).start();
}, "client query thread:" + i).start();
log.info("===== read thread exit ==========");
}
latch.await();
@ -537,7 +578,7 @@ public class HgThread2DB {
metrics.end();
log.info("*************************************************");
log.info(" 主进程执行时间:" + metrics.past() / 1000 + "秒; 查询：" + totalQueryCount.get()
+ "次，qps:" + totalQueryCount.get() * 1000 / metrics.past());
+ "次，qps:" + totalQueryCount.get() * 1000 / metrics.past());
log.info("*************************************************");
System.out.println("-----主进程执行结束---------");
}

27
hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/scan/GrpcShardScanner.java → hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/GrpcShardScanner.java
View File

@ -1,4 +1,21 @@
package com.baidu.hugegraph.store.cli.scan;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli.scan;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
@ -26,9 +43,9 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class GrpcShardScanner {
private volatile boolean closed = false;
private AtomicInteger sum = new AtomicInteger();
private ConcurrentHashMap<Long, StreamObserver<ScanPartitionRequest>>
private final boolean closed = false;
private final AtomicInteger sum = new AtomicInteger();
private final ConcurrentHashMap<Long, StreamObserver<ScanPartitionRequest>>
observers = new ConcurrentHashMap<>();
public void getData() {
@ -137,8 +154,6 @@ public class GrpcShardScanner {
observer.onNext(builder.build());
} catch (Exception e) {
e.printStackTrace();
} finally {
}
}

38
hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/scan/HgStoreCommitter.java → hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreCommitter.java
View File

@ -1,12 +1,29 @@
package com.baidu.hugegraph.store.cli.scan;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli.scan;
import org.apache.hugegraph.store.cli.util.HgCliUtil;
import com.baidu.hugegraph.store.HgOwnerKey;
import com.baidu.hugegraph.store.HgSessionManager;
import com.baidu.hugegraph.store.HgStoreSession;
import com.baidu.hugegraph.store.client.HgStoreNodeManager;
import static com.baidu.hugegraph.store.cli.util.HgCliUtil.*;
/**
* @author lynn.bond@hotmail.com on 2022/2/28
*/
@ -33,7 +50,7 @@ public class HgStoreCommitter {
return HgSessionManager.getInstance().openSession(graphName);
}
public void put(String tableName,int amount) {
public void put(String tableName, int amount) {
//*************** Put Benchmark **************//*
String keyPrefix = "PUT-BENCHMARK";
HgStoreSession session = getStoreSession();
@ -44,16 +61,19 @@ public class HgStoreCommitter {
long start = System.currentTimeMillis();
for (int i = 0; i < amount; i++) {
HgOwnerKey key = toOwnerKey(keyPrefix + "-" + padLeftZeros(String.valueOf(i), length));
byte[] value = toBytes(keyPrefix + "-V-" + i);
HgOwnerKey key = HgCliUtil.toOwnerKey(
keyPrefix + "-" + HgCliUtil.padLeftZeros(String.valueOf(i), length));
byte[] value = HgCliUtil.toBytes(keyPrefix + "-V-" + i);
session.put(tableName, key, value);
if ((i + 1) % 100_000 == 0) {
println("---------- " + (i + 1) + " --------");
println("Preparing took: " + (System.currentTimeMillis() - start) + " ms.");
HgCliUtil.println("---------- " + (i + 1) + " --------");
HgCliUtil.println(
"Preparing took: " + (System.currentTimeMillis() - start) + " ms.");
session.commit();
println("Committing took: " + (System.currentTimeMillis() - start) + " ms.");
HgCliUtil.println(
"Committing took: " + (System.currentTimeMillis() - start) + " ms.");
start = System.currentTimeMillis();
session.beginTx();
}

70
hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/scan/HgStoreScanner.java → hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreScanner.java
View File

@ -1,18 +1,43 @@
package com.baidu.hugegraph.store.cli.scan;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli.scan;
import java.util.Arrays;
import java.util.List;
import org.apache.hugegraph.store.cli.util.HgCliUtil;
import org.apache.hugegraph.store.cli.util.HgMetricX;
import com.baidu.hugegraph.pd.client.PDClient;
import com.baidu.hugegraph.pd.common.PDException;
import com.baidu.hugegraph.pd.grpc.Metapb;
import com.baidu.hugegraph.store.*;
import com.baidu.hugegraph.store.cli.util.HgCliUtil;
import com.baidu.hugegraph.store.cli.util.HgMetricX;
import com.baidu.hugegraph.store.HgKvEntry;
import com.baidu.hugegraph.store.HgKvIterator;
import com.baidu.hugegraph.store.HgKvStore;
import com.baidu.hugegraph.store.HgScanQuery;
import com.baidu.hugegraph.store.HgSessionManager;
import com.baidu.hugegraph.store.HgStoreClient;
import com.baidu.hugegraph.store.HgStoreSession;
import com.baidu.hugegraph.store.client.grpc.KvCloseableIterator;
import com.baidu.hugegraph.store.client.util.HgStoreClientConfig;
import com.baidu.hugegraph.store.client.util.MetricX;
import lombok.extern.slf4j.Slf4j;
import java.util.Arrays;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com on 2022/2/14
@ -20,7 +45,7 @@ import java.util.List;
@Slf4j
public class HgStoreScanner {
private HgStoreClient storeClient;
private final HgStoreClient storeClient;
private final String graphName;
private long modNumber = 1_000_000;
private int max = 10_000_000;
@ -69,7 +94,8 @@ public class HgStoreScanner {
HgMetricX hgMetricX = HgMetricX.ofStart();
HgStoreSession session = getStoreSession();
int count = 0;
KvCloseableIterator<HgKvIterator<HgKvEntry>> iterator = session.scanBatch2(HgScanQuery.tableOf(tableName));
KvCloseableIterator<HgKvIterator<HgKvEntry>> iterator =
session.scanBatch2(HgScanQuery.tableOf(tableName));
long start = System.currentTimeMillis();
while (iterator.hasNext()) {
@ -80,7 +106,9 @@ public class HgStoreScanner {
iterator2.next();
if (count % (modNumber) == 0) {
log.info("Scanning keys: " + count + " time is " + modNumber * 1000
/ (System.currentTimeMillis() - start));
/
(System.currentTimeMillis() -
start));
start = System.currentTimeMillis();
}
if (count == max) {
@ -112,13 +140,18 @@ public class HgStoreScanner {
String graphName = "/DEFAULT/graphs/hugegraph1/";
HgStoreSession session = getStoreSession(graphName);
int count = 0;
String query = "{\"conditions\":[{\"cls\":\"S\",\"el\":{\"key\":\"ID\",\"relation\":\"SCAN\",\"value\"" +
":{\"start\":\"61180\",\"end\":\"63365\",\"length\":0}}}],\"optimizedType\":\"NONE\",\"ids\":[]," +
String query =
"{\"conditions\":[{\"cls\":\"S\",\"el\":{\"key\":\"ID\",\"relation\":\"SCAN\"," +
"\"value\"" +
":{\"start\":\"61180\",\"end\":\"63365\",\"length\":0}}}]," +
"\"optimizedType\":\"NONE\",\"ids\":[]," +
"\"mustSortByInput\":true,\"resultType\":\"EDGE\",\"offset\":0," +
"\"actualOffset\":0,\"actualStoreOffset\":" +
"0,\"limit\":9223372036854775807,\"capacity\":-1,\"showHidden\":false,\"showDeleting\":false," +
"0,\"limit\":9223372036854775807,\"capacity\":-1,\"showHidden\":false," +
"\"showDeleting\":false," +
"\"showExpired\":false,\"olap\":false,\"withProperties\":false,\"olapPks\":[]}";
//HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,0,715827883, HgKvStore.SCAN_ANY,null);
//HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,0,715827883,
// HgKvStore.SCAN_ANY,null);
//HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,61180,63365, 348, null);
//HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,0,65535, 348, null);
@ -154,7 +187,8 @@ public class HgStoreScanner {
public static final byte[] EMPTY_BYTES = new byte[0];
public void scanTable2(String tableName) throws PDException {
// java -jar hg-store-cli-3.6.0-SNAPSHOT.jar -scan 10.45.30.212:8989 "DEFAULT/case_112/g" g+ie
// java -jar hg-store-cli-3.6.0-SNAPSHOT.jar -scan 10.45.30.212:8989 "DEFAULT/case_112/g"
// g+ie
PDClient pdClient = storeClient.getPdClient();
List<Metapb.Partition> partitions = pdClient.getPartitions(0, graphName);
HgStoreSession session = storeClient.openSession(graphName);
@ -164,8 +198,10 @@ public class HgStoreScanner {
for (Metapb.Partition partition : partitions) {
while (true) {
try (HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,
(int) (partition.getStartKey()), (int) (partition.getEndKey()),
HgKvStore.SCAN_HASHCODE, EMPTY_BYTES)) {
(int) (partition.getStartKey()),
(int) (partition.getEndKey()),
HgKvStore.SCAN_HASHCODE,
EMPTY_BYTES)) {
if (position != null) {
iterator.seek(position);
}

50
hg-store-cli/src/main/java/com/baidu/hugegraph/store/cli/util/HgCliUtil.java → hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgCliUtil.java
View File

@ -1,15 +1,36 @@
package com.baidu.hugegraph.store.cli.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli.util;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import com.baidu.hugegraph.store.HgKvEntry;
import com.baidu.hugegraph.store.HgOwnerKey;
import com.baidu.hugegraph.store.HgStoreSession;
import com.baidu.hugegraph.store.client.util.HgStoreClientConst;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.function.Function;
/**
* @author lynn.bond@hotmail.com on 2022/2/14
@ -23,27 +44,28 @@ public class HgCliUtil {
return batchPut(session, keyPrefix, 100);
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String keyPrefix, int loop) {
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String keyPrefix,
int loop) {
return batchPut(session, TABLE_NAME, keyPrefix, loop);
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
, String keyPrefix, int loop) {
return batchPut(session, tableName, keyPrefix, loop,1, key -> toOwnerKey(key));
return batchPut(session, tableName, keyPrefix, loop, 1, key -> toOwnerKey(key));
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
, String keyPrefix, int loop,int start) {
return batchPut(session, tableName, keyPrefix, loop,start, key -> toOwnerKey(key));
, String keyPrefix, int loop, int start) {
return batchPut(session, tableName, keyPrefix, loop, start, key -> toOwnerKey(key));
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
, String keyPrefix, int loop, Function<String, HgOwnerKey> f){
return batchPut(session,tableName,keyPrefix,loop,1,f);
, String keyPrefix, int loop, Function<String, HgOwnerKey> f) {
return batchPut(session, tableName, keyPrefix, loop, 1, f);
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
, String keyPrefix, int loop, int start,Function<String, HgOwnerKey> f) {
, String keyPrefix, int loop, int start, Function<String, HgOwnerKey> f) {
Map<HgOwnerKey, byte[]> res = new LinkedHashMap<>();
@ -59,7 +81,7 @@ public class HgCliUtil {
session.put(tableName, key, value);
if ((i + 1) % 10000 == 0) {
println("commit: " + (i+1));
println("commit: " + (i + 1));
session.commit();
session.beginTx();
}

61
hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgMetricX.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli.util;
/**
* @author lynn.bond@hotmail.com on 2022/1/29
*/
public class HgMetricX {
private long start;
private long end;
private long waitStart = System.currentTimeMillis();
private long waitTotal;
public static HgMetricX ofStart() {
return new HgMetricX(System.currentTimeMillis());
}
private HgMetricX(long start) {
this.start = start;
}
public long start() {
return this.start = System.currentTimeMillis();
}
public long end() {
return this.end = System.currentTimeMillis();
}
public long past() {
return this.end - this.start;
}
public long getWaitTotal() {
return this.waitTotal;
}
public void startWait() {
this.waitStart = System.currentTimeMillis();
}
public void appendWait() {
this.waitTotal += System.currentTimeMillis() - waitStart;
}
}

18
hg-store-cli/src/main/resources/hg-store-client.properties
View File

@ -1,7 +1,23 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
#grpc.timeout.seconds=10
#grpc.max.inbound.message.size=
#grpc.max.outbound.message.size=
net.kv.scanner.buffer.size=10000
net.kv.scanner.page.size = 20000
net.kv.scanner.page.size=20000
#Unit:second
net.kv.scanner.have.next.timeout=1000

23
hg-store-cli/src/main/resources/log4j2.xml
View File

@ -1,5 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<!-- Config will be auto loaded every 60s -->
<configuration status="error" monitorInterval="60">
<properties>
@ -28,7 +45,7 @@
<!-- Trigger after exceeding 1day or 50MB -->
<Policies>
<SizeBasedTriggeringPolicy size="50MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true" />
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<!-- Keep 5 files per day & auto Delete after over 2GB or 100 files -->
<DefaultRolloverStrategy max="5">
@ -36,8 +53,8 @@
<IfFileName glob="*/*.log"/>
<!-- Limit log amount & size -->
<IfAny>
<IfAccumulatedFileSize exceeds="2GB" />
<IfAccumulatedFileCount exceeds="100" />
<IfAccumulatedFileSize exceeds="2GB"/>
<IfAccumulatedFileCount exceeds="100"/>
</IfAny>
</Delete>
</DefaultRolloverStrategy>

2
hg-store-core/src/main/java/com/baidu/hugegraph/store/HgStoreEngine.java
View File

@ -11,7 +11,7 @@ import com.alipay.sofa.jraft.rpc.RpcServer;
import com.alipay.sofa.jraft.util.Endpoint;
import com.baidu.hugegraph.pd.common.PDException;
import com.baidu.hugegraph.pd.grpc.Metapb;
import com.baidu.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import com.baidu.hugegraph.store.business.DataMover;
import com.baidu.hugegraph.store.business.BusinessHandler;
import com.baidu.hugegraph.store.business.BusinessHandlerImpl;

2
hg-store-core/src/main/java/com/baidu/hugegraph/store/business/AbstractSelectIterator.java
View File

@ -6,7 +6,7 @@ import com.baidu.hugegraph.backend.serializer.AbstractSerializer;
import com.baidu.hugegraph.backend.serializer.BinarySerializer;
import com.baidu.hugegraph.backend.store.BackendEntry;
import com.baidu.hugegraph.iterator.CIter;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.structure.HugeElement;
import com.baidu.hugegraph.util.Bytes;

2
hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandler.java
View File

@ -13,7 +13,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.baidu.hugegraph.pd.grpc.pulse.CleanType;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.store.grpc.Graphpb;
import com.baidu.hugegraph.store.grpc.common.Key;
import com.baidu.hugegraph.store.grpc.common.OpType;

16
hg-store-core/src/main/java/com/baidu/hugegraph/store/business/BusinessHandlerImpl.java
View File

@ -27,13 +27,13 @@ import com.alipay.sofa.jraft.util.Utils;
import com.baidu.hugegraph.config.HugeConfig;
import com.baidu.hugegraph.config.OptionSpace;
import com.baidu.hugegraph.pd.grpc.pulse.CleanType;
import com.baidu.hugegraph.rocksdb.access.DBStoreException;
import com.baidu.hugegraph.rocksdb.access.RocksDBFactory;
import com.baidu.hugegraph.rocksdb.access.RocksDBFactory.RocksdbChangedListener;
import com.baidu.hugegraph.rocksdb.access.RocksDBOptions;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.rocksdb.access.SessionOperator;
import org.apache.hugegraph.rocksdb.access.DBStoreException;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory.RocksdbChangedListener;
import org.apache.hugegraph.rocksdb.access.RocksDBOptions;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.SessionOperator;
import com.baidu.hugegraph.store.HgStoreEngine;
import com.baidu.hugegraph.store.cmd.CleanDataRequest;
import com.baidu.hugegraph.store.grpc.Graphpb.ScanPartitionRequest;
@ -89,7 +89,7 @@ public class BusinessHandlerImpl implements BusinessHandler {
public static HugeConfig initRocksdb(Map<String, Object> rocksdbConfig, RocksdbChangedListener listener) {
// 注册rocksdb配置
OptionSpace.register("rocksdb",
"com.baidu.hugegraph.rocksdb.access.RocksDBOptions");
"org.apache.hugegraph.rocksdb.access.RocksDBOptions");
RocksDBOptions.instance();
HugeConfig hConfig = new HugeConfig(rocksdbConfig);
factory.setHugeConfig(hConfig);

4
hg-store-core/src/main/java/com/baidu/hugegraph/store/business/DefaultDataMover.java
View File

@ -3,8 +3,8 @@ package com.baidu.hugegraph.store.business;
import com.alipay.sofa.jraft.Status;
import com.baidu.hugegraph.pd.grpc.Metapb;
import com.baidu.hugegraph.pd.grpc.pulse.CleanType;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.store.cmd.BatchPutRequest;
import com.baidu.hugegraph.store.cmd.BatchPutResponse;
import com.baidu.hugegraph.store.cmd.CleanDataRequest;

4
hg-store-core/src/main/java/com/baidu/hugegraph/store/business/FilterIterator.java
View File

@ -5,8 +5,8 @@ import org.apache.commons.lang3.ArrayUtils;
import com.baidu.hugegraph.backend.query.ConditionQuery;
import com.baidu.hugegraph.backend.serializer.BinaryBackendEntry;
import com.baidu.hugegraph.backend.store.BackendEntry;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession.BackendColumn;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.structure.HugeElement;
import lombok.extern.slf4j.Slf4j;

4
hg-store-core/src/main/java/com/baidu/hugegraph/store/business/GraphStoreIterator.java
View File

@ -20,8 +20,8 @@ import org.codehaus.groovy.jsr223.GroovyScriptEngineImpl;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.serializer.BinaryBackendEntry;
import com.baidu.hugegraph.backend.store.BackendEntry;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession.BackendColumn;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.schema.EdgeLabel;
import com.baidu.hugegraph.schema.PropertyKey;
import com.baidu.hugegraph.schema.VertexLabel;

4
hg-store-core/src/main/java/com/baidu/hugegraph/store/business/InnerKeyFilter.java
View File

@ -1,7 +1,7 @@
package com.baidu.hugegraph.store.business;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession.BackendColumn;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.store.term.Bits;
import java.util.Arrays;

2
hg-store-core/src/main/java/com/baidu/hugegraph/store/business/MultiPartitionIterator.java
View File

@ -1,6 +1,6 @@
package com.baidu.hugegraph.store.business;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import java.nio.ByteBuffer;
import java.util.*;

4
hg-store-core/src/main/java/com/baidu/hugegraph/store/business/SelectIterator.java
View File

@ -6,8 +6,8 @@ import java.util.Set;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.serializer.BytesBuffer;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession.BackendColumn;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.RocksDBSession.BackendColumn;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.type.define.DataType;
import com.baidu.hugegraph.type.define.SerialEnum;

2
hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/DeletedFileManager.java
View File

@ -1,6 +1,6 @@
package com.baidu.hugegraph.store.meta;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.store.meta.base.GlobalMetaStore;
import com.baidu.hugegraph.store.options.MetadataOptions;
import lombok.extern.slf4j.Slf4j;

2
hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/DBSessionBuilder.java
View File

@ -1,6 +1,6 @@
package com.baidu.hugegraph.store.meta.base;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.store.util.HgStoreException;
public interface DBSessionBuilder {

4
hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/GlobalMetaStore.java
View File

@ -1,7 +1,7 @@
package com.baidu.hugegraph.store.meta.base;
import com.baidu.hugegraph.rocksdb.access.RocksDBFactory;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.store.options.MetadataOptions;
import java.util.Arrays;

6
hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/MetaStoreBase.java
View File

@ -1,8 +1,8 @@
package com.baidu.hugegraph.store.meta.base;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.rocksdb.access.SessionOperator;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.SessionOperator;
import com.baidu.hugegraph.store.util.HgStoreException;
import com.baidu.hugegraph.store.util.Asserts;
import com.google.protobuf.GeneratedMessageV3;

2
hg-store-core/src/main/java/com/baidu/hugegraph/store/meta/base/PartitionMetaStore.java
View File

@ -1,6 +1,6 @@
package com.baidu.hugegraph.store.meta.base;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
/**
* 元数据存储在分区的default cf中

4
hg-store-core/src/main/java/com/baidu/hugegraph/store/metric/SystemMetricService.java
View File

@ -1,7 +1,7 @@
package com.baidu.hugegraph.store.metric;
import com.baidu.hugegraph.rocksdb.access.RocksDBFactory;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.store.HgStoreEngine;
import com.sun.management.OperatingSystemMXBean;
import lombok.extern.slf4j.Slf4j;

2
hg-store-core/src/main/java/com/baidu/hugegraph/store/options/RaftRocksdbOptions.java
View File

@ -3,7 +3,7 @@ package com.baidu.hugegraph.store.options;
import com.alipay.sofa.jraft.storage.impl.RocksDBLogStorage;
import com.alipay.sofa.jraft.util.StorageOptionsFactory;
import com.baidu.hugegraph.config.HugeConfig;
import com.baidu.hugegraph.rocksdb.access.RocksDBOptions;
import org.apache.hugegraph.rocksdb.access.RocksDBOptions;
import com.baidu.hugegraph.store.business.BusinessHandlerImpl;
import lombok.extern.slf4j.Slf4j;

2
hg-store-core/src/main/java/com/baidu/hugegraph/store/util/PartitionMetaStoreWrapper.java
View File

@ -1,6 +1,6 @@
package com.baidu.hugegraph.store.util;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.store.HgStoreEngine;
import com.baidu.hugegraph.store.meta.base.MetaStoreBase;

6
hg-store-core/src/test/java/com/baidu/hugegraph/store/HgBusinessImplTest.java
View File

@ -1,8 +1,8 @@
package com.baidu.hugegraph.store;
import com.baidu.hugegraph.rocksdb.access.RocksDBFactory;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.store.business.BusinessHandler;
import com.baidu.hugegraph.store.business.BusinessHandlerImpl;
import com.baidu.hugegraph.store.options.HgStoreEngineOptions;

4
hg-store-core/src/test/java/com/baidu/hugegraph/store/UnitTestBase.java
View File

@ -1,7 +1,7 @@
package com.baidu.hugegraph.store;
import com.baidu.hugegraph.rocksdb.access.RocksDBFactory;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.store.business.BusinessHandler;
import com.baidu.hugegraph.store.business.BusinessHandlerImpl;
import com.baidu.hugegraph.store.meta.PartitionManager;

2
hg-store-core/src/test/java/com/baidu/hugegraph/store/meta/GraphIDManagerTest.java
View File

@ -1,7 +1,7 @@
package com.baidu.hugegraph.store.meta;
import com.baidu.hugegraph.pd.common.PDException;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.store.UnitTestBase;
import com.baidu.hugegraph.store.meta.base.DBSessionBuilder;
import org.junit.Assert;

17
hg-store-dist/pom.xml
View File

@ -1,4 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

17
hg-store-dist/src/assembly/descriptor/server-assembly.xml
View File

@ -1,3 +1,20 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<assembly>
<id>distribution</id>
<includeBaseDirectory>false</includeBaseDirectory>

17
hg-store-dist/src/assembly/static/bin/restart-hugegraph-store.sh
View File

@ -1,5 +1,22 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
function abs_path() {
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do

17
hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh
View File

@ -1,5 +1,22 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
function abs_path() {
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do

17
hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh
View File

@ -1,5 +1,22 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
abs_path() {
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do

17
hg-store-dist/src/assembly/static/bin/util.sh
View File

@ -1,5 +1,22 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
function command_available() {
local cmd=$1
if [ `command -v $cmd >/dev/null 2>&1` ]; then

35
hg-store-dist/src/assembly/static/conf/log4j2.xml
View File

@ -1,5 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<!-- Config will be auto loaded every 60s -->
<configuration status="error" monitorInterval="60">
<properties>
@ -27,7 +44,7 @@
<!-- Trigger after exceeding 1day or 50MB -->
<Policies>
<SizeBasedTriggeringPolicy size="128MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true" />
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<!-- Keep 5 files per day & auto Delete after over 2GB or 100 files -->
<DefaultRolloverStrategy max="16">
@ -35,8 +52,8 @@
<IfFileName glob="*/${FILE_NAME}*.log"/>
<!-- Limit log amount & size -->
<IfAny>
<IfAccumulatedFileSize exceeds="2GB" />
<IfAccumulatedFileCount exceeds="100" />
<IfAccumulatedFileSize exceeds="2GB"/>
<IfAccumulatedFileCount exceeds="100"/>
</IfAny>
</Delete>
</DefaultRolloverStrategy>
@ -54,7 +71,7 @@
<!-- Trigger after exceeding 1day or 50MB -->
<Policies>
<SizeBasedTriggeringPolicy size="128MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true" />
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<!-- Keep 5 files per day & auto Delete after over 2GB or 100 files -->
<DefaultRolloverStrategy max="16">
@ -62,8 +79,8 @@
<IfFileName glob="*/${RAFT_FILE_NAME}*.log"/>
<!-- Limit log amount & size -->
<IfAny>
<IfAccumulatedFileSize exceeds="2GB" />
<IfAccumulatedFileCount exceeds="100" />
<IfAccumulatedFileSize exceeds="2GB"/>
<IfAccumulatedFileCount exceeds="100"/>
</IfAny>
</Delete>
</DefaultRolloverStrategy>
@ -82,7 +99,7 @@
<!-- Trigger after exceeding 1hour or 500MB -->
<Policies>
<SizeBasedTriggeringPolicy size="512MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true" />
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<!-- Keep 2 files per hour & auto Delete [after 60 days] or [over 5GB or 500 files] -->
<DefaultRolloverStrategy max="16">
@ -90,8 +107,8 @@
<IfFileName glob="*/${AUDIT_FILE_NAME}*.log.gz"/>
<IfLastModified age="60d"/>
<IfAny>
<IfAccumulatedFileSize exceeds="5GB" />
<IfAccumulatedFileCount exceeds="500" />
<IfAccumulatedFileSize exceeds="5GB"/>
<IfAccumulatedFileCount exceeds="500"/>
</IfAny>
</Delete>
</DefaultRolloverStrategy>

19
hg-store-node/pom.xml
View File

@ -1,4 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@ -140,7 +157,7 @@
</goals>
<configuration>
<mainClass>
com.baidu.hugegraph.store.node.StoreNodeApplication
org.apache.hugegraph.store.node.StoreNodeApplication
</mainClass>
</configuration>
</execution>

17
hg-store-node/src/assembly/static/bin/app.sh
View File

@ -1,4 +1,21 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
#ver 0.1.0 liyan75 on 2021/10/08
#readonly CUR_SHELL=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
readonly CUR_SHELL_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )

17
hg-store-node/src/assembly/static/bin/hg-get-release.sh
View File

@ -1,4 +1,21 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
#ver 0.1.0 liyan75 on 2021/12/17
readonly REPO_SNAPSHOT_PATH=http://10.14.139.8:8082/artifactory/star
readonly REPO_FILE_PATH=http://10.14.139.8:8082/artifactory/star-file

17
hg-store-node/src/assembly/static/bin/hg-update.sh
View File

@ -1,4 +1,21 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to You under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
#ver 0.1.0 liyan75 on 2021/12/17
readonly REPO_SNAPSHOT_PATH=http://10.14.139.8:8082/artifactory/star-snapshot
readonly REPO_FILE_PATH=http://10.14.139.8:8082/artifactory/star-file

42
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/AppShutdownHook.java
View File

@ -1,42 +0,0 @@
package com.baidu.hugegraph.store.node;
import com.baidu.hugegraph.rocksdb.access.RocksDBFactory;
/**
* @author lynn.bond@hotmail.com copy from web
*/
public class AppShutdownHook extends Thread {
private Thread mainThread;
private boolean shutDownSignalReceived;
public AppShutdownHook(Thread mainThread) {
super();
this.mainThread = mainThread;
this.shutDownSignalReceived = false;
Runtime.getRuntime().addShutdownHook(this);
}
@Override
public void run() {
System.out.println("Shut down signal received.");
this.shutDownSignalReceived = true;
mainThread.interrupt();
doSomethingForShutdown();
try {
mainThread.join(); //当收到停止信号时，等待mainThread的执行完成
} catch (InterruptedException e) {
}
System.out.println("Shut down complete.");
}
public boolean shouldShutDown() {
return shutDownSignalReceived;
}
private void doSomethingForShutdown() {
RocksDBFactory.getInstance().releaseAllGraphDB();
}
}

49
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/HgStoreMetricsController.java
View File

@ -1,49 +0,0 @@
package com.baidu.hugegraph.store.node.controller;
import com.alipay.sofa.jraft.core.NodeMetrics;
import com.baidu.hugegraph.store.node.grpc.HgStoreNodeService;
import com.baidu.hugegraph.store.node.metrics.DriveMetrics;
import com.baidu.hugegraph.store.node.metrics.SystemMetrics;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import java.util.HashMap;
import java.util.Map;
/**
* @author lynn.bond@hotmail.com on 2021/11/23
*/
@RestController
@RequestMapping(value = "/metrics",method = RequestMethod.GET)
public class HgStoreMetricsController {
@Autowired
HgStoreNodeService nodeService;
private SystemMetrics systemMetrics = new SystemMetrics();
private DriveMetrics driveMetrics = new DriveMetrics();
@GetMapping
public Map<String,String> index(){
return new HashMap<>();
}
@GetMapping("system")
public Map<String, Map<String, Object>> system(){
return this.systemMetrics.metrics();
}
@GetMapping("drive")
public Map<String,Map<String,Object>> drive(){
return this.driveMetrics.metrics();
}
@GetMapping("raft")
public Map<String, NodeMetrics> getRaftMetrics(){
return nodeService.getNodeMetrics();
}
}

14
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/entry/RestResult.java
View File

@ -1,14 +0,0 @@
package com.baidu.hugegraph.store.node.entry;
import java.io.Serializable;
import lombok.Data;
@Data
public class RestResult implements Serializable {
public static final String OK = "OK";
public static final String ERR = "ERR";
String state;
String message;
Serializable data;
}

39
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/EmptyIterator.java
View File

@ -1,39 +0,0 @@
package com.baidu.hugegraph.store.node.grpc;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
/**
* @author lynn.bond@hotmail.com on 2021/11/29
*/
final class EmptyIterator implements ScanIterator {
@Override
public boolean hasNext() {
return false;
}
@Override
public boolean isValid() {
return false;
}
@Override
public <T> T next() {
return null;
}
@Override
public long count() {
return 0;
}
@Override
public byte[] position() {
return new byte[0];
}
@Override
public void close() {
}
}

29
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/GRpcServerConfig.java
View File

@ -1,29 +0,0 @@
package com.baidu.hugegraph.store.node.grpc;
import org.lognet.springboot.grpc.GRpcServerBuilderConfigurer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.baidu.hugegraph.store.node.AppConfig;
import com.baidu.hugegraph.store.node.util.HgExecutorUtil;
import io.grpc.ServerBuilder;
/**
* @author lynn.bond@hotmail.com on 2022/3/4
*/
@Component
public class GRpcServerConfig extends GRpcServerBuilderConfigurer {
public final static String EXECUTOR_NAME = "hg-grpc";
@Autowired
private AppConfig appConfig;
@Override
public void configure(ServerBuilder<?> serverBuilder) {
AppConfig.ThreadPoolGrpc grpc = appConfig.getThreadPoolGrpc();
serverBuilder.executor(HgExecutorUtil.createExecutor(EXECUTOR_NAME, grpc.getCore(), grpc.getMax(),
grpc.getQueue())
);
}
}

40
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreNodeState.java
View File

@ -1,40 +0,0 @@
package com.baidu.hugegraph.store.node.grpc;
import com.baidu.hugegraph.store.grpc.state.NodeStateType;
import javax.annotation.concurrent.ThreadSafe;
/**
* @author lynn.bond@hotmail.com created on 2021/11/3
*/
@ThreadSafe
public final class HgStoreNodeState {
private static NodeStateType curState = NodeStateType.STARTING;
public static NodeStateType getState() {
return curState;
}
private static void setState(NodeStateType state) {
curState = state;
change();
}
private static void change() {
HgStoreStateSubject.notifyAll(curState);
}
public static void goOnline() {
setState(NodeStateType.ONLINE);
}
public static void goStarting() {
setState(NodeStateType.STARTING);
}
public static void goStopping() {
setState(NodeStateType.STOPPING);
}
}

22
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/QueryCondition.java
View File

@ -1,22 +0,0 @@
package com.baidu.hugegraph.store.node.grpc;
/**
* @author lynn.bond@hotmail.com on 2023/2/8
*/
public interface QueryCondition {
byte[] getStart();
byte[] getEnd();
byte[] getPrefix();
int getKeyCode();
int getScanType();
byte[] getQuery();
byte[] getPosition();
int getSerialNo();
}

78
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanOneShotResponse.java
View File

@ -1,78 +0,0 @@
package com.baidu.hugegraph.store.node.grpc;
import static com.baidu.hugegraph.store.node.grpc.ScanUtil.getIterator;
import static com.baidu.hugegraph.store.node.grpc.ScanUtil.toSq;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.store.grpc.common.Kv;
import com.baidu.hugegraph.store.grpc.stream.KvPageRes;
import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq;
import com.baidu.hugegraph.store.node.util.HgGrpc;
import com.baidu.hugegraph.store.node.util.HgStoreNodeUtil;
import com.google.protobuf.ByteString;
import io.grpc.Status;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2022/02/17
* @version 3.6.0
*/
@Slf4j
public class ScanOneShotResponse{
/**
* Handle one-shot scan
*
* @param request
* @param responseObserver
*/
public static void scanOneShot(ScanStreamReq request, StreamObserver<KvPageRes> responseObserver,
HgStoreWrapperEx wrapper) {
KvPageRes.Builder resBuilder = KvPageRes.newBuilder();
Kv.Builder kvBuilder = Kv.newBuilder();
ScanIterator iterator = getIterator(toSq(request), wrapper);
long limit = request.getLimit();
if (limit <= 0) {
responseObserver.onError(HgGrpc.toErr("limit<=0, please to invoke stream scan."));
return;
}
int count = 0;
try {
while (iterator.hasNext()) {
if (++count > limit) {
break;
}
RocksDBSession.BackendColumn col = iterator.next();
resBuilder.addData(kvBuilder
.setKey(ByteString.copyFrom(col.name))
.setValue(ByteString.copyFrom(col.value))
.setCode(HgStoreNodeUtil.toInt(iterator.position())) //position == partition-id.
);
}
responseObserver.onNext(resBuilder.build());
responseObserver.onCompleted();
} catch (Throwable t) {
String msg = "an exception occurred during data scanning";
responseObserver.onError(HgGrpc.toErr(Status.INTERNAL, msg, t));
}finally {
iterator.close();
}
}
}

87
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanQuery.java
View File

@ -1,87 +0,0 @@
package com.baidu.hugegraph.store.node.grpc;
import com.baidu.hugegraph.store.grpc.common.ScanMethod;
import java.util.Arrays;
/**
* @author lynn.bond@hotmail.com on 2022/2/28
*/
class ScanQuery implements QueryCondition{
String graph;
String table;
ScanMethod method;
byte[] start;
byte[] end;
byte[] prefix;
int keyCode;
int scanType;
byte[] query;
byte[] position;
int serialNo;
@Override
public byte[] getStart() {
return this.start;
}
@Override
public byte[] getEnd() {
return this.end;
}
@Override
public byte[] getPrefix() {
return this.prefix;
}
@Override
public int getKeyCode() {
return this.keyCode;
}
@Override
public int getScanType() {
return this.scanType;
}
@Override
public byte[] getQuery() {
return this.query;
}
@Override
public byte[] getPosition() {
return this.position;
}
@Override
public int getSerialNo() {
return this.serialNo;
}
static ScanQuery of() {
return new ScanQuery();
}
private ScanQuery() {
}
@Override
public String toString() {
return "ScanQuery{" +
"graph='" + graph + '\'' +
", table='" + table + '\'' +
", method=" + method +
", start=" + Arrays.toString(start) +
", end=" + Arrays.toString(end) +
", prefix=" + Arrays.toString(prefix) +
", partition=" + keyCode +
", scanType=" + scanType +
", serialNo=" + serialNo +
", query=" + Arrays.toString(query) +
", position=" + Arrays.toString(position) +
'}';
}
}

40
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/listener/ContextClosedListener.java
View File

@ -1,40 +0,0 @@
package com.baidu.hugegraph.store.node.listener;
import java.util.concurrent.ThreadPoolExecutor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationListener;
import org.springframework.context.event.ContextClosedEvent;
import com.baidu.hugegraph.store.node.grpc.HgStoreStreamImpl;
import lombok.extern.slf4j.Slf4j;
/**
* @author zhangyingjie
* @date 2023/2/17
**/
@Slf4j
public class ContextClosedListener implements ApplicationListener<ContextClosedEvent> {
@Autowired
HgStoreStreamImpl storeStream;
@Override
public void onApplicationEvent(ContextClosedEvent event) {
try {
log.info("closing scan threads....");
ThreadPoolExecutor executor = storeStream.getRealExecutor();
if (executor != null) {
try {
executor.shutdownNow();
} catch (Exception e) {
}
}
} catch (Exception e) {
} finally {
log.info("closed scan threads");
}
}
}

39
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/DriveMetrics.java
View File

@ -1,39 +0,0 @@
package com.baidu.hugegraph.store.node.metrics;
import java.io.File;
import java.util.*;
import java.util.stream.Collectors;
/**
* @author lynn.bond@hotmail.com on 2021/11/23
*/
@Deprecated
public class DriveMetrics {
private static long MIB = 1024 * 1024;
// TODO: add a cache
public Map<String, Map<String, Object>> metrics() {
File[] rootDrive = File.listRoots();
if (rootDrive == null) {
return new LinkedHashMap(0);
}
Map<String, Map<String, Object>> metrics = new HashMap<>();
for (File d : rootDrive) {
Map<String, Object> buf = new HashMap<>();
buf.put("total_space", d.getTotalSpace() / MIB);
buf.put("free_space", d.getFreeSpace() / MIB);
buf.put("usable_space", d.getUsableSpace() / MIB);
buf.put("size_unit", "MB");
metrics.put(d.getPath().replace("\\",""), buf);
}
return metrics;
}
}

73
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/GRpcExMetrics.java
View File

@ -1,73 +0,0 @@
package com.baidu.hugegraph.store.node.metrics;
import com.baidu.hugegraph.store.node.grpc.GRpcServerConfig;
import com.baidu.hugegraph.store.node.util.HgExecutorUtil;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import java.util.concurrent.ThreadPoolExecutor;
/**
* @author lynn.bond@hotmail.com on 2022/3/8
*/
public class GRpcExMetrics {
public final static String PREFIX = "grpc";
private static MeterRegistry registry;
private final static ExecutorWrapper wrapper = new ExecutorWrapper();
private GRpcExMetrics() {}
public synchronized static void init(MeterRegistry meterRegistry) {
if (registry == null) {
registry = meterRegistry;
registerMeters();
}
}
private static void registerMeters() {
registerExecutor();
}
private static void registerExecutor(){
Gauge.builder(PREFIX + ".executor.pool.size",wrapper,(e)->e.getPoolSize())
.description("The current number of threads in the pool.")
.register(registry);
Gauge.builder(PREFIX + ".executor.core.pool.size",wrapper,(e)->e.getCorePoolSize())
.description("The largest number of threads that have ever simultaneously been in the pool.")
.register(registry);
Gauge.builder(PREFIX + ".executor.active.count",wrapper,(e)->e.getActiveCount())
.description("The approximate number of threads that are actively executing tasks.")
.register(registry);
}
private static class ExecutorWrapper{
ThreadPoolExecutor pool;
void init(){
if(this.pool==null){
pool=HgExecutorUtil.getThreadPoolExecutor(GRpcServerConfig.EXECUTOR_NAME);
}
}
double getPoolSize(){
init();
return this.pool==null?0d:this.pool.getPoolSize();
}
int getCorePoolSize(){
init();
return this.pool==null?0:this.pool.getCorePoolSize();
}
int getActiveCount(){
init();
return this.pool==null?0:this.pool.getActiveCount();
}
}
}

30
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/MetricsConfig.java
View File

@ -1,30 +0,0 @@
package com.baidu.hugegraph.store.node.metrics;
import io.micrometer.core.instrument.MeterRegistry;
import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* @author lynn.bond@hotmail.com on 2021/11/24
*/
@Configuration
public class MetricsConfig {
@Bean
public MeterRegistryCustomizer<MeterRegistry> metricsCommonTags() {
return (registry) -> registry.config().commonTags("hg", "store");
}
@Bean
public MeterRegistryCustomizer<MeterRegistry> registerMeters() {
return (registry) -> {
StoreMetrics.init(registry);
RocksDBMetrics.init(registry);
JRaftMetrics.init(registry);
ProcfsMetrics.init(registry);
GRpcExMetrics.init(registry);
};
}
}

49
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/metrics/ProcfsMetrics.java
View File

@ -1,49 +0,0 @@
package com.baidu.hugegraph.store.node.metrics;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
/**
* @author lynn.bond@hotmail.com on 2022/3/1
* @version 0.1.0
*/
public class ProcfsMetrics {
private static MeterRegistry registry;
public final static String PREFIX = "process_memory";
private final static ProcfsSmaps smaps = new ProcfsSmaps();;
private ProcfsMetrics() {
}
public synchronized static void init(MeterRegistry meterRegistry) {
if (registry == null) {
registry = meterRegistry;
registerMeters();
}
}
private static void registerMeters() {
registerProcessGauge();
}
private static void registerProcessGauge() {
Gauge.builder(PREFIX + ".rss.bytes",()-> smaps.get(ProcfsSmaps.KEY.RSS))
.register(registry);
Gauge.builder(PREFIX + ".pss.bytes",()-> smaps.get(ProcfsSmaps.KEY.PSS))
.register(registry);
Gauge.builder(PREFIX + ".vss.bytes",()-> smaps.get(ProcfsSmaps.KEY.VSS))
.register(registry);
Gauge.builder(PREFIX + ".swap.bytes",()-> smaps.get(ProcfsSmaps.KEY.SWAP))
.register(registry);
Gauge.builder(PREFIX + ".swappss.bytes",()-> smaps.get(ProcfsSmaps.KEY.SWAPPSS))
.register(registry);
}
}

23
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/Err.java
View File

@ -1,23 +0,0 @@
package com.baidu.hugegraph.store.node.util;
/**
* @author lynn.bond@hotmail.com
*/
class Err {
private String msg;
public static Err of(String msg){
return new Err(msg);
}
private Err(String msg){
this.msg=msg;
}
@Override
public String toString() {
return "Err{" +
"msg='" + msg + '\'' +
'}';
}
}

21
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/HgStoreConst.java
View File

@ -1,21 +0,0 @@
package com.baidu.hugegraph.store.node.util;
import java.util.Collections;
import java.util.List;
/**
* @author lynn.bond@hotmail.com created on 2021/10/22
*/
public final class HgStoreConst {
public final static int SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS=300;
public final static byte[] EMPTY_BYTES=new byte[0];
public static final List EMPTY_LIST = Collections.EMPTY_LIST;
public final static int SCAN_ALL_PARTITIONS_ID=-1; // means scan all partitions.
private HgStoreConst(){}
}

22
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/Result.java
View File

@ -1,22 +0,0 @@
package com.baidu.hugegraph.store.node.util;
/**
* @author lynn.bond@hotmail.com
*/
public class Result<T>{
private Err err;
private T t;
public static Result of(){
return new Result();
}
private Result (){}
public T get(){
return t;
}
public void set(T t){
this.t=t;
}
public void err(String msg){
this.err=Err.of(msg);
}
}

48
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/util/TkEntry.java
View File

@ -1,48 +0,0 @@
package com.baidu.hugegraph.store.node.util;
import java.util.Arrays;
import java.util.Objects;
/**
* Table Key pair.
*/
public class TkEntry {
private String table;
private byte[] key;
public TkEntry(String table, byte[] key) {
this.table = table;
this.key = key;
}
public String getTable() {
return table;
}
public byte[] getKey() {
return key;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof TkEntry)) return false;
TkEntry tk = (TkEntry) o;
return Objects.equals(table, tk.table) && Arrays.equals(key, tk.key);
}
@Override
public int hashCode() {
int result = Objects.hash(table);
result = 31 * result + Arrays.hashCode(key);
return result;
}
@Override
public String toString() {
return "Tk{" +
"table='" + table + '\'' +
", key=" + Arrays.toString(key) +
'}';
}
}

23
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/AppConfig.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java
View File

@ -1,4 +1,21 @@
package com.baidu.hugegraph.store.node;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node;
import java.util.HashMap;
import java.util.Map;
@ -158,14 +175,14 @@ public class AppConfig {
@Configuration
@ConfigurationProperties(prefix = "app")
public class LabelConfig {
private Map<String, String> label = new HashMap<>();
private final Map<String, String> label = new HashMap<>();
}
@Data
@Configuration
@ConfigurationProperties(prefix = "")
public class RocksdbConfig {
private Map<String, String> rocksdb = new HashMap<>();
private final Map<String, String> rocksdb = new HashMap<>();
}
@PostConstruct

59
hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppShutdownHook.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
/**
* @author lynn.bond@hotmail.com copy from web
*/
public class AppShutdownHook extends Thread {
private final Thread mainThread;
private boolean shutDownSignalReceived;
public AppShutdownHook(Thread mainThread) {
super();
this.mainThread = mainThread;
this.shutDownSignalReceived = false;
Runtime.getRuntime().addShutdownHook(this);
}
@Override
public void run() {
System.out.println("Shut down signal received.");
this.shutDownSignalReceived = true;
mainThread.interrupt();
doSomethingForShutdown();
try {
mainThread.join(); //当收到停止信号时，等待mainThread的执行完成
} catch (InterruptedException ignored) {
}
System.out.println("Shut down complete.");
}
public boolean shouldShutDown() {
return shutDownSignalReceived;
}
private void doSomethingForShutdown() {
RocksDBFactory.getInstance().releaseAllGraphDB();
}
}

38
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/StoreNodeApplication.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java
View File

@ -1,12 +1,30 @@
package com.baidu.hugegraph.store.node;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
import com.alipay.remoting.util.StringUtils;
import com.baidu.hugegraph.store.node.listener.ContextClosedListener;
import com.baidu.hugegraph.store.node.listener.PdConfigureListener;
package org.apache.hugegraph.store.node;
import org.apache.hugegraph.store.node.listener.ContextClosedListener;
import org.apache.hugegraph.store.node.listener.PdConfigureListener;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.ConfigurableApplicationContext;
import com.alipay.remoting.util.StringUtils;
/**
*
*/
@ -27,10 +45,14 @@ public class StoreNodeApplication {
System.setProperty("logging.path", "logs");
}
System.setProperty("com.alipay.remoting.client.log.level", "WARN");
if (System.getProperty("bolt.channel_write_buf_low_water_mark") == null)
System.setProperty("bolt.channel_write_buf_low_water_mark", Integer.toString(4 * 1024 * 1024));
if (System.getProperty("bolt.channel_write_buf_high_water_mark") == null)
System.setProperty("bolt.channel_write_buf_high_water_mark", Integer.toString(8 * 1024 * 1024));
if (System.getProperty("bolt.channel_write_buf_low_water_mark") == null) {
System.setProperty("bolt.channel_write_buf_low_water_mark",
Integer.toString(4 * 1024 * 1024));
}
if (System.getProperty("bolt.channel_write_buf_high_water_mark") == null) {
System.setProperty("bolt.channel_write_buf_high_water_mark",
Integer.toString(8 * 1024 * 1024));
}
SpringApplication application = new SpringApplication(StoreNodeApplication.class);
PdConfigureListener listener = new PdConfigureListener();
ContextClosedListener closedListener = new ContextClosedListener();

67
hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgStoreMetricsController.java Normal file
View File

@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.controller;
import java.util.HashMap;
import java.util.Map;
import org.apache.hugegraph.store.node.grpc.HgStoreNodeService;
import org.apache.hugegraph.store.node.metrics.DriveMetrics;
import org.apache.hugegraph.store.node.metrics.SystemMetrics;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import com.alipay.sofa.jraft.core.NodeMetrics;
/**
* @author lynn.bond@hotmail.com on 2021/11/23
*/
@RestController
@RequestMapping(value = "/metrics", method = RequestMethod.GET)
public class HgStoreMetricsController {
@Autowired
HgStoreNodeService nodeService;
private final SystemMetrics systemMetrics = new SystemMetrics();
private final DriveMetrics driveMetrics = new DriveMetrics();
@GetMapping
public Map<String, String> index() {
return new HashMap<>();
}
@GetMapping("system")
public Map<String, Map<String, Object>> system() {
return this.systemMetrics.metrics();
}
@GetMapping("drive")
public Map<String, Map<String, Object>> drive() {
return this.driveMetrics.metrics();
}
@GetMapping("raft")
public Map<String, NodeMetrics> getRaftMetrics() {
return nodeService.getNodeMetrics();
}
}

30
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/HgStoreStatusController.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgStoreStatusController.java
View File

@ -1,7 +1,28 @@
package com.baidu.hugegraph.store.node.controller;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.controller;
import java.io.Serializable;
import org.apache.hugegraph.store.node.entry.RestResult;
import org.apache.hugegraph.store.node.grpc.HgStoreNodeState;
import org.apache.hugegraph.store.node.grpc.HgStoreStreamImpl;
import org.apache.hugegraph.store.node.model.HgNodeStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
@ -11,10 +32,6 @@ import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import com.baidu.hugegraph.store.grpc.state.ScanState;
import com.baidu.hugegraph.store.node.entry.RestResult;
import com.baidu.hugegraph.store.node.grpc.HgStoreNodeState;
import com.baidu.hugegraph.store.node.grpc.HgStoreStreamImpl;
import com.baidu.hugegraph.store.node.model.HgNodeStatus;
import com.google.protobuf.util.JsonFormat;
/**
@ -27,7 +44,8 @@ public class HgStoreStatusController {
HgStoreStreamImpl streamImpl;
@GetMapping("/-/echo")
public HgNodeStatus greeting(@RequestParam(value = "name", defaultValue = "World") String name) {
public HgNodeStatus greeting(
@RequestParam(value = "name", defaultValue = "World") String name) {
return new HgNodeStatus(0, name + " is ok.");
}

54
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/HgTestController.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/HgTestController.java
View File

@ -1,11 +1,26 @@
package com.baidu.hugegraph.store.node.controller;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
import com.baidu.hugegraph.store.PartitionEngine;
import com.baidu.hugegraph.store.node.grpc.HgStoreNodeService;
import com.baidu.hugegraph.store.meta.Partition;
import com.baidu.hugegraph.store.meta.Store;
import com.baidu.hugegraph.store.cmd.HgCmdProcessor;
import lombok.extern.slf4j.Slf4j;
package org.apache.hugegraph.store.node.controller;
import java.util.ArrayList;
import java.util.List;
import org.apache.hugegraph.store.node.grpc.HgStoreNodeService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
@ -13,8 +28,11 @@ import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.List;
import com.baidu.hugegraph.store.PartitionEngine;
import com.baidu.hugegraph.store.meta.Partition;
import com.baidu.hugegraph.store.meta.Store;
import lombok.extern.slf4j.Slf4j;
/**
* 仅用于测试
@ -32,10 +50,10 @@ public class HgTestController {
Store store = null;
PartitionEngine engine = nodeService.getStoreEngine().getPartitionEngine(0);
for(Partition partition : engine.getPartitions().values()){
for (Partition partition : engine.getPartitions().values()) {
store = nodeService.getStoreEngine().getHgCmdClient()
.getStoreInfo(engine.getLeader().toString());
};
.getStoreInfo(engine.getLeader().toString());
}
return store;
}
@ -50,14 +68,15 @@ public class HgTestController {
public String deleteRaftNode(@PathVariable(value = "groupId") int groupId) {
List<String> graphs = new ArrayList<>();
PartitionEngine engine = nodeService.getStoreEngine().getPartitionEngine(groupId);
if ( engine != null ){
engine.getPartitions().forEach((k, v)->{
if (engine != null) {
engine.getPartitions().forEach((k, v) -> {
graphs.add(v.getGraphName());
});
nodeService.getStoreEngine().destroyPartitionEngine(groupId, graphs);
return "OK";
}else
} else {
return "未找到分区";
}
}
@ -79,9 +98,10 @@ public class HgTestController {
nodeService.getStoreEngine().getBusinessHandler().closeAll();
return "close all!";
}
@GetMapping(value = "/snapshot", produces = MediaType.APPLICATION_JSON_VALUE)
public String doSnapshot() {
nodeService.getStoreEngine().getPartitionEngines().forEach((k,v)->{
nodeService.getStoreEngine().getPartitionEngines().forEach((k, v) -> {
v.snapshot();
});
return "snapshot OK!";
@ -89,7 +109,7 @@ public class HgTestController {
@GetMapping(value = "/compact", produces = MediaType.APPLICATION_JSON_VALUE)
public String dbCompaction() {
nodeService.getStoreEngine().getPartitionEngines().forEach((k,v)->{
nodeService.getStoreEngine().getPartitionEngines().forEach((k, v) -> {
nodeService.getStoreEngine().getBusinessHandler().dbCompaction("", k);
});
return "snapshot OK!";

64
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/IndexAPI.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/IndexAPI.java
View File

@ -1,23 +1,42 @@
package com.baidu.hugegraph.store.node.controller;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
import com.alipay.sofa.jraft.entity.PeerId;
import com.alipay.sofa.jraft.util.Endpoint;
import com.baidu.hugegraph.pd.grpc.Metapb;
import com.baidu.hugegraph.store.metric.HgStoreMetric;
import com.baidu.hugegraph.store.node.grpc.HgStoreNodeService;
import com.baidu.hugegraph.store.meta.Partition;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
package org.apache.hugegraph.store.node.controller;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hugegraph.store.node.grpc.HgStoreNodeService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.alipay.sofa.jraft.entity.PeerId;
import com.alipay.sofa.jraft.util.Endpoint;
import com.baidu.hugegraph.pd.grpc.Metapb;
import com.baidu.hugegraph.store.meta.Partition;
import com.baidu.hugegraph.store.metric.HgStoreMetric;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@RestController
@Slf4j
@RequestMapping("/")
@ -34,12 +53,13 @@ public class IndexAPI {
}
@Data
class StoreInfo{
class StoreInfo {
private int leaderCount;
private int partitionCount;
}
@Data
public class Raft{
public class Raft {
private int groupId;
private String role;
private String conf;
@ -47,19 +67,19 @@ public class IndexAPI {
private long logIndex;
private List<PeerId> peers;
private List<PeerId> learners;
private List<PartitionInfo> partitions = new ArrayList<>();
private final List<PartitionInfo> partitions = new ArrayList<>();
}
@Data
public class PartitionInfo {
private int id; // region id
private String graphName;
private final int id; // region id
private final String graphName;
// Region key range [startKey, endKey)
private long startKey;
private long endKey;
private final long startKey;
private final long endKey;
private HgStoreMetric.Partition metric;
private String version;
private Metapb.PartitionState workState;
private final String version;
private final Metapb.PartitionState workState;
private String leader;

90
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/controller/PartitionAPI.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/controller/PartitionAPI.java
View File

@ -1,28 +1,55 @@
package com.baidu.hugegraph.store.node.controller;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.controller;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.node.AppConfig;
import org.apache.hugegraph.store.node.grpc.HgStoreNodeService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.alipay.sofa.jraft.entity.PeerId;
import com.alipay.sofa.jraft.util.Endpoint;
import com.baidu.hugegraph.pd.common.PDException;
import com.baidu.hugegraph.pd.grpc.Metapb;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.store.HgStoreEngine;
import com.baidu.hugegraph.store.PartitionEngine;
import com.baidu.hugegraph.store.business.BusinessHandler;
import com.baidu.hugegraph.store.metric.HgStoreMetric;
import com.baidu.hugegraph.store.business.InnerKeyCreator;
import com.baidu.hugegraph.store.node.AppConfig;
import com.baidu.hugegraph.store.node.grpc.HgStoreNodeService;
import com.baidu.hugegraph.store.meta.Partition;
import com.baidu.hugegraph.store.metric.HgStoreMetric;
import com.baidu.hugegraph.util.Bytes;
import com.taobao.arthas.agent.attach.ArthasAgent;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
import java.util.*;
@RestController
@Slf4j
@ -35,12 +62,13 @@ public class PartitionAPI {
AppConfig appConfig;
@GetMapping(value = "/partitions", produces = "application/json")
public Map<String, Object> getPartitions(@RequestParam(required= false, defaultValue = "") String flags) {
public Map<String, Object> getPartitions(
@RequestParam(required = false, defaultValue = "") String flags) {
boolean accurate = false;
if(!flags.isEmpty()) {
if (!flags.isEmpty()) {
List<String> flagList = Arrays.asList(flags.split(","));
if(flagList.contains("accurate")) {
if (flagList.contains("accurate")) {
accurate = true;
}
}
@ -65,13 +93,14 @@ public class PartitionAPI {
raft.setTerm(engine.getLeaderTerm());
raft.setLogIndex(engine.getCommittedIndex());
raft.setPartitionCount(engine.getPartitions().size());
for(Map.Entry<String, Partition> partitionEntry : engine.getPartitions().entrySet()) {
for (Map.Entry<String, Partition> partitionEntry : engine.getPartitions().entrySet()) {
String graphName = partitionEntry.getKey();
Partition pt = partitionEntry.getValue();
PartitionInfo partition = new PartitionInfo(pt);
// 此处为了打开所有的图，metric只返回已打开的图
businessHandler.getLatestSequenceNumber(graphName, pt.getId());
partition.setMetric(businessHandler.getPartitionMetric(graphName, pt.getId(), accurate));
partition.setMetric(
businessHandler.getPartitionMetric(graphName, pt.getId(), accurate));
partition.setLeader(pt.isLeader() == engine.isLeader() ? "OK" : "Error");
raft.getPartitions().add(partition);
}
@ -110,11 +139,13 @@ public class PartitionAPI {
return raft;
//return okMap("partition", rafts);
}
/**
* 打印分区的所有key
*/
@GetMapping(value = "/partition/dump/{id}", produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, Object> dumpPartition(@PathVariable(value = "id") int id) throws PDException {
public Map<String, Object> dumpPartition(@PathVariable(value = "id") int id) throws
PDException {
HgStoreEngine storeEngine = nodeService.getStoreEngine();
BusinessHandler handler = storeEngine.getBusinessHandler();
InnerKeyCreator innerKeyCreator = new InnerKeyCreator(handler);
@ -142,18 +173,20 @@ public class PartitionAPI {
* 打印分区的所有key
*/
@GetMapping(value = "/partition/clean/{id}", produces = MediaType.APPLICATION_JSON_VALUE)
public Map<String, Object> cleanPartition(@PathVariable(value = "id") int id) throws PDException {
public Map<String, Object> cleanPartition(@PathVariable(value = "id") int id) throws
PDException {
HgStoreEngine storeEngine = nodeService.getStoreEngine();
BusinessHandler handler = storeEngine.getBusinessHandler();
storeEngine.getPartitionEngine(id).getPartitions().forEach((graph, partition) -> {
handler.cleanPartition(graph, id);
handler.cleanPartition(graph, id);
});
return okMap("ok", null);
}
@GetMapping(value = "/arthasstart", produces = "application/json")
public Map<String, Object> arthasstart(@RequestParam(required= false, defaultValue = "") String flags) {
public Map<String, Object> arthasstart(
@RequestParam(required = false, defaultValue = "") String flags) {
HashMap<String, String> configMap = new HashMap<String, String>();
configMap.put("arthas.telnetPort", appConfig.getArthasConfig().getTelnetPort());
configMap.put("arthas.httpPort", appConfig.getArthasConfig().getHttpPort());
@ -165,8 +198,9 @@ public class PartitionAPI {
ret.add("Arthas 启动成功");
return okMap("arthasstart", ret);
}
@Data
public class Raft{
public class Raft {
private int groupId;
private String role;
private String conf;
@ -176,19 +210,19 @@ public class PartitionAPI {
private List<PeerId> peers;
private List<PeerId> learners;
private int partitionCount;
private List<PartitionInfo> partitions = new ArrayList<>();
private final List<PartitionInfo> partitions = new ArrayList<>();
}
@Data
public class PartitionInfo {
private int id; // region id
private String graphName;
private final int id; // region id
private final String graphName;
// Region key range [startKey, endKey)
private long startKey;
private long endKey;
private final long startKey;
private final long endKey;
private HgStoreMetric.Partition metric;
private String version;
private Metapb.PartitionState workState;
private final String version;
private final Metapb.PartitionState workState;
private String leader;

31
hg-store-node/src/main/java/org/apache/hugegraph/store/node/entry/RestResult.java Normal file
View File

@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.entry;
import java.io.Serializable;
import lombok.Data;
@Data
public class RestResult implements Serializable {
public static final String OK = "OK";
public static final String ERR = "ERR";
String state;
String message;
Serializable data;
}

82
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/BatchGrpcClosure.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchGrpcClosure.java
View File

@ -1,22 +1,26 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
/**
* @author lynn.bond@hotmail.com on 2022/1/27
*/
import com.alipay.sofa.jraft.Status;
import com.baidu.hugegraph.store.grpc.common.ResCode;
import com.baidu.hugegraph.store.grpc.common.ResStatus;
import com.baidu.hugegraph.store.grpc.session.FeedbackRes;
import com.baidu.hugegraph.store.grpc.session.PartitionFaultResponse;
import com.baidu.hugegraph.store.grpc.session.PartitionFaultType;
import com.baidu.hugegraph.store.grpc.session.PartitionLeader;
import com.baidu.hugegraph.store.raft.RaftClosure;
import com.baidu.hugegraph.store.util.HgRaftError;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -28,6 +32,21 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Function;
import org.apache.commons.collections.CollectionUtils;
import com.alipay.sofa.jraft.Status;
import com.baidu.hugegraph.store.grpc.common.ResCode;
import com.baidu.hugegraph.store.grpc.common.ResStatus;
import com.baidu.hugegraph.store.grpc.session.FeedbackRes;
import com.baidu.hugegraph.store.grpc.session.PartitionFaultResponse;
import com.baidu.hugegraph.store.grpc.session.PartitionFaultType;
import com.baidu.hugegraph.store.grpc.session.PartitionLeader;
import com.baidu.hugegraph.store.raft.RaftClosure;
import com.baidu.hugegraph.store.util.HgRaftError;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
/**
* 批量处理的grpc回调封装类
*
@ -80,6 +99,7 @@ class BatchGrpcClosure<V> {
/**
* 不使用计数器latch
*
* @return
*/
public RaftClosure newClosureNoLatch() {
@ -102,11 +122,12 @@ class BatchGrpcClosure<V> {
if (leaderMap.size() > 0) {
PartitionFaultResponse.Builder partitionFault =
PartitionFaultResponse.newBuilder().setFaultType(PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LEADER);
PartitionFaultResponse.newBuilder().setFaultType(
PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LEADER);
leaderMap.forEach((k, v) -> {
partitionFault.addPartitionLeaders(PartitionLeader.newBuilder()
.setPartitionId(k)
.setLeaderId(v).build());
.setPartitionId(k)
.setLeaderId(v).build());
});
errorResponse = partitionFault.build();
} else {
@ -122,7 +143,7 @@ class BatchGrpcClosure<V> {
faultType = PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LOCAL;
break;
default:
log.error("Unmatchable errorStatus: "+errorStatus);
log.error("Unmatchable errorStatus: " + errorStatus);
}
errorResponse = PartitionFaultResponse.newBuilder().setFaultType(faultType).build();
}
@ -151,15 +172,19 @@ class BatchGrpcClosure<V> {
observer.onNext(ok.apply(results));
} else {
observer.onNext((V) FeedbackRes.newBuilder()
.setStatus(ResStatus.newBuilder().setCode(ResCode.RES_CODE_FAIL).setMsg(getErrorMsg()))
.setPartitionFaultResponse(this.getErrorResponse())
.build());
.setStatus(ResStatus.newBuilder()
.setCode(ResCode.RES_CODE_FAIL)
.setMsg(getErrorMsg()))
.setPartitionFaultResponse(this.getErrorResponse())
.build());
}
} catch (InterruptedException e) {
log.error("waitFinish exception: ", e);
observer.onNext((V) FeedbackRes.newBuilder()
.setStatus(ResStatus.newBuilder().setCode(ResCode.RES_CODE_FAIL)
.setMsg(e.getLocalizedMessage()).build()).build());
.setStatus(ResStatus.newBuilder()
.setCode(ResCode.RES_CODE_FAIL)
.setMsg(e.getLocalizedMessage())
.build()).build());
}
observer.onCompleted();
}
@ -172,18 +197,19 @@ class BatchGrpcClosure<V> {
AtomicReference<FeedbackRes> res = new AtomicReference<>(results.get(0));
results.forEach(e -> {
try {
if (e.getStatus().getCode() != ResCode.RES_CODE_OK)
if (e.getStatus().getCode() != ResCode.RES_CODE_OK) {
res.set(e);
}catch (Exception ex){
}
} catch (Exception ex) {
log.error("{}", ex);
}
});
return res.get();
} else {
return FeedbackRes.newBuilder()
.setStatus(ResStatus.newBuilder()
.setCode(ResCode.RES_CODE_OK).build())
.build();
.setStatus(ResStatus.newBuilder()
.setCode(ResCode.RES_CODE_OK).build())
.build();
}
}

46
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/BatchScanIterator.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchScanIterator.java
View File

@ -1,16 +1,34 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
import com.baidu.hugegraph.pd.common.KVPair;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.store.node.util.HgAssert;
import com.baidu.hugegraph.store.node.util.HgStoreConst;
import lombok.extern.slf4j.Slf4j;
package org.apache.hugegraph.store.node.grpc;
import javax.annotation.concurrent.NotThreadSafe;
import java.util.NoSuchElementException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.node.util.HgAssert;
import org.apache.hugegraph.store.node.util.HgStoreConst;
import com.baidu.hugegraph.pd.common.KVPair;
/**
* @author lynn.bond@hotmail.com on 2022/2/28
* @version 0.3.0 added limit support
@ -24,9 +42,11 @@ public final class BatchScanIterator implements ScanIterator {
private boolean hasNext = false;
private long curCount;
private long curLimit;
private AtomicBoolean closed=new AtomicBoolean();
public static BatchScanIterator of(Supplier<KVPair<QueryCondition, ScanIterator>> iteratorSupplier,
Supplier<Long> limitSupplier) {
private final AtomicBoolean closed = new AtomicBoolean();
public static BatchScanIterator of(
Supplier<KVPair<QueryCondition, ScanIterator>> iteratorSupplier,
Supplier<Long> limitSupplier) {
HgAssert.isArgumentNotNull(iteratorSupplier, "iteratorSupplier");
HgAssert.isArgumentNotNull(limitSupplier, "limitSupplier");
return new BatchScanIterator(iteratorSupplier, limitSupplier);
@ -39,9 +59,9 @@ public final class BatchScanIterator implements ScanIterator {
}
private ScanIterator getIterator() {
ScanIterator buf = null;
ScanIterator buf;
int count = 0;
this.curCount = 0l;
this.curCount = 0L;
do {
buf = this.batchSupplier.get().getValue();
@ -114,7 +134,7 @@ public final class BatchScanIterator implements ScanIterator {
@Override
public void close() {
if(this.closed.getAndSet(true)==false){
if (!this.closed.getAndSet(true)) {
if (this.iterator != null) {
this.iterator.close();
}

56
hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/EmptyIterator.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
/**
* @author lynn.bond@hotmail.com on 2021/11/29
*/
final class EmptyIterator implements ScanIterator {
@Override
public boolean hasNext() {
return false;
}
@Override
public boolean isValid() {
return false;
}
@Override
public <T> T next() {
return null;
}
@Override
public long count() {
return 0;
}
@Override
public byte[] position() {
return new byte[0];
}
@Override
public void close() {
}
}

24
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/FusingScanIterator.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/FusingScanIterator.java
View File

@ -1,10 +1,27 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
package org.apache.hugegraph.store.node.grpc;
import java.util.NoSuchElementException;
import java.util.function.Supplier;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
/**
* This is a wrapper of the ScanIterator that provides a mechanism
* to set a threshold value in order to abort the iterating operation.
@ -19,7 +36,8 @@ final class FusingScanIterator implements ScanIterator {
private ScanIterator iterator;
private byte[] position = EMPTY_BYTES;
public static FusingScanIterator maxOf(long maxThreshold, Supplier<ScanIterator> iteratorSupplier) {
public static FusingScanIterator maxOf(long maxThreshold,
Supplier<ScanIterator> iteratorSupplier) {
FusingScanIterator res = new FusingScanIterator();
res.max = maxThreshold;
res.supplier = iteratorSupplier;

46
hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GRpcServerConfig.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import org.apache.hugegraph.store.node.AppConfig;
import org.apache.hugegraph.store.node.util.HgExecutorUtil;
import org.lognet.springboot.grpc.GRpcServerBuilderConfigurer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import io.grpc.ServerBuilder;
/**
* @author lynn.bond@hotmail.com on 2022/3/4
*/
@Component
public class GRpcServerConfig extends GRpcServerBuilderConfigurer {
public final static String EXECUTOR_NAME = "hg-grpc";
@Autowired
private AppConfig appConfig;
@Override
public void configure(ServerBuilder<?> serverBuilder) {
AppConfig.ThreadPoolGrpc grpc = appConfig.getThreadPoolGrpc();
serverBuilder.executor(
HgExecutorUtil.createExecutor(EXECUTOR_NAME, grpc.getCore(), grpc.getMax(),
grpc.getQueue())
);
}
}

31
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/GrpcClosure.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcClosure.java
View File

@ -1,20 +1,38 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
import com.baidu.hugegraph.store.grpc.session.FeedbackRes;
import com.baidu.hugegraph.store.raft.RaftClosure;
import io.grpc.stub.StreamObserver;
package org.apache.hugegraph.store.node.grpc;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.baidu.hugegraph.store.grpc.session.FeedbackRes;
import com.baidu.hugegraph.store.raft.RaftClosure;
import io.grpc.stub.StreamObserver;
/**
* @author lynn.bond@hotmail.com on 2022/1/27
*/
abstract class GrpcClosure<V> implements RaftClosure {
private V result;
private Map<Integer, Long> leaderMap = new HashMap<>();
private final Map<Integer, Long> leaderMap = new HashMap<>();
public V getResult() {
return result;
@ -38,8 +56,9 @@ abstract class GrpcClosure<V> implements RaftClosure {
*/
public static <V> void setResult(RaftClosure raftClosure, V result) {
GrpcClosure closure = (GrpcClosure) raftClosure;
if (closure != null)
if (closure != null) {
closure.setResult(result);
}
}
public static <V> RaftClosure newRaftClosure(StreamObserver<V> observer) {

111
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreNodeService.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java
View File

@ -1,61 +1,84 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import static com.baidu.hugegraph.store.grpc.common.GraphMethod.GRAPH_METHOD_DELETE;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import org.apache.hugegraph.store.node.AppConfig;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.alipay.sofa.jraft.Status;
import com.alipay.sofa.jraft.core.NodeMetrics;
import com.baidu.hugegraph.store.HgStoreEngine;
import com.baidu.hugegraph.store.business.DefaultDataMover;
import com.baidu.hugegraph.store.util.HgStoreException;
import com.baidu.hugegraph.store.grpc.session.*;
import com.baidu.hugegraph.store.node.AppConfig;
import com.baidu.hugegraph.store.options.HgStoreEngineOptions;
import com.baidu.hugegraph.store.options.RaftRocksdbOptions;
import com.baidu.hugegraph.store.raft.RaftClosure;
import com.baidu.hugegraph.store.raft.RaftOperation;
import com.baidu.hugegraph.store.raft.RaftTaskHandler;
import com.baidu.hugegraph.store.util.HgRaftError;
import com.baidu.hugegraph.store.util.HgStoreException;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.CodedOutputStream;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import static com.baidu.hugegraph.store.grpc.common.GraphMethod.GRAPH_METHOD_DELETE;
/**
* @projectName: raft task executor
*/
@Slf4j
@Service
public class HgStoreNodeService implements RaftTaskHandler{
public static final byte BATCH_OP = 0x12;
public static final byte TABLE_OP = 0x13;
public static final byte GRAPH_OP = 0x14;
public static final byte CLEAN_OP = 0x15;
public class HgStoreNodeService implements RaftTaskHandler {
public static final byte BATCH_OP = 0x12;
public static final byte TABLE_OP = 0x13;
public static final byte GRAPH_OP = 0x14;
public static final byte CLEAN_OP = 0x15;
public static final byte MAX_OP = 0x59;
public static final byte MAX_OP = 0x59;
@Autowired
HgStoreSessionImpl hgStoreSession;
private HgStoreEngine storeEngine;
private AppConfig appConfig;
private final AppConfig appConfig;
public HgStoreNodeService(@Autowired AppConfig appConfig) {
this.appConfig = appConfig;
}
public HgStoreEngine getStoreEngine(){ return this.storeEngine;}
public HgStoreEngine getStoreEngine() {
return this.storeEngine;
}
@PostConstruct
public void init(){
public void init() {
log.info("{}", appConfig.toString());
HgStoreEngineOptions options = new HgStoreEngineOptions(){{
HgStoreEngineOptions options = new HgStoreEngineOptions() {{
setRaftAddress(appConfig.getRaft().getAddress());
setDataPath(appConfig.getDataPath());
setRaftPath(appConfig.getRaftPath());
@ -64,7 +87,7 @@ public class HgStoreNodeService implements RaftTaskHandler{
setRocksdbConfig(appConfig.getRocksdbConfig());
setGrpcAddress(appConfig.getStoreServerAddress());
setLabels(appConfig.getLabelConfig().getLabel());
setRaftOptions(new RaftOptions(){{
setRaftOptions(new RaftOptions() {{
setMetrics(appConfig.getRaft().isMetrics());
setRpcDefaultTimeout(appConfig.getRaft().getRpcTimeOut());
setSnapshotLogIndexMargin(appConfig.getRaft().getSnapshotLogIndexMargin());
@ -76,7 +99,7 @@ public class HgStoreNodeService implements RaftTaskHandler{
setMaxSegmentFileSize(appConfig.getRaft().getMaxSegmentFileSize());
setMaxReplicatorInflightMsgs(appConfig.getRaft().getMaxReplicatorInflightMsgs());
}});
setFakePdOptions(new FakePdOptions(){{
setFakePdOptions(new FakePdOptions() {{
setStoreList(appConfig.getFakePdConfig().getStoreList());
setPeersList(appConfig.getFakePdConfig().getPeersList());
setPartitionCount(appConfig.getFakePdConfig().getPartitionCount());
@ -95,19 +118,21 @@ public class HgStoreNodeService implements RaftTaskHandler{
}
public List<Integer> getGraphLeaderPartitionIds(String graphName){
public List<Integer> getGraphLeaderPartitionIds(String graphName) {
return storeEngine.getPartitionManager().getLeaderPartitionIds(graphName);
}
/**
* 添加raft 任务，转发数据给raft
*
* @return true 表示数据已被提交，false表示未提交，用于单副本入库减少批次拆分
*/
public <Req extends com.google.protobuf.GeneratedMessageV3>
void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, RaftClosure closure) {
void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req,
RaftClosure closure) {
if (!storeEngine.isClusterReady()) {
closure.run(new Status(HgRaftError.CLUSTER_NOT_READY.getNumber(),
"The cluster is not ready, please check active stores number!"));
"The cluster is not ready, please check active stores number!"));
log.error("The cluster is not ready, please check active stores number!");
return;
}
@ -122,7 +147,7 @@ public class HgStoreNodeService implements RaftTaskHandler{
output.flush();
// 传送给raft
storeEngine.addRaftTask(graphName, partitionId,
RaftOperation.create(methodId, buffer, req), closure);
RaftOperation.create(methodId, buffer, req), closure);
} catch (Exception e) {
closure.run(new Status(HgRaftError.UNKNOWN.getNumber(), e.getMessage()));
@ -135,14 +160,15 @@ public class HgStoreNodeService implements RaftTaskHandler{
* 来自日志的任务，一般是follower 或者 日志回滚的任务
*/
@Override
public boolean invoke(int partId, byte[] request, RaftClosure response) throws HgStoreException {
public boolean invoke(int partId, byte[] request, RaftClosure response) throws
HgStoreException {
try {
CodedInputStream input = CodedInputStream.newInstance(request);
byte methodId = input.readRawByte();
switch (methodId){
switch (methodId) {
case HgStoreNodeService.BATCH_OP:
invoke(partId, methodId, BatchReq.parseFrom(input), response);
break ;
break;
case HgStoreNodeService.TABLE_OP:
invoke(partId, methodId, TableReq.parseFrom(input), response);
break;
@ -155,8 +181,8 @@ public class HgStoreNodeService implements RaftTaskHandler{
default:
return false; // 未处理
}
}catch (IOException e){
throw new HgStoreException(e.getMessage(), e);
} catch (IOException e) {
throw new HgStoreException(e.getMessage(), e);
}
return true;
}
@ -165,8 +191,9 @@ public class HgStoreNodeService implements RaftTaskHandler{
* 处理raft传送过来的数据
*/
@Override
public boolean invoke(int partId, byte methodId, Object req, RaftClosure response) throws HgStoreException {
switch (methodId){
public boolean invoke(int partId, byte methodId, Object req, RaftClosure response) throws
HgStoreException {
switch (methodId) {
case HgStoreNodeService.BATCH_OP:
hgStoreSession.doBatch(partId, (BatchReq) req, response);
break;
@ -189,20 +216,20 @@ public class HgStoreNodeService implements RaftTaskHandler{
}
@PreDestroy
public void destroy(){
public void destroy() {
storeEngine.shutdown();
}
private String getSerializingExceptionMessage(String target) {
return "Serializing "
+ getClass().getName()
+ " to a "
+ target
+ " threw an IOException (should never happen).";
+ getClass().getName()
+ " to a "
+ target
+ " threw an IOException (should never happen).";
}
public Map<String, NodeMetrics> getNodeMetrics(){
public Map<String, NodeMetrics> getNodeMetrics() {
return storeEngine.getNodeMetrics();
}
}

57
hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeState.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import javax.annotation.concurrent.ThreadSafe;
import com.baidu.hugegraph.store.grpc.state.NodeStateType;
/**
* @author lynn.bond@hotmail.com created on 2021/11/3
*/
@ThreadSafe
public final class HgStoreNodeState {
private static NodeStateType curState = NodeStateType.STARTING;
public static NodeStateType getState() {
return curState;
}
private static void setState(NodeStateType state) {
curState = state;
change();
}
private static void change() {
HgStoreStateSubject.notifyAll(curState);
}
public static void goOnline() {
setState(NodeStateType.ONLINE);
}
public static void goStarting() {
setState(NodeStateType.STARTING);
}
public static void goStopping() {
setState(NodeStateType.STOPPING);
}
}

190
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreSessionImpl.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreSessionImpl.java
View File

@ -1,7 +1,21 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
import static com.baidu.hugegraph.store.node.util.HgGrpc.toHgPair;
import static com.baidu.hugegraph.store.node.util.HgGrpc.toKv;
package org.apache.hugegraph.store.node.grpc;
import java.util.HashMap;
import java.util.LinkedList;
@ -9,6 +23,9 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hugegraph.store.node.AppConfig;
import org.apache.hugegraph.store.node.util.HgGrpc;
import org.apache.hugegraph.store.node.util.HgStoreNodeUtil;
import org.lognet.springboot.grpc.GRpcService;
import org.springframework.beans.factory.annotation.Autowired;
@ -33,9 +50,6 @@ import com.baidu.hugegraph.store.grpc.session.TableReq;
import com.baidu.hugegraph.store.grpc.session.ValueResponse;
import com.baidu.hugegraph.store.meta.Graph;
import com.baidu.hugegraph.store.meta.GraphManager;
import com.baidu.hugegraph.store.node.AppConfig;
import com.baidu.hugegraph.store.node.util.HgGrpc;
import com.baidu.hugegraph.store.node.util.HgStoreNodeUtil;
import com.baidu.hugegraph.store.pd.PdProvider;
import com.baidu.hugegraph.store.raft.RaftClosure;
import com.baidu.hugegraph.store.util.HgStoreConst;
@ -53,11 +67,14 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas
private HgStoreNodeService storeService;
private HgStoreWrapperEx wrapper;
private PdProvider pdProvider;
private HgStoreWrapperEx getWrapper() {
if (this.wrapper == null) {
synchronized (this) {
if (this.wrapper == null)
this.wrapper = new HgStoreWrapperEx(storeService.getStoreEngine().getBusinessHandler());
if (this.wrapper == null) {
this.wrapper = new HgStoreWrapperEx(
storeService.getStoreEngine().getBusinessHandler());
}
}
}
return this.wrapper;
@ -66,8 +83,9 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas
private PdProvider getPD() {
if (pdProvider == null) {
synchronized (this) {
if (pdProvider == null)
if (pdProvider == null) {
pdProvider = storeService.getStoreEngine().getPdProvider();
}
}
}
return pdProvider;
@ -86,20 +104,21 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas
FeedbackRes res = null;
if (value != null) {
res = builder.setStatus(HgGrpc.success())
.setValueResponse(
ValueResponse.newBuilder()
.setValue(ByteString.copyFrom(value))
).build();
.setValueResponse(
ValueResponse.newBuilder()
.setValue(ByteString.copyFrom(value))
).build();
} else {
res = builder.setStatus(HgGrpc.success())
.setStatus(HgGrpc.not())
.build();
.setStatus(HgGrpc.not())
.build();
}
responseObserver.onNext(res);
responseObserver.onCompleted();
}
@Override
public void clean(CleanReq request,
StreamObserver<FeedbackRes> responseObserver) {
@ -109,20 +128,22 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas
// 发给不同的raft执行
BatchGrpcClosure<FeedbackRes> closure = new BatchGrpcClosure<>(1);
storeService.addRaftTask(HgStoreNodeService.CLEAN_OP, graph, partition,
request,
closure.newRaftClosure());
request,
closure.newRaftClosure());
// 等待返回结果
closure.waitFinish(responseObserver, r -> closure.selectError(r), appConfig.getRaft().getRpcTimeOut());
closure.waitFinish(responseObserver, r -> closure.selectError(r),
appConfig.getRaft().getRpcTimeOut());
}
public void doClean(int partId, CleanReq request, RaftClosure response) {
String graph = request.getHeader().getGraph();
FeedbackRes.Builder builder = FeedbackRes.newBuilder();
try {
if (getWrapper().doClean(graph, partId))
if (getWrapper().doClean(graph, partId)) {
builder.setStatus(HgGrpc.success());
else
} else {
builder.setStatus(HgGrpc.not());
}
} catch (Throwable t) {
String msg = "Failed to doClean, graph: " + graph + "; partitionId = " + partId;
log.error(msg, t);
@ -151,23 +172,28 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas
AtomicInteger count = new AtomicInteger(-1);
Kv.Builder kvBuilder = Kv.newBuilder();
getWrapper().batchGet(graph, table,
() -> {
if (count.getAndAdd(1) == max) {
return null;
}
() -> {
if (count.getAndAdd(1) == max) {
return null;
}
Key key = keyList.get(count.get());
if(log.isDebugEnabled())log.debug("batch-get: " + HgStoreNodeUtil.toStr(key.getKey().toByteArray()));
return toHgPair(key);
},
(
pair -> {
if (pair.getValue() == null || pair.getKey() == null) {
return;
}
keyValueBuilder.addKv(toKv(pair, kvBuilder));
}
)
Key key = keyList.get(count.get());
if (log.isDebugEnabled()) {
log.debug("batch-get: " +
HgStoreNodeUtil.toStr(
key.getKey()
.toByteArray()));
}
return HgGrpc.toHgPair(key);
},
(
pair -> {
if (pair.getValue() == null || pair.getKey() == null) {
return;
}
keyValueBuilder.addKv(HgGrpc.toKv(pair, kvBuilder));
}
)
);
@ -299,7 +325,7 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas
builder.setStatus(HgGrpc.success());
} catch (Throwable t) {
String msg = "Failed to doBatch, graph: " + graph + "; batchId= " + batchId;
log.error(msg,t);
log.error(msg, t);
builder.setStatus(HgGrpc.fail(msg));
}
GrpcClosure.setResult(response, builder.build());
@ -342,11 +368,13 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas
@Override
public void table(TableReq request, StreamObserver<FeedbackRes> observer) {
if(log.isDebugEnabled())log.debug("table: method = {}, graph = {}, table = {}"
, request.getMethod().name()
, request.getHeader().getGraph()
, request.getTableName()
);
if (log.isDebugEnabled()) {
log.debug("table: method = {}, graph = {}, table = {}"
, request.getMethod().name()
, request.getHeader().getGraph()
, request.getTableName()
);
}
String graph = request.getHeader().getGraph();
// 所有Leader分区
@ -362,17 +390,18 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas
BatchGrpcClosure<FeedbackRes> closure = new BatchGrpcClosure<>(groups.size());
groups.forEach((partition, entries) -> {
storeService.addRaftTask(HgStoreNodeService.TABLE_OP, graph, partition,
TableReq.newBuilder(request).build(),
closure.newRaftClosure());
TableReq.newBuilder(request).build(),
closure.newRaftClosure());
});
if (!groups.isEmpty()) {
// log.info(" table waiting raft...");
// log.info(" table waiting raft...");
// 等待返回结果
closure.waitFinish(observer, r -> closure.selectError(r), appConfig.getRaft().getRpcTimeOut());
// log.info(" table ended waiting raft");
closure.waitFinish(observer, r -> closure.selectError(r),
appConfig.getRaft().getRpcTimeOut());
// log.info(" table ended waiting raft");
} else {
// log.info(" table none leader logic");
// log.info(" table none leader logic");
ResStatus status = null;
switch (request.getMethod()) {
@ -383,7 +412,7 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas
status = HgGrpc.success();
}
// log.info(" table none leader status: {}", status.getCode());
// log.info(" table none leader status: {}", status.getCode());
observer.onNext(FeedbackRes.newBuilder().setStatus(status).build());
observer.onCompleted();
}
@ -391,20 +420,22 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas
}
public void doTable(int partId, TableReq request, RaftClosure response) {
if(log.isDebugEnabled())log.debug(" - doTable[{}]: graph = {}, table = {}"
, request.getMethod().name()
, request.getHeader().getGraph()
, request.getTableName()
);
if (log.isDebugEnabled()) {
log.debug(" - doTable[{}]: graph = {}, table = {}"
, request.getMethod().name()
, request.getHeader().getGraph()
, request.getTableName()
);
}
FeedbackRes.Builder builder = FeedbackRes.newBuilder();
try {
log.debug(" - starting wrapper:doTable ");
if (getWrapper().doTable(partId,
request.getMethod(),
request.getHeader().getGraph(),
request.getTableName())) {
request.getMethod(),
request.getHeader().getGraph(),
request.getTableName())) {
builder.setStatus(HgGrpc.success());
} else {
builder.setStatus(HgGrpc.not());
@ -412,9 +443,9 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas
log.debug(" - ended wrapper:doTable ");
} catch (Throwable t) {
String msg = "Failed to invoke doTable[ "
+ request.getMethod().name() + " ], graph="
+ request.getHeader().getGraph() + " , table="
+ request.getTableName();
+ request.getMethod().name() + " ], graph="
+ request.getHeader().getGraph() + " , table="
+ request.getTableName();
log.error(msg, t);
builder.setStatus(HgGrpc.fail(msg));
}
@ -426,11 +457,13 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas
@Override
public void graph(GraphReq request, StreamObserver<FeedbackRes> observer) {
if(log.isDebugEnabled())log.debug("graph: method = {}, graph = {}, table = {}"
, request.getMethod().name()
, request.getHeader().getGraph()
, request.getGraphName()
);
if (log.isDebugEnabled()) {
log.debug("graph: method = {}, graph = {}, table = {}"
, request.getMethod().name()
, request.getHeader().getGraph()
, request.getGraphName()
);
}
String graph = request.getHeader().getGraph();
// 所有Leader分区
@ -446,13 +479,14 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas
BatchGrpcClosure<FeedbackRes> closure = new BatchGrpcClosure<>(groups.size());
groups.forEach((partition, entries) -> {
storeService.addRaftTask(HgStoreNodeService.GRAPH_OP, graph, partition,
GraphReq.newBuilder(request).build(),
closure.newRaftClosure());
GraphReq.newBuilder(request).build(),
closure.newRaftClosure());
});
if (!groups.isEmpty()) {
// 等待返回结果
closure.waitFinish(observer, r -> closure.selectError(r), appConfig.getRaft().getRpcTimeOut());
closure.waitFinish(observer, r -> closure.selectError(r),
appConfig.getRaft().getRpcTimeOut());
} else {
observer.onNext(FeedbackRes.newBuilder().setStatus(HgGrpc.success()).build());
@ -462,26 +496,28 @@ public class HgStoreSessionImpl extends HgStoreSessionGrpc.HgStoreSessionImplBas
}
public void doGraph(int partId, GraphReq request, RaftClosure response) {
if (log.isDebugEnabled()) log.debug(" - doGraph[{}]: graph = {}, table = {}"
, request.getMethod().name()
, request.getHeader().getGraph()
, request.getGraphName()
);
if (log.isDebugEnabled()) {
log.debug(" - doGraph[{}]: graph = {}, table = {}"
, request.getMethod().name()
, request.getHeader().getGraph()
, request.getGraphName()
);
}
FeedbackRes.Builder builder = FeedbackRes.newBuilder();
try {
if (getWrapper().doGraph(partId,
request.getMethod(),
request.getHeader().getGraph())) {
request.getMethod(),
request.getHeader().getGraph())) {
builder.setStatus(HgGrpc.success());
} else {
builder.setStatus(HgGrpc.not());
}
} catch (Throwable t) {
String msg = "Failed to invoke doGraph[ "
+ request.getMethod().name() + " ], graph="
+ request.getHeader().getGraph();
+ request.getMethod().name() + " ], graph="
+ request.getHeader().getGraph();
log.error(msg, t);
builder.setStatus(HgGrpc.fail(msg));
}

19
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreStateService.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStateService.java
View File

@ -1,4 +1,21 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import org.lognet.springboot.grpc.GRpcService;

45
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreStateSubject.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStateSubject.java
View File

@ -1,16 +1,33 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
import com.baidu.hugegraph.store.grpc.state.NodeStateRes;
import com.baidu.hugegraph.store.grpc.state.NodeStateType;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
package org.apache.hugegraph.store.node.grpc;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import static com.baidu.hugegraph.store.node.util.HgAssert.isArgumentNotNull;
import static com.baidu.hugegraph.store.node.util.HgAssert.isArgumentValid;
import org.apache.hugegraph.store.node.util.HgAssert;
import com.baidu.hugegraph.store.grpc.state.NodeStateRes;
import com.baidu.hugegraph.store.grpc.state.NodeStateType;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2021/11/3
@ -22,22 +39,23 @@ public final class HgStoreStateSubject {
public static void addObserver(String subId, StreamObserver<NodeStateRes> observer) {
isArgumentValid(subId, "subId");
isArgumentNotNull(observer == null, "observer");
HgAssert.isArgumentValid(subId, "subId");
HgAssert.isArgumentNotNull(observer == null, "observer");
subObserverHolder.put(subId, observer);
}
public static void removeObserver(String subId) {
isArgumentValid(subId, "subId");
HgAssert.isArgumentValid(subId, "subId");
subObserverHolder.remove(subId);
}
public static void notifyAll(NodeStateType nodeState) {
isArgumentNotNull(nodeState == null, "nodeState");
HgAssert.isArgumentNotNull(nodeState == null, "nodeState");
NodeStateRes res = NodeStateRes.newBuilder().setState(nodeState).build();
Iterator<Map.Entry<String, StreamObserver<NodeStateRes>>> iter = subObserverHolder.entrySet().iterator();
Iterator<Map.Entry<String, StreamObserver<NodeStateRes>>> iter =
subObserverHolder.entrySet().iterator();
while (iter.hasNext()) {
Map.Entry<String, StreamObserver<NodeStateRes>> entry = iter.next();
@ -45,7 +63,8 @@ public final class HgStoreStateSubject {
try {
entry.getValue().onNext(res);
} catch (Throwable e) {
log.error("Failed to send node-state[" + nodeState + "] to subscriber[" + entry.getKey() + "].", e);
log.error("Failed to send node-state[" + nodeState + "] to subscriber[" +
entry.getKey() + "].", e);
iter.remove();
log.error("Removed the subscriber[" + entry.getKey() + "].", e);
}

30
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreStreamImpl.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStreamImpl.java
View File

@ -15,13 +15,15 @@
* under the License.
*/
package com.baidu.hugegraph.store.node.grpc;
package org.apache.hugegraph.store.node.grpc;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc;
import org.apache.hugegraph.store.grpc.stream.KvStream;
import org.apache.hugegraph.store.node.AppConfig;
import org.apache.hugegraph.store.node.util.HgExecutorUtil;
import org.lognet.springboot.grpc.GRpcService;
import org.springframework.beans.factory.annotation.Autowired;
@ -29,8 +31,6 @@ import com.baidu.hugegraph.store.grpc.state.ScanState;
import com.baidu.hugegraph.store.grpc.stream.KvPageRes;
import com.baidu.hugegraph.store.grpc.stream.ScanStreamBatchReq;
import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq;
import com.baidu.hugegraph.store.node.AppConfig;
import com.baidu.hugegraph.store.node.util.HgExecutorUtil;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
@ -53,7 +53,8 @@ public class HgStoreStreamImpl extends HgStoreStreamGrpc.HgStoreStreamImplBase {
if (this.wrapper == null) {
synchronized (this) {
if (this.wrapper == null) {
this.wrapper = new HgStoreWrapperEx(storeService.getStoreEngine().getBusinessHandler());
this.wrapper = new HgStoreWrapperEx(
storeService.getStoreEngine().getBusinessHandler());
}
}
}
@ -69,8 +70,9 @@ public class HgStoreStreamImpl extends HgStoreStreamGrpc.HgStoreStreamImplBase {
synchronized (this) {
if (this.executor == null) {
AppConfig.ThreadPoolScan scan = this.appConfig.getThreadPoolScan();
this.executor = HgExecutorUtil.createExecutor("hg-scan", scan.getCore(), scan.getMax(),
scan.getQueue());
this.executor =
HgExecutorUtil.createExecutor("hg-scan", scan.getCore(), scan.getMax(),
scan.getQueue());
}
}
}
@ -81,13 +83,15 @@ public class HgStoreStreamImpl extends HgStoreStreamGrpc.HgStoreStreamImplBase {
ThreadPoolExecutor ex = getExecutor();
ScanState.Builder builder = ScanState.newBuilder();
BlockingQueue<Runnable> queue = ex.getQueue();
ScanState state = builder.setActiveCount(ex.getActiveCount()).setTaskCount(ex.getTaskCount())
.setCompletedTaskCount(ex.getCompletedTaskCount())
.setMaximumPoolSize(ex.getMaximumPoolSize())
.setLargestPoolSize(ex.getLargestPoolSize()).setPoolSize(ex.getPoolSize())
.setAddress(appConfig.getStoreServerAddress())
.setQueueSize(queue.size()).setQueueRemainingCapacity(queue.remainingCapacity())
.build();
ScanState state =
builder.setActiveCount(ex.getActiveCount()).setTaskCount(ex.getTaskCount())
.setCompletedTaskCount(ex.getCompletedTaskCount())
.setMaximumPoolSize(ex.getMaximumPoolSize())
.setLargestPoolSize(ex.getLargestPoolSize()).setPoolSize(ex.getPoolSize())
.setAddress(appConfig.getStoreServerAddress())
.setQueueSize(queue.size())
.setQueueRemainingCapacity(queue.remainingCapacity())
.build();
return state;
}

45
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/HgStoreWrapperEx.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreWrapperEx.java
View File

@ -1,10 +1,28 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Supplier;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.store.business.BusinessHandler;
import com.baidu.hugegraph.store.business.FilterIterator;
import com.baidu.hugegraph.store.grpc.common.GraphMethod;
@ -45,9 +63,11 @@ public class HgStoreWrapperEx {
return FilterIterator.of(scanIterator, query);
}
public ScanIterator scan(String graph, int partId, String table, byte[] start, byte[] end, int scanType,
public ScanIterator scan(String graph, int partId, String table, byte[] start, byte[] end,
int scanType,
byte[] query) {
ScanIterator scanIterator = this.handler.scan(graph, partId, table, start, end, scanType, query);
ScanIterator scanIterator =
this.handler.scan(graph, partId, table, start, end, scanType, query);
return FilterIterator.of(scanIterator, query);
}
@ -58,9 +78,11 @@ public class HgStoreWrapperEx {
}));
}
public ScanIterator scanPrefix(String graph, int partition, String table, byte[] prefix, int scanType,
public ScanIterator scanPrefix(String graph, int partition, String table, byte[] prefix,
int scanType,
byte[] query) {
ScanIterator scanIterator = this.handler.scanPrefix(graph, partition, table, prefix, scanType);
ScanIterator scanIterator =
this.handler.scanPrefix(graph, partition, table, prefix, scanType);
return FilterIterator.of(scanIterator, query);
}
@ -99,13 +121,10 @@ public class HgStoreWrapperEx {
public boolean doGraph(int partId, GraphMethod method, String graph) {
boolean flag = true;
switch (method) {
case GRAPH_METHOD_DELETE:
// 交给raft执行，此处不处理
flag = true;
break;
default:
throw new UnsupportedOperationException("GraphMethod: " + method.name());
if (method == GRAPH_METHOD_DELETE) {// 交给raft执行，此处不处理
flag = true;
} else {
throw new UnsupportedOperationException("GraphMethod: " + method.name());
}
return flag;
}

69
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ParallelScanIterator.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java
View File

@ -1,4 +1,21 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import java.util.ArrayList;
import java.util.LinkedList;
@ -11,15 +28,16 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Supplier;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.node.util.HgAssert;
import org.apache.hugegraph.store.node.util.PropertyUtil;
import com.alipay.sofa.jraft.util.Utils;
import com.baidu.hugegraph.pd.common.KVPair;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.store.buffer.KVByteBuffer;
import com.baidu.hugegraph.store.grpc.common.ScanOrderType;
import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest;
import com.baidu.hugegraph.store.node.util.HgAssert;
import com.baidu.hugegraph.store.node.util.PropertyUtil;
import com.baidu.hugegraph.store.term.Bits;
import lombok.extern.slf4j.Slf4j;
@ -30,31 +48,33 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class ParallelScanIterator implements ScanIterator {
private int batchSize = PropertyUtil.getInt("app.scan.stream.entries.size", 20000);
protected static int maxBodySize = PropertyUtil.getInt("app.scan.stream.body.size", 1 * 1024 * 1024);
private static int waitDataMaxTryTimes = 600;
private final int batchSize = PropertyUtil.getInt("app.scan.stream.entries.size", 20000);
protected static int maxBodySize =
PropertyUtil.getInt("app.scan.stream.body.size", 1024 * 1024);
private static final int waitDataMaxTryTimes = 600;
private int maxWorkThreads = Utils.cpus() / 8;
private int maxInQueue = maxWorkThreads * 2;
private final Supplier<KVPair<QueryCondition, ScanIterator>> batchSupplier;
private final Supplier<Long> limitSupplier;
private volatile boolean finished;
private BlockingQueue<List<KV>> queue;
private final BlockingQueue<List<KV>> queue;
private final ReentrantLock queueLock = new ReentrantLock();
final private ThreadPoolExecutor executor;
private List<KV> current = null;
private ScanQueryRequest query;
private Queue<KVScanner> scanners = new LinkedList<>();
private Queue<KVScanner> pauseScanners = new LinkedList<>();
private final ScanQueryRequest query;
private final Queue<KVScanner> scanners = new LinkedList<>();
private final Queue<KVScanner> pauseScanners = new LinkedList<>();
final private List<KV> NO_DATA = new ArrayList<>();
private boolean orderVertex;
private boolean orderEdge;
private final boolean orderVertex;
private final boolean orderEdge;
public static ParallelScanIterator of(Supplier<KVPair<QueryCondition, ScanIterator>> iteratorSupplier,
Supplier<Long> limitSupplier,
ScanQueryRequest query,
ThreadPoolExecutor executor) {
public static ParallelScanIterator of(
Supplier<KVPair<QueryCondition, ScanIterator>> iteratorSupplier,
Supplier<Long> limitSupplier,
ScanQueryRequest query,
ThreadPoolExecutor executor) {
HgAssert.isArgumentNotNull(iteratorSupplier, "iteratorSupplier");
HgAssert.isArgumentNotNull(limitSupplier, "limitSupplier");
return new ParallelScanIterator(iteratorSupplier, limitSupplier, query, executor);
@ -74,7 +94,8 @@ public class ParallelScanIterator implements ScanIterator {
if (orderVertex) {
this.maxWorkThreads = 1;
} else {
this.maxWorkThreads = Math.max(1, Math.min(query.getConditionCount() / 16, maxWorkThreads));
this.maxWorkThreads =
Math.max(1, Math.min(query.getConditionCount() / 16, maxWorkThreads));
}
this.maxInQueue = maxWorkThreads * 2;
// 边有序需要更大的队列
@ -102,7 +123,8 @@ public class ParallelScanIterator implements ScanIterator {
tryTimes++;
}
if (current == null && tryTimes >= waitDataMaxTryTimes) {
log.error("Wait data timeout!!!, scanner is {}/{}", scanners.size(), pauseScanners.size());
log.error("Wait data timeout!!!, scanner is {}/{}", scanners.size(),
pauseScanners.size());
}
return current != null && current != NO_DATA;
}
@ -219,7 +241,7 @@ public class ParallelScanIterator implements ScanIterator {
}
}
// 数据未结束，线程继续执行
return hasNext ? true : this.queue.size() < maxInQueue;
return hasNext || this.queue.size() < maxInQueue;
}
private synchronized KVPair<QueryCondition, ScanIterator> getIterator() {
@ -241,7 +263,7 @@ public class ParallelScanIterator implements ScanIterator {
private long limit;
private long counter;
private volatile boolean closed = false;
private ReentrantLock iteratorLock = new ReentrantLock();
private final ReentrantLock iteratorLock = new ReentrantLock();
private ScanIterator getIterator() {
// 迭代器没有数据，或该点以达到limit，切换新的迭代器
@ -270,7 +292,8 @@ public class ParallelScanIterator implements ScanIterator {
if (iterator == null) {
break;
}
while (iterator.hasNext() && entriesSize < batchSize && bodySize < maxBodySize &&
while (iterator.hasNext() && entriesSize < batchSize &&
bodySize < maxBodySize &&
counter < limit && !closed) {
KV kv = KV.of(iterator.next());
dataList.add(orderVertex ? kv.setNo(query.getSerialNo()) : kv);

39
hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/QueryCondition.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
/**
* @author lynn.bond@hotmail.com on 2023/2/8
*/
public interface QueryCondition {
byte[] getStart();
byte[] getEnd();
byte[] getPrefix();
int getKeyCode();
int getScanType();
byte[] getQuery();
byte[] getPosition();
int getSerialNo();
}

40
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java
View File

@ -1,15 +1,33 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
import static com.baidu.hugegraph.store.node.grpc.ScanUtil.getIterator;
package org.apache.hugegraph.store.node.grpc;
import static org.apache.hugegraph.store.node.grpc.ScanUtil.getIterator;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.node.util.HgGrpc;
import org.apache.hugegraph.store.node.util.HgStoreNodeUtil;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.store.grpc.common.Kv;
import com.baidu.hugegraph.store.grpc.stream.KvPageRes;
import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest;
import com.baidu.hugegraph.store.grpc.stream.ScanStreamBatchReq;
import com.baidu.hugegraph.store.node.util.HgGrpc;
import com.baidu.hugegraph.store.node.util.HgStoreNodeUtil;
import com.google.protobuf.ByteString;
import io.grpc.Status;
@ -45,7 +63,8 @@ public class ScanBatchOneShotResponse {
if (limit <= 0) {
limit = Integer.MAX_VALUE;
log.warn("As limit is less than or equals 0, default limit was effective:[ {} ]", Integer.MAX_VALUE);
log.warn("As limit is less than or equals 0, default limit was effective:[ {} ]",
Integer.MAX_VALUE);
}
int count = 0;
@ -60,9 +79,10 @@ public class ScanBatchOneShotResponse {
RocksDBSession.BackendColumn col = iterator.next();
resBuilder.addData(kvBuilder
.setKey(ByteString.copyFrom(col.name))
.setValue(ByteString.copyFrom(col.value))
.setCode(HgStoreNodeUtil.toInt(iterator.position())) //position == partition-id.
.setKey(ByteString.copyFrom(col.name))
.setValue(ByteString.copyFrom(col.value))
.setCode(HgStoreNodeUtil.toInt(iterator.position()))
//position == partition-id.
);
}

48
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchResponse.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java
View File

@ -15,24 +15,22 @@
* under the License.
*/
package com.baidu.hugegraph.store.node.grpc;
package org.apache.hugegraph.store.node.grpc;
import static com.baidu.hugegraph.store.node.grpc.ScanUtil.getParallelIterator;
import java.util.List;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.grpc.stream.KvStream;
import org.apache.hugegraph.store.node.util.HgGrpc;
import org.apache.hugegraph.store.node.util.PropertyUtil;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.store.buffer.ByteBufferAllocator;
import com.baidu.hugegraph.store.buffer.KVByteBuffer;
import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest;
import com.baidu.hugegraph.store.grpc.stream.ScanStreamBatchReq;
import com.baidu.hugegraph.store.node.util.HgGrpc;
import com.baidu.hugegraph.store.node.util.PropertyUtil;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
@ -49,7 +47,7 @@ public class ScanBatchResponse implements StreamObserver<ScanStreamBatchReq> {
private final int activeTimeout = PropertyUtil.getInt("app.scan.stream.timeout", 60); //单位秒
static ByteBufferAllocator bfAllocator =
new ByteBufferAllocator(ParallelScanIterator.maxBodySize*3/2, 1000);
new ByteBufferAllocator(ParallelScanIterator.maxBodySize * 3 / 2, 1000);
// 当前正在遍历的迭代器
private ScanIterator iterator;
// 下一次发送的序号
@ -65,14 +63,15 @@ public class ScanBatchResponse implements StreamObserver<ScanStreamBatchReq> {
private long activeTime;
private volatile State state;
private final Object stateLock = new Object();
private ReentrantLock iteratorLock = new ReentrantLock();
private final ReentrantLock iteratorLock = new ReentrantLock();
private final StreamObserver<KvStream> sender;
private final HgStoreWrapperEx wrapper;
private final ThreadPoolExecutor executor;
private final long logId;
public ScanBatchResponse(StreamObserver<KvStream> response, HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) {
public ScanBatchResponse(StreamObserver<KvStream> response, HgStoreWrapperEx wrapper,
ThreadPoolExecutor executor) {
this.sender = response;
this.wrapper = wrapper;
this.executor = executor;
@ -143,12 +142,13 @@ public class ScanBatchResponse implements StreamObserver<ScanStreamBatchReq> {
// log.info("Stream {} startQuery graphName is {}, query degree/keylimit/limit is " +
// "{}/{}/{}, scanType is {}, orderType is {}",
// this.logId, graphName, query.getSkipDegree(), query.getPerKeyLimit(), query.getLimit(),
// this.logId, graphName, query.getSkipDegree(), query.getPerKeyLimit(), query
// .getLimit(),
// query.getScanType(), query.getOrderType());
this.clientLimit = request.getLimit();
this.entriesCounter = 0;
this.iterator = getParallelIterator(graphName, request, this.wrapper, executor);
this.iterator = ScanUtil.getParallelIterator(graphName, request, this.wrapper, executor);
synchronized (stateLock) {
if (state == State.IDLE) {
state = State.DOING;
@ -186,16 +186,16 @@ public class ScanBatchResponse implements StreamObserver<ScanStreamBatchReq> {
private void sendEntries() {
iteratorLock.lock();
try {
if ( state == State.DONE || iterator == null) {
if (state == State.DONE || iterator == null) {
setStateIdle();
return;
}
KvStream.Builder dataBuilder = KvStream.newBuilder()
.setVersion(1);
.setVersion(1);
while (iterator.hasNext()
&& (nextSeqNo - clientSeqNo < maxInFlightCount)
&& this.entriesCounter < clientLimit
&& state != State.DONE) {
&& (nextSeqNo - clientSeqNo < maxInFlightCount)
&& this.entriesCounter < clientLimit
&& state != State.DONE) {
KVByteBuffer buffer = new KVByteBuffer(bfAllocator.get());
List<ParallelScanIterator.KV> dataList = iterator.next();
dataList.forEach(kv -> {
@ -204,7 +204,7 @@ public class ScanBatchResponse implements StreamObserver<ScanStreamBatchReq> {
});
dataBuilder.setStream(buffer.flip().getBuffer());
dataBuilder.setSeqNo(nextSeqNo++);
dataBuilder.complete(e->{
dataBuilder.complete(e -> {
bfAllocator.release(buffer.getBuffer());
});
this.sender.onNext(dataBuilder.build());
@ -216,12 +216,13 @@ public class ScanBatchResponse implements StreamObserver<ScanStreamBatchReq> {
} else {
setStateIdle();
}
}catch (Throwable e){
} catch (Throwable e) {
log.error("exception ", e);
setStateIdle();
if ( this.sender != null)
if (this.sender != null) {
this.sender.onError(e);
}finally {
}
} finally {
iteratorLock.unlock();
}
}
@ -229,7 +230,7 @@ public class ScanBatchResponse implements StreamObserver<ScanStreamBatchReq> {
private void sendNoDataEntries() {
try {
this.sender.onNext(KvStream.newBuilder().setOver(true).build());
}catch (Exception e){
} catch (Exception e) {
}
}
@ -242,8 +243,9 @@ public class ScanBatchResponse implements StreamObserver<ScanStreamBatchReq> {
private State setStateIdle() {
synchronized (this.stateLock) {
if (this.state != State.DONE)
if (this.state != State.DONE) {
this.state = State.IDLE;
}
}
return state;
}
@ -252,7 +254,7 @@ public class ScanBatchResponse implements StreamObserver<ScanStreamBatchReq> {
* 检查是否活跃，超过一定时间客户端没有请求数据，认为已经不活跃，关闭连接释放资源
*/
public void checkActiveTimeout() {
if ((System.currentTimeMillis() - activeTime) > activeTimeout * 1000) {
if ((System.currentTimeMillis() - activeTime) > activeTimeout * 1000L) {
log.warn("The stream is not closed, and the timeout is forced to close");
closeQuery();
}

105
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchResponse3.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java
View File

@ -1,6 +1,21 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
import static com.baidu.hugegraph.store.node.grpc.ScanUtil.getIterator;
package org.apache.hugegraph.store.node.grpc;
import java.util.List;
import java.util.concurrent.ThreadPoolExecutor;
@ -12,18 +27,19 @@ import java.util.concurrent.locks.ReentrantLock;
import javax.annotation.concurrent.NotThreadSafe;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.node.util.Base58;
import org.apache.hugegraph.store.node.util.HgAssert;
import org.apache.hugegraph.store.node.util.HgGrpc;
import org.apache.hugegraph.store.node.util.HgStoreConst;
import org.apache.hugegraph.store.node.util.HgStoreNodeUtil;
import com.baidu.hugegraph.store.grpc.common.Kv;
import com.baidu.hugegraph.store.grpc.stream.KvPageRes;
import com.baidu.hugegraph.store.grpc.stream.ScanCondition;
import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest;
import com.baidu.hugegraph.store.grpc.stream.ScanStreamBatchReq;
import com.baidu.hugegraph.store.node.util.Base58;
import com.baidu.hugegraph.store.node.util.HgAssert;
import com.baidu.hugegraph.store.node.util.HgGrpc;
import com.baidu.hugegraph.store.node.util.HgStoreConst;
import com.baidu.hugegraph.store.node.util.HgStoreNodeUtil;
import com.google.protobuf.ByteString;
import io.grpc.Status;
@ -39,14 +55,16 @@ public class ScanBatchResponse3 {
private final static long DEFAULT_PACKAGE_SIZE = 10_000;
private final static int MAX_NOT_RECEIPT = 10;
public static StreamObserver of(StreamObserver<KvPageRes> responseObserver, HgStoreWrapperEx wrapper,ThreadPoolExecutor executor) {
public static StreamObserver of(StreamObserver<KvPageRes> responseObserver,
HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) {
HgAssert.isArgumentNotNull(responseObserver, "responseObserver");
HgAssert.isArgumentNotNull(wrapper, "wrapper");
return new Broker(responseObserver, wrapper,executor);
return new Broker(responseObserver, wrapper, executor);
}
private enum OrderState {
NEW(0), WORKING(1);//, PAUSE(2), COMPLETE(10);
NEW(0),
WORKING(1);//, PAUSE(2), COMPLETE(10);
int value;
OrderState(int value) {
@ -61,12 +79,13 @@ public class ScanBatchResponse3 {
private final ThreadPoolExecutor executor;
private String graph;
private OrderManager manager = new OrderManager();
private final OrderManager manager = new OrderManager();
Broker(StreamObserver<KvPageRes> responseObserver, HgStoreWrapperEx wrapper,ThreadPoolExecutor executor) {
Broker(StreamObserver<KvPageRes> responseObserver, HgStoreWrapperEx wrapper,
ThreadPoolExecutor executor) {
this.responseObserver = responseObserver;
this.wrapper = wrapper;
this.executor=executor;
this.executor = executor;
}
@Override
@ -113,7 +132,8 @@ public class ScanBatchResponse3 {
ScanCondition c = conditions.get(0);
if (c.getPrefix() != null && c.getPrefix().size() > 0) {
deliverId = Base58.encode(c.getPrefix().toByteArray());
log.info("[ANALYSIS DEAL] [{}] prefixLength: {}", deliverId, conditions.size());
log.info("[ANALYSIS DEAL] [{}] prefixLength: {}", deliverId,
conditions.size());
}
}
@ -123,7 +143,7 @@ public class ScanBatchResponse3 {
OrderWorker worker = new OrderWorker(
request.getLimit(),
request.getPageSize(),
getIterator(this.graph, request, this.wrapper),
ScanUtil.getIterator(this.graph, request, this.wrapper),
deliverer,
this.executor);
@ -171,9 +191,9 @@ public class ScanBatchResponse3 {
private static class OrderDeliverer {
private final StreamObserver<KvPageRes> responseObserver;
private AtomicBoolean finishFlag = new AtomicBoolean();
private final AtomicBoolean finishFlag = new AtomicBoolean();
private final String delivererId;
private AtomicLong count = new AtomicLong();
private final AtomicLong count = new AtomicLong();
OrderDeliverer(String delivererId, StreamObserver<KvPageRes> responseObserver) {
this.responseObserver = responseObserver;
@ -189,9 +209,10 @@ public class ScanBatchResponse3 {
if (log.isDebugEnabled()) log.debug("deliver times : {}, over: {}", times, isOver);
if (isOver) {
if(log.isDebugEnabled()){
if (log.isDebugEnabled()) {
if (delivererId != null && !delivererId.isEmpty()) {
log.debug("[ANALYSIS OVER] [{}] count: {}, times: {}", delivererId, count, times);
log.debug("[ANALYSIS OVER] [{}] count: {}, times: {}", delivererId, count,
times);
}
}
this.finish();
@ -199,19 +220,19 @@ public class ScanBatchResponse3 {
}
void finish() {
if (finishFlag.getAndSet(true) == false) {
if (!finishFlag.getAndSet(true)) {
this.responseObserver.onCompleted();
}
}
void error(String msg) {
if (finishFlag.getAndSet(true) == false) {
if (!finishFlag.getAndSet(true)) {
this.responseObserver.onError(HgGrpc.toErr(msg));
}
}
void error(String msg, Throwable t) {
if (finishFlag.getAndSet(true) == false) {
if (!finishFlag.getAndSet(true)) {
this.responseObserver.onError(HgGrpc.toErr(Status.INTERNAL,
msg, t));
}
@ -230,20 +251,24 @@ public class ScanBatchResponse3 {
private final AtomicInteger receiptTimes = new AtomicInteger();
private final AtomicInteger curTimes = new AtomicInteger();
private final ThreadPoolExecutor executor;
private long limit;
private final long limit;
private long packageSize;
private long counter;
OrderWorker(long limit, long packageSize, ScanIterator iterator, OrderDeliverer deliverer,ThreadPoolExecutor executor) {
OrderWorker(long limit, long packageSize, ScanIterator iterator, OrderDeliverer deliverer,
ThreadPoolExecutor executor) {
this.limit = limit;
this.packageSize = packageSize;
this.iterator = iterator;
this.deliverer = deliverer;
this.executor=executor;
this.executor = executor;
if (this.packageSize <= 0) {
this.packageSize = DEFAULT_PACKAGE_SIZE;
log.warn("As page-Size is less than or equals 0, default package-size was effective.[ {} ]", DEFAULT_PACKAGE_SIZE);
log.warn(
"As page-Size is less than or equals 0, default package-size was " +
"effective.[ {} ]",
DEFAULT_PACKAGE_SIZE);
}
}
@ -292,7 +317,7 @@ public class ScanBatchResponse3 {
}
private void working() {
if (this.isWorking.getAndSet(true) == true) {
if (this.isWorking.getAndSet(true)) {
return;
}
@ -321,12 +346,17 @@ public class ScanBatchResponse3 {
if (!this.checkContinue()) {
long start = System.currentTimeMillis();
iterator.wait(HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * 1000);
iterator.wait(
HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS *
1000);
if (System.currentTimeMillis() - start
>= HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * 1000) {
>=
HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * 1000) {
throw new TimeoutException("Waiting continue more than "
+ HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS + " seconds.");
+
HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS +
" seconds.");
}
if (this.breakdown.get()) {
@ -342,9 +372,11 @@ public class ScanBatchResponse3 {
RocksDBSession.BackendColumn col = iterator.next();
dataBuilder.addData(kvBuilder
.setKey(ByteString.copyFrom(col.name))
.setValue(ByteString.copyFrom(col.value))
.setCode(HgStoreNodeUtil.toInt(iterator.position())) //position == partition-id.
.setKey(ByteString.copyFrom(col.name))
.setValue(ByteString.copyFrom(col.value))
.setCode(HgStoreNodeUtil.toInt(
iterator.position()))
//position == partition-id.
);
}
@ -361,7 +393,8 @@ public class ScanBatchResponse3 {
} catch (TimeoutException t) {
log.info(t.getMessage());
this.deliverer.error("Sever waiting exceeded ["
+ HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS + "] seconds.");
+ HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS +
"] seconds.");
} catch (Throwable t) {
log.error("Failed to do while for scanning, cause by:", t);
this.deliverer.error("Failed to finish scanning ", t);

7
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanBatchResponseFactory.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponseFactory.java
View File

@ -15,7 +15,7 @@
* under the License.
*/
package com.baidu.hugegraph.store.node.grpc;
package org.apache.hugegraph.store.node.grpc;
import java.util.Set;
import java.util.concurrent.ThreadPoolExecutor;
@ -33,7 +33,7 @@ public class ScanBatchResponseFactory {
return instance;
}
private Set<StreamObserver> streamObservers = new ConcurrentHashSet<>();
private final Set<StreamObserver> streamObservers = new ConcurrentHashSet<>();
public int addStreamObserver(StreamObserver observer) {
streamObservers.add(observer);
@ -54,7 +54,8 @@ public class ScanBatchResponseFactory {
});
}
public static StreamObserver of(StreamObserver<KvStream> responseObserver, HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) {
public static StreamObserver of(StreamObserver<KvStream> responseObserver,
HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) {
StreamObserver observer = new ScanBatchResponse(responseObserver, wrapper, executor);
getInstance().addStreamObserver(observer);
getInstance().checkStreamActive();

95
hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanOneShotResponse.java Normal file
View File

@ -0,0 +1,95 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.node.util.HgGrpc;
import org.apache.hugegraph.store.node.util.HgStoreNodeUtil;
import com.baidu.hugegraph.store.grpc.common.Kv;
import com.baidu.hugegraph.store.grpc.stream.KvPageRes;
import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq;
import com.google.protobuf.ByteString;
import io.grpc.Status;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2022/02/17
* @version 3.6.0
*/
@Slf4j
public class ScanOneShotResponse {
/**
* Handle one-shot scan
*
* @param request
* @param responseObserver
*/
public static void scanOneShot(ScanStreamReq request,
StreamObserver<KvPageRes> responseObserver,
HgStoreWrapperEx wrapper) {
KvPageRes.Builder resBuilder = KvPageRes.newBuilder();
Kv.Builder kvBuilder = Kv.newBuilder();
ScanIterator iterator = ScanUtil.getIterator(ScanUtil.toSq(request), wrapper);
long limit = request.getLimit();
if (limit <= 0) {
responseObserver.onError(HgGrpc.toErr("limit<=0, please to invoke stream scan."));
return;
}
int count = 0;
try {
while (iterator.hasNext()) {
if (++count > limit) {
break;
}
RocksDBSession.BackendColumn col = iterator.next();
resBuilder.addData(kvBuilder
.setKey(ByteString.copyFrom(col.name))
.setValue(ByteString.copyFrom(col.value))
.setCode(HgStoreNodeUtil.toInt(iterator.position()))
//position == partition-id.
);
}
responseObserver.onNext(resBuilder.build());
responseObserver.onCompleted();
} catch (Throwable t) {
String msg = "an exception occurred during data scanning";
responseObserver.onError(HgGrpc.toErr(Status.INTERNAL, msg, t));
} finally {
iterator.close();
}
}
}

104
hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanQuery.java Normal file
View File

@ -0,0 +1,104 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import java.util.Arrays;
import com.baidu.hugegraph.store.grpc.common.ScanMethod;
/**
* @author lynn.bond@hotmail.com on 2022/2/28
*/
class ScanQuery implements QueryCondition {
String graph;
String table;
ScanMethod method;
byte[] start;
byte[] end;
byte[] prefix;
int keyCode;
int scanType;
byte[] query;
byte[] position;
int serialNo;
@Override
public byte[] getStart() {
return this.start;
}
@Override
public byte[] getEnd() {
return this.end;
}
@Override
public byte[] getPrefix() {
return this.prefix;
}
@Override
public int getKeyCode() {
return this.keyCode;
}
@Override
public int getScanType() {
return this.scanType;
}
@Override
public byte[] getQuery() {
return this.query;
}
@Override
public byte[] getPosition() {
return this.position;
}
@Override
public int getSerialNo() {
return this.serialNo;
}
static ScanQuery of() {
return new ScanQuery();
}
private ScanQuery() {
}
@Override
public String toString() {
return "ScanQuery{" +
"graph='" + graph + '\'' +
", table='" + table + '\'' +
", method=" + method +
", start=" + Arrays.toString(start) +
", end=" + Arrays.toString(end) +
", prefix=" + Arrays.toString(prefix) +
", partition=" + keyCode +
", scanType=" + scanType +
", serialNo=" + serialNo +
", query=" + Arrays.toString(query) +
", position=" + Arrays.toString(position) +
'}';
}
}

48
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanQueryProducer.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanQueryProducer.java
View File

@ -1,17 +1,37 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
import com.baidu.hugegraph.store.node.util.HgAssert;
import com.baidu.hugegraph.store.grpc.common.ScanMethod;
import com.baidu.hugegraph.store.grpc.stream.ScanCondition;
import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest;
import lombok.extern.slf4j.Slf4j;
package org.apache.hugegraph.store.node.grpc;
import javax.annotation.concurrent.NotThreadSafe;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.store.node.util.HgAssert;
import com.baidu.hugegraph.store.grpc.common.ScanMethod;
import com.baidu.hugegraph.store.grpc.stream.ScanCondition;
import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest;
import lombok.extern.slf4j.Slf4j;
/**
* Buffering the data of ScanQueryRequest and generating ScanQuery.
* It will not hold the reference of ScanQueryRequest.
@ -34,7 +54,8 @@ class ScanQueryProducer implements Iterable<ScanQuery> {
private ScanQueryProducer() {
}
public static ScanQueryProducer requestOf(String graph, String[] tables, ScanQueryRequest request) {
public static ScanQueryProducer requestOf(String graph, String[] tables,
ScanQueryRequest request) {
HgAssert.isArgumentValid(graph, "graph");
HgAssert.isArgumentNotNull(tables, "tables");
HgAssert.isArgumentNotNull(request, "ScanQueryRequest");
@ -124,7 +145,7 @@ class ScanQueryProducer implements Iterable<ScanQuery> {
@Override
public ScanQuery[] next() {
if (! this.hasNext()) {
if (!this.hasNext()) {
throw new NoSuchElementException();
}
@ -141,7 +162,8 @@ class ScanQueryProducer implements Iterable<ScanQuery> {
}
private class GroupedConditionsIterator implements Iterator<ScanQuery[]> {
private Iterator<ScanCondition> conditionIterator = ScanQueryProducer.this.conditionList.iterator();
private final Iterator<ScanCondition> conditionIterator =
ScanQueryProducer.this.conditionList.iterator();
@Override
public boolean hasNext() {
@ -154,7 +176,8 @@ class ScanQueryProducer implements Iterable<ScanQuery> {
ScanQuery[] res = new ScanQuery[ScanQueryProducer.this.tables.length];
for (int i = 0; i < res.length; i++) {
res[i] = ScanQueryProducer.this.createQuery(ScanQueryProducer.this.tables[i], condition);
res[i] = ScanQueryProducer.this.createQuery(ScanQueryProducer.this.tables[i],
condition);
}
return res;
@ -198,7 +221,8 @@ class ScanQueryProducer implements Iterable<ScanQuery> {
* TODO: no testing
*/
private class ConditionsIterator implements Iterator<ScanQuery> {
private Iterator<ScanCondition> conditionIterator = ScanQueryProducer.this.conditionList.iterator();
private final Iterator<ScanCondition> conditionIterator =
ScanQueryProducer.this.conditionList.iterator();
private ScanCondition condition;
private String tableName;
private int tableIndex;

66
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanStreamResponse.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanStreamResponse.java
View File

@ -1,21 +1,39 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
import static com.baidu.hugegraph.store.node.grpc.ScanUtil.getIterator;
package org.apache.hugegraph.store.node.grpc;
import static org.apache.hugegraph.store.node.grpc.ScanUtil.getIterator;
import java.util.Collections;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicBoolean;
import com.baidu.hugegraph.rocksdb.access.RocksDBSession;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.node.AppConfig;
import org.apache.hugegraph.store.node.util.HgAssert;
import org.apache.hugegraph.store.node.util.HgChannel;
import org.apache.hugegraph.store.node.util.HgGrpc;
import org.apache.hugegraph.store.node.util.HgStoreNodeUtil;
import com.baidu.hugegraph.store.grpc.common.Kv;
import com.baidu.hugegraph.store.grpc.stream.KvPageRes;
import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq;
import com.baidu.hugegraph.store.node.AppConfig;
import com.baidu.hugegraph.store.node.util.HgAssert;
import com.baidu.hugegraph.store.node.util.HgChannel;
import com.baidu.hugegraph.store.node.util.HgGrpc;
import com.baidu.hugegraph.store.node.util.HgStoreNodeUtil;
import com.google.protobuf.ByteString;
import io.grpc.Status;
@ -40,12 +58,13 @@ public class ScanStreamResponse implements StreamObserver<ScanStreamReq> {
private int total = 0;
private String graph;
private String table;
private AtomicBoolean isStarted = new AtomicBoolean();
private AtomicBoolean isStop = new AtomicBoolean(false);
private AppConfig config;
private int waitTime;
private HgChannel<KvPageRes.Builder> channel;
private static String msg = "to wait for client taking data exceeded max time: [{}] seconds,stop scanning.";
private final AtomicBoolean isStarted = new AtomicBoolean();
private final AtomicBoolean isStop = new AtomicBoolean(false);
private final AppConfig config;
private final int waitTime;
private final HgChannel<KvPageRes.Builder> channel;
private static final String msg =
"to wait for client taking data exceeded max time: [{}] seconds,stop scanning.";
public static ScanStreamResponse of(StreamObserver<KvPageRes> responseObserver,
HgStoreWrapperEx wrapper,
@ -84,7 +103,8 @@ public class ScanStreamResponse implements StreamObserver<ScanStreamReq> {
public void onError(Throwable t) {
this.isStop.set(true);
this.finishServer();
log.warn("onError from client [ graph: {} , table: {}]; Reason: {}]", graph, table, t.getMessage());
log.warn("onError from client [ graph: {} , table: {}]; Reason: {}]", graph, table,
t.getMessage());
}
@Override
@ -104,7 +124,9 @@ public class ScanStreamResponse implements StreamObserver<ScanStreamReq> {
this.limit = request.getLimit();
this.pageSize = request.getPageSize();
if (this.pageSize <= 0) {
log.warn("As page-Size is less than or equals 0, no data will be send to the client.");
log.warn(
"As page-Size is less than or equals 0, no data will be send to the " +
"client.");
}
/*** Start scanning loop ***/
Runnable scanning = () ->
@ -123,7 +145,7 @@ public class ScanStreamResponse implements StreamObserver<ScanStreamReq> {
if (++pageCount > pageSize) {
long start = System.currentTimeMillis();
if (!this.channel.send(dataBuilder)) {
if (System.currentTimeMillis() - start >= waitTime * 1000) {
if (System.currentTimeMillis() - start >= waitTime * 1000L) {
log.warn(msg, waitTime);
this.timeoutSever();
}
@ -140,7 +162,8 @@ public class ScanStreamResponse implements StreamObserver<ScanStreamReq> {
this.channel.send(dataBuilder);
} catch (Throwable t) {
String msg = "an exception occurred while scanning data:";
StatusRuntimeException ex = HgGrpc.toErr(Status.INTERNAL, msg + t.getMessage(), t);
StatusRuntimeException ex =
HgGrpc.toErr(Status.INTERNAL, msg + t.getMessage(), t);
responseObserver.onError(ex);
} finally {
try {
@ -162,8 +185,6 @@ public class ScanStreamResponse implements StreamObserver<ScanStreamReq> {
} catch (Exception exception) {
}
} finally {
}
/*** Scanning loop end ***/
@ -207,7 +228,8 @@ public class ScanStreamResponse implements StreamObserver<ScanStreamReq> {
return;
}
boolean isOver = false;
if (resBuilder == null || resBuilder.getDataList() == null || resBuilder.getDataList().isEmpty()) {
if (resBuilder == null || resBuilder.getDataList() == null ||
resBuilder.getDataList().isEmpty()) {
isOver = true;
resBuilder = KvPageRes.newBuilder().addAllData(Collections.EMPTY_LIST);
}

144
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/ScanUtil.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanUtil.java
View File

@ -1,4 +1,21 @@
package com.baidu.hugegraph.store.node.grpc;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import java.util.Arrays;
import java.util.Collections;
@ -14,14 +31,14 @@ import java.util.stream.Collectors;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.pd.common.KVPair;
import com.baidu.hugegraph.rocksdb.access.ScanIterator;
import com.baidu.hugegraph.store.business.SelectIterator;
import com.baidu.hugegraph.store.grpc.common.ScanMethod;
import com.baidu.hugegraph.store.grpc.stream.ScanQueryRequest;
import com.baidu.hugegraph.store.grpc.stream.ScanStreamReq;
import com.baidu.hugegraph.store.grpc.stream.SelectParam;
import com.baidu.hugegraph.store.node.util.HgStoreNodeUtil;
import lombok.extern.slf4j.Slf4j;
@ -82,10 +99,12 @@ class ScanUtil {
iter = wrapper.scanAll(sq.graph, sq.table, sq.query);
break;
case PREFIX:
iter = wrapper.scanPrefix(sq.graph, sq.keyCode, sq.table, sq.prefix, sq.scanType, sq.query);
iter = wrapper.scanPrefix(sq.graph, sq.keyCode, sq.table, sq.prefix, sq.scanType,
sq.query);
break;
case RANGE:
iter = wrapper.scan(sq.graph, sq.keyCode, sq.table, sq.start, sq.end, sq.scanType, sq.query);
iter = wrapper.scan(sq.graph, sq.keyCode, sq.table, sq.start, sq.end, sq.scanType,
sq.query);
break;
}
@ -121,7 +140,8 @@ class ScanUtil {
return res;
}
static ScanIterator getIterator(String graph, ScanQueryRequest request, HgStoreWrapperEx wrapper) {
static ScanIterator getIterator(String graph, ScanQueryRequest request,
HgStoreWrapperEx wrapper) {
ScanIteratorSupplier supplier = new ScanIteratorSupplier(graph, request, wrapper);
return BatchScanIterator.of(supplier, supplier.getLimitSupplier());
}
@ -133,19 +153,20 @@ class ScanUtil {
HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) {
ScanIteratorSupplier supplier = new ScanIteratorSupplier(graph, request, wrapper);
return ParallelScanIterator.of(supplier, supplier.getLimitSupplier(),
request, executor);
request, executor);
}
@NotThreadSafe
private static class ScanIteratorSupplier implements Supplier<KVPair<QueryCondition, ScanIterator>> {
private static class ScanIteratorSupplier implements
Supplier<KVPair<QueryCondition, ScanIterator>> {
private AtomicBoolean isEmpty = new AtomicBoolean();
private final AtomicBoolean isEmpty = new AtomicBoolean();
private String graph;
private final String graph;
private long perKeyLimit;
private long perKeyMax;
private long skipDegree;
private HgStoreWrapperEx wrapper;
private final long perKeyMax;
private final long skipDegree;
private final HgStoreWrapperEx wrapper;
private List<ScanQuery> sqs = new LinkedList<>();
private Iterator<ScanQuery> sqIterator;
@ -156,14 +177,15 @@ class ScanUtil {
ScanIteratorSupplier(String graph, ScanQueryRequest request, HgStoreWrapperEx wrapper) {
this.graph = graph;
this.perKeyLimit = request.getPerKeyLimit();
this.perKeyMax=request.getPerKeyMax();
this.perKeyMax = request.getPerKeyMax();
this.skipDegree =
request.getSkipDegree() == 0 ? Integer.MAX_VALUE : request.getSkipDegree();
this.wrapper = wrapper;
if (this.perKeyLimit <= 0) {
this.perKeyLimit = Integer.MAX_VALUE;
log.warn("as perKeyLimit <=0 so default perKeyLimit was effective: {}", Integer.MAX_VALUE);
log.warn("as perKeyLimit <=0 so default perKeyLimit was effective: {}",
Integer.MAX_VALUE);
}
//init(request);
init2(request);
@ -171,47 +193,61 @@ class ScanUtil {
private void init(ScanQueryRequest request) {
this.sqs = Arrays.stream(request.getTable().split(","))
.map(table -> {
if (table == null) return null;
if (table.isEmpty()) return null;
.map(table -> {
if (table == null) return null;
if (table.isEmpty()) return null;
List<ScanQuery> list = request.getConditionList()
.stream()
.map(condition -> {
ScanQuery sq = ScanQuery.of();
sq.graph = this.graph;
sq.table = table;
sq.method = request.getMethod();
sq.scanType = request.getScanType();
sq.query = request.getQuery().toByteArray();
sq.position = request.getPosition().toByteArray();
List<ScanQuery> list = request.getConditionList()
.stream()
.map(condition -> {
ScanQuery sq =
ScanQuery.of();
sq.graph = this.graph;
sq.table = table;
sq.method =
request.getMethod();
sq.scanType =
request.getScanType();
sq.query =
request.getQuery()
.toByteArray();
sq.position =
request.getPosition()
.toByteArray();
sq.keyCode = condition.getCode();
sq.start = condition.getStart().toByteArray();
sq.end = condition.getEnd().toByteArray();
sq.prefix = condition.getPrefix().toByteArray();
sq.serialNo = condition.getSerialNo();
return sq;
})
.filter(e -> e != null)
.collect(Collectors.toList());
sq.keyCode =
condition.getCode();
sq.start =
condition.getStart()
.toByteArray();
sq.end = condition.getEnd()
.toByteArray();
sq.prefix =
condition.getPrefix()
.toByteArray();
sq.serialNo =
condition.getSerialNo();
return sq;
})
.filter(e -> e != null)
.collect(Collectors.toList());
if (list == null || list.isEmpty()) {
ScanQuery sq = ScanQuery.of();
sq.graph = this.graph;
sq.table = table;
sq.method = request.getMethod();
sq.scanType = request.getScanType();
sq.query = request.getQuery().toByteArray();
sq.position = request.getPosition().toByteArray();
list = Collections.singletonList(sq);
}
return list;
if (list == null || list.isEmpty()) {
ScanQuery sq = ScanQuery.of();
sq.graph = this.graph;
sq.table = table;
sq.method = request.getMethod();
sq.scanType = request.getScanType();
sq.query = request.getQuery().toByteArray();
sq.position = request.getPosition().toByteArray();
list = Collections.singletonList(sq);
}
return list;
}
)
.flatMap(e -> e.stream())
.collect(Collectors.toList());
}
)
.flatMap(e -> e.stream())
.collect(Collectors.toList());
this.sqIterator = this.sqs.iterator();
}
@ -239,8 +275,8 @@ class ScanUtil {
private void init2(ScanQueryRequest request) {
List<String> tableList = Arrays.stream(request.getTable().split(","))
.filter(e -> e != null && !e.isEmpty())
.collect(Collectors.toList());
.filter(e -> e != null && !e.isEmpty())
.collect(Collectors.toList());
if (tableList.isEmpty()) {
throw new RuntimeException("table name is invalid");

25
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/scan/GraphStoreImpl.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/GraphStoreImpl.java
View File

@ -1,7 +1,26 @@
package com.baidu.hugegraph.store.node.grpc.scan;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc.scan;
import java.util.concurrent.ThreadPoolExecutor;
import org.apache.hugegraph.store.node.grpc.HgStoreNodeService;
import org.apache.hugegraph.store.node.grpc.HgStoreStreamImpl;
import org.lognet.springboot.grpc.GRpcService;
import org.springframework.beans.factory.annotation.Autowired;
@ -11,8 +30,6 @@ import com.baidu.hugegraph.store.grpc.Graphpb;
import com.baidu.hugegraph.store.grpc.Graphpb.ResponseHeader;
import com.baidu.hugegraph.store.grpc.Graphpb.ScanPartitionRequest;
import com.baidu.hugegraph.store.grpc.Graphpb.ScanResponse;
import com.baidu.hugegraph.store.node.grpc.HgStoreNodeService;
import com.baidu.hugegraph.store.node.grpc.HgStoreStreamImpl;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
@ -30,7 +47,7 @@ public class GraphStoreImpl extends GraphStoreImplBase {
private HgStoreStreamImpl storeStream;
BusinessHandler handler;
private ResponseHeader okHeader =
private final ResponseHeader okHeader =
ResponseHeader.newBuilder().setError(
Graphpb.Error.newBuilder().setType(Graphpb.ErrorType.OK))
.build();

44
hg-store-node/src/main/java/com/baidu/hugegraph/store/node/grpc/scan/ScanResponseObserver.java → hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/scan/ScanResponseObserver.java
View File

@ -1,4 +1,21 @@
package com.baidu.hugegraph.store.node.grpc.scan;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc.scan;
import java.util.ArrayList;
import java.util.concurrent.Future;
@ -9,9 +26,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import com.baidu.hugegraph.store.business.GraphStoreIterator;
import com.baidu.hugegraph.store.business.BusinessHandler;
import com.baidu.hugegraph.store.grpc.Graphpb.Error;
import com.baidu.hugegraph.store.business.GraphStoreIterator;
import com.baidu.hugegraph.store.grpc.Graphpb.ErrorType;
import com.baidu.hugegraph.store.grpc.Graphpb.ResponseHeader;
import com.baidu.hugegraph.store.grpc.Graphpb.ScanPartitionRequest;
@ -35,26 +51,26 @@ public class ScanResponseObserver<T> implements
private final BusinessHandler handler;
private static final int BATCH_SIZE = 100000;
private static final int MAX_PAGE = 8; //
private AtomicInteger nextSeqNo = new AtomicInteger(0);
private AtomicInteger cltSeqNo = new AtomicInteger(0);
private final AtomicInteger nextSeqNo = new AtomicInteger(0);
private final AtomicInteger cltSeqNo = new AtomicInteger(0);
private final ThreadPoolExecutor executor;
private ScanPartitionRequest scanReq;
private GraphStoreIterator iter;
private static Error ok = Error.newBuilder().setType(ErrorType.OK).build();
private static ResponseHeader okHeader =
private static final Error ok = Error.newBuilder().setType(ErrorType.OK).build();
private static final ResponseHeader okHeader =
ResponseHeader.newBuilder().setError(ok).build();
private volatile long leftCount;
private volatile AtomicBoolean readOver = new AtomicBoolean(false);
private final AtomicBoolean readOver = new AtomicBoolean(false);
private volatile Future<?> sendTask;
private volatile Future<?> readTask;
private final LinkedBlockingQueue<ScanResponse> packages =
new LinkedBlockingQueue(MAX_PAGE * 2);
private Descriptors.FieldDescriptor vertexField =
private final Descriptors.FieldDescriptor vertexField =
ScanResponse.getDescriptor().findFieldByNumber(3);
private Descriptors.FieldDescriptor edgeField =
private final Descriptors.FieldDescriptor edgeField =
ScanResponse.getDescriptor().findFieldByNumber(4);
private ReentrantLock readLock =new ReentrantLock();
private ReentrantLock sendLock =new ReentrantLock();
private final ReentrantLock readLock = new ReentrantLock();
private final ReentrantLock sendLock = new ReentrantLock();
/*
* 2022年11月1日
@ -116,7 +132,7 @@ public class ScanResponseObserver<T> implements
}
}
} catch (Exception e) {
log.warn("read data with error: ",e);
log.warn("read data with error: ", e);
sender.onError(e);
}
}
@ -246,8 +262,6 @@ public class ScanResponseObserver<T> implements
iter.close();
} catch (Exception e) {
log.warn("on Complete with error:", e);
} finally {
}
}
}

57
hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/ContextClosedListener.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.listener;
import java.util.concurrent.ThreadPoolExecutor;
import org.apache.hugegraph.store.node.grpc.HgStoreStreamImpl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationListener;
import org.springframework.context.event.ContextClosedEvent;
import lombok.extern.slf4j.Slf4j;
/**
* @author zhangyingjie
* @date 2023/2/17
**/
@Slf4j
public class ContextClosedListener implements ApplicationListener<ContextClosedEvent> {
@Autowired
HgStoreStreamImpl storeStream;
@Override
public void onApplicationEvent(ContextClosedEvent event) {
try {
log.info("closing scan threads....");
ThreadPoolExecutor executor = storeStream.getRealExecutor();
if (executor != null) {
try {
executor.shutdownNow();
} catch (Exception e) {
}
}
} catch (Exception ignored) {
} finally {
log.info("closed scan threads");
}
}
}

Some files were not shown because too many files have changed in this diff Show More