anph
/
hugegraph-sync
forked from hugegraph/hugegraph-sync
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refact: fix all CRLF files to LF together (Breaking Change) 

GraphPlatform-1906 梳理入库与查询流程

Change-Id: I5e7f57893c904e50be320e263a01283b147545a8
This commit is contained in:
imbajin 2023-04-26 11:12:02 +08:00
parent 740563bc61
commit 6bce8990af
46 changed files with 8011 additions and 8011 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/AppConfig.java
View File

@ -1,58 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import lombok.Data;
/**
* @author lynn.bond@hotmail.com on 2022/2/15
*/
@Data
@Component
public class AppConfig {
@Value("${pd.address}")
private String pdAddress;
@Value("${net.kv.scanner.page.size}")
private int scannerPageSize;
@Value("${scanner.graph}")
private String scannerGraph;
@Value("${scanner.table}")
private String scannerTable;
@Value("${scanner.max}")
private int scannerMax;
@Value("${scanner.mod}")
private int scannerModNumber;
@Value("${committer.graph}")
private String committerGraph;
@Value("${committer.table}")
private String committerTable;
@Value("${committer.amount}")
private int committerAmount;
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import lombok.Data;
/**
* @author lynn.bond@hotmail.com on 2022/2/15
*/
@Data
@Component
public class AppConfig {
@Value("${pd.address}")
private String pdAddress;
@Value("${net.kv.scanner.page.size}")
private int scannerPageSize;
@Value("${scanner.graph}")
private String scannerGraph;
@Value("${scanner.table}")
private String scannerTable;
@Value("${scanner.max}")
private int scannerMax;
@Value("${scanner.mod}")
private int scannerModNumber;
@Value("${committer.graph}")
private String committerGraph;
@Value("${committer.table}")
private String committerTable;
@Value("${committer.amount}")
private int committerAmount;
}

236
hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/StoreConsoleApplication.java
View File

@ -1,118 +1,118 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli;
import java.io.IOException;
import org.apache.hugegraph.pd.client.PDConfig;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.store.HgStoreClient;
import org.apache.hugegraph.store.cli.loader.HgThread2DB;
import org.apache.hugegraph.store.cli.scan.GrpcShardScanner;
import org.apache.hugegraph.store.cli.scan.HgStoreCommitter;
import org.apache.hugegraph.store.cli.scan.HgStoreScanner;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com on 2022/2/14
*/
@SpringBootApplication
@Slf4j
public class StoreConsoleApplication implements CommandLineRunner {
@Autowired
private AppConfig appConfig;
public static void main(String[] args) {
log.info("Starting StoreConsoleApplication");
SpringApplication.run(StoreConsoleApplication.class, args);
log.info("StoreConsoleApplication finished.");
}
@Override
public void run(String... args) throws IOException, InterruptedException, PDException {
if (args.length <= 0) {
// HgThread2DB hB = new HgThread2DB("localhost:8686");
// hB.startMultiprocessQuery("12", "10");
System.out.println("参数类型 cmd[-load, -query, -scan]");
} else {
switch (args[0]) {
case "-load":
HgThread2DB hgThread2DB = new HgThread2DB(args[1]);
if (!args[3].isEmpty()) {
hgThread2DB.setGraphName(args[3]);
}
try {
if (args[2].equals("order")) {
hgThread2DB.testOrder(args[4]);
} else {
hgThread2DB.startMultiprocessInsert(args[2]);
}
} catch (IOException e) {
e.printStackTrace();
}
break;
case "-query":
HgThread2DB hgDB = new HgThread2DB(args[1]);
try {
hgDB.startMultiprocessQuery("12", args[2]);
} catch (IOException e) {
e.printStackTrace();
}
break;
case "-scan":
if (args.length < 4) {
System.out.println("参数类型 -scan pd graphName tableName");
} else {
doScan(args[1], args[2], args[3]);
}
break;
case "-shard":
GrpcShardScanner scanner = new GrpcShardScanner();
scanner.getData();
break;
case "-shard-single":
scanner = new GrpcShardScanner();
scanner.getDataSingle();
break;
default:
System.out.println("参数类型错误，未执行任何程序");
}
}
}
private void doCommit() {
HgStoreCommitter committer = HgStoreCommitter.of(appConfig.getCommitterGraph());
committer.put(appConfig.getScannerTable(), appConfig.getCommitterAmount());
}
private void doScan(String pd, String graphName, String tableName) throws PDException {
HgStoreClient storeClient = HgStoreClient.create(PDConfig.of(pd)
.setEnableCache(true));
HgStoreScanner storeScanner = HgStoreScanner.of(storeClient, graphName);
storeScanner.scanTable2(tableName);
// storeScanner.scanHash();
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli;
import java.io.IOException;
import org.apache.hugegraph.pd.client.PDConfig;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.store.HgStoreClient;
import org.apache.hugegraph.store.cli.loader.HgThread2DB;
import org.apache.hugegraph.store.cli.scan.GrpcShardScanner;
import org.apache.hugegraph.store.cli.scan.HgStoreCommitter;
import org.apache.hugegraph.store.cli.scan.HgStoreScanner;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com on 2022/2/14
*/
@SpringBootApplication
@Slf4j
public class StoreConsoleApplication implements CommandLineRunner {
@Autowired
private AppConfig appConfig;
public static void main(String[] args) {
log.info("Starting StoreConsoleApplication");
SpringApplication.run(StoreConsoleApplication.class, args);
log.info("StoreConsoleApplication finished.");
}
@Override
public void run(String... args) throws IOException, InterruptedException, PDException {
if (args.length <= 0) {
// HgThread2DB hB = new HgThread2DB("localhost:8686");
// hB.startMultiprocessQuery("12", "10");
System.out.println("参数类型 cmd[-load, -query, -scan]");
} else {
switch (args[0]) {
case "-load":
HgThread2DB hgThread2DB = new HgThread2DB(args[1]);
if (!args[3].isEmpty()) {
hgThread2DB.setGraphName(args[3]);
}
try {
if (args[2].equals("order")) {
hgThread2DB.testOrder(args[4]);
} else {
hgThread2DB.startMultiprocessInsert(args[2]);
}
} catch (IOException e) {
e.printStackTrace();
}
break;
case "-query":
HgThread2DB hgDB = new HgThread2DB(args[1]);
try {
hgDB.startMultiprocessQuery("12", args[2]);
} catch (IOException e) {
e.printStackTrace();
}
break;
case "-scan":
if (args.length < 4) {
System.out.println("参数类型 -scan pd graphName tableName");
} else {
doScan(args[1], args[2], args[3]);
}
break;
case "-shard":
GrpcShardScanner scanner = new GrpcShardScanner();
scanner.getData();
break;
case "-shard-single":
scanner = new GrpcShardScanner();
scanner.getDataSingle();
break;
default:
System.out.println("参数类型错误，未执行任何程序");
}
}
}
private void doCommit() {
HgStoreCommitter committer = HgStoreCommitter.of(appConfig.getCommitterGraph());
committer.put(appConfig.getScannerTable(), appConfig.getCommitterAmount());
}
private void doScan(String pd, String graphName, String tableName) throws PDException {
HgStoreClient storeClient = HgStoreClient.create(PDConfig.of(pd)
.setEnableCache(true));
HgStoreScanner storeScanner = HgStoreScanner.of(storeClient, graphName);
storeScanner.scanTable2(tableName);
// storeScanner.scanHash();
}
}

174
hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreCommitter.java
View File

@ -1,87 +1,87 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli.scan;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgSessionManager;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.cli.util.HgCliUtil;
import org.apache.hugegraph.store.client.HgStoreNodeManager;
/**
* @author lynn.bond@hotmail.com on 2022/2/28
*/
public class HgStoreCommitter {
protected final static HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance();
private final String graph;
public static HgStoreCommitter of(String graph) {
return new HgStoreCommitter(graph);
}
private HgStoreCommitter(String graph) {
this.graph = graph;
}
protected HgStoreSession getStoreSession() {
return HgSessionManager.getInstance().openSession(this.graph);
}
protected HgStoreSession getStoreSession(String graphName) {
return HgSessionManager.getInstance().openSession(graphName);
}
public void put(String tableName, int amount) {
//*************** Put Benchmark **************//*
String keyPrefix = "PUT-BENCHMARK";
HgStoreSession session = getStoreSession();
int length = String.valueOf(amount).length();
session.beginTx();
long start = System.currentTimeMillis();
for (int i = 0; i < amount; i++) {
HgOwnerKey key = HgCliUtil.toOwnerKey(
keyPrefix + "-" + HgCliUtil.padLeftZeros(String.valueOf(i), length));
byte[] value = HgCliUtil.toBytes(keyPrefix + "-V-" + i);
session.put(tableName, key, value);
if ((i + 1) % 100_000 == 0) {
HgCliUtil.println("---------- " + (i + 1) + " --------");
HgCliUtil.println(
"Preparing took: " + (System.currentTimeMillis() - start) + " ms.");
session.commit();
HgCliUtil.println(
"Committing took: " + (System.currentTimeMillis() - start) + " ms.");
start = System.currentTimeMillis();
session.beginTx();
}
}
if (session.isTx()) {
session.commit();
}
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli.scan;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgSessionManager;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.cli.util.HgCliUtil;
import org.apache.hugegraph.store.client.HgStoreNodeManager;
/**
* @author lynn.bond@hotmail.com on 2022/2/28
*/
public class HgStoreCommitter {
protected final static HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance();
private final String graph;
public static HgStoreCommitter of(String graph) {
return new HgStoreCommitter(graph);
}
private HgStoreCommitter(String graph) {
this.graph = graph;
}
protected HgStoreSession getStoreSession() {
return HgSessionManager.getInstance().openSession(this.graph);
}
protected HgStoreSession getStoreSession(String graphName) {
return HgSessionManager.getInstance().openSession(graphName);
}
public void put(String tableName, int amount) {
//*************** Put Benchmark **************//*
String keyPrefix = "PUT-BENCHMARK";
HgStoreSession session = getStoreSession();
int length = String.valueOf(amount).length();
session.beginTx();
long start = System.currentTimeMillis();
for (int i = 0; i < amount; i++) {
HgOwnerKey key = HgCliUtil.toOwnerKey(
keyPrefix + "-" + HgCliUtil.padLeftZeros(String.valueOf(i), length));
byte[] value = HgCliUtil.toBytes(keyPrefix + "-V-" + i);
session.put(tableName, key, value);
if ((i + 1) % 100_000 == 0) {
HgCliUtil.println("---------- " + (i + 1) + " --------");
HgCliUtil.println(
"Preparing took: " + (System.currentTimeMillis() - start) + " ms.");
session.commit();
HgCliUtil.println(
"Committing took: " + (System.currentTimeMillis() - start) + " ms.");
start = System.currentTimeMillis();
session.beginTx();
}
}
if (session.isTx()) {
session.commit();
}
}
}

476
hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreScanner.java
View File

@ -1,238 +1,238 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli.scan;
import java.util.Arrays;
import java.util.List;
import org.apache.hugegraph.pd.client.PDClient;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgKvStore;
import org.apache.hugegraph.store.HgScanQuery;
import org.apache.hugegraph.store.HgSessionManager;
import org.apache.hugegraph.store.HgStoreClient;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.cli.util.HgCliUtil;
import org.apache.hugegraph.store.cli.util.HgMetricX;
import org.apache.hugegraph.store.client.grpc.KvCloseableIterator;
import org.apache.hugegraph.store.client.util.HgStoreClientConfig;
import org.apache.hugegraph.store.client.util.MetricX;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com on 2022/2/14
*/
@Slf4j
public class HgStoreScanner {
private final HgStoreClient storeClient;
private final String graphName;
private long modNumber = 1_000_000;
private int max = 10_000_000;
public long getModNumber() {
return modNumber;
}
public void setModNumber(int modNumber) {
if (modNumber <= 0) {
return;
}
this.modNumber = modNumber;
}
public int getMax() {
return max;
}
public void setMax(int max) {
if (modNumber <= 0) {
return;
}
this.max = max;
}
public static HgStoreScanner of(HgStoreClient storeClient, String graph) {
return new HgStoreScanner(storeClient, graph);
}
private HgStoreScanner(HgStoreClient storeClient, String graph) {
this.storeClient = storeClient;
this.graphName = graph;
}
protected HgStoreSession getStoreSession() {
return HgSessionManager.getInstance().openSession(this.graphName);
}
protected HgStoreSession getStoreSession(String graphName) {
return HgSessionManager.getInstance().openSession(graphName);
}
public void scanTable(String tableName) {
log.info("Starting scan table [{}] of graph [{}] ...", tableName, graphName);
HgMetricX hgMetricX = HgMetricX.ofStart();
HgStoreSession session = getStoreSession();
int count = 0;
KvCloseableIterator<HgKvIterator<HgKvEntry>> iterator =
session.scanBatch2(HgScanQuery.tableOf(tableName));
long start = System.currentTimeMillis();
while (iterator.hasNext()) {
HgKvIterator<HgKvEntry> iterator2 = iterator.next();
while (iterator2.hasNext()) {
count++;
iterator2.next();
if (count % (modNumber) == 0) {
log.info("Scanning keys: " + count + " time is " + modNumber * 1000
/
(System.currentTimeMillis() -
start));
start = System.currentTimeMillis();
}
if (count == max) {
break;
}
}
}
iterator.close();
hgMetricX.end();
log.info("*************************************************");
log.info("************* Scanning Completed **************");
log.info("Graph: {}", graphName);
log.info("Table: {}", tableName);
log.info("Keys: {}", count);
log.info("Max: {}", max);
log.info("Waiting: {} seconds.", MetricX.getIteratorWait() / 1000);
log.info("Total: {} seconds.", hgMetricX.past() / 1000);
log.info("Iterator: [{}]", iterator.getClass().getSimpleName());
log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize());
log.info("*************************************************");
}
public void scanHash() {
String tableName = "g+i";
HgMetricX hgMetricX = HgMetricX.ofStart();
String graphName = "/DEFAULT/graphs/hugegraph1/";
HgStoreSession session = getStoreSession(graphName);
int count = 0;
String query =
"{\"conditions\":[{\"cls\":\"S\",\"el\":{\"key\":\"ID\",\"relation\":\"SCAN\"," +
"\"value\"" +
":{\"start\":\"61180\",\"end\":\"63365\",\"length\":0}}}]," +
"\"optimizedType\":\"NONE\",\"ids\":[]," +
"\"mustSortByInput\":true,\"resultType\":\"EDGE\",\"offset\":0," +
"\"actualOffset\":0,\"actualStoreOffset\":" +
"0,\"limit\":9223372036854775807,\"capacity\":-1,\"showHidden\":false," +
"\"showDeleting\":false," +
"\"showExpired\":false,\"olap\":false,\"withProperties\":false,\"olapPks\":[]}";
//HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,0,715827883,
// HgKvStore.SCAN_ANY,null);
//HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,61180,63365, 348, null);
//HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,0,65535, 348, null);
HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName);
while (iterator.hasNext()) {
count++;
//iterator.next();
// if (count % (modNumber) == 0) {
// log.info("Scanning keys: " + count);
HgCliUtil.println(Arrays.toString(iterator.next().key()));
// }
if (count == max) {
break;
}
}
hgMetricX.end();
log.info("*************************************************");
log.info("************* Scanning Completed **************");
log.info("Graph: {}", this.graphName);
log.info("Table: {}", tableName);
log.info("Keys: {}", count);
log.info("Max: {}", max);
log.info("Waiting: {} seconds.", MetricX.getIteratorWait() / 1000);
log.info("Total: {} seconds.", hgMetricX.past() / 1000);
log.info("Iterator: [{}]", iterator.getClass().getSimpleName());
log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize());
log.info("*************************************************");
}
public static final byte[] EMPTY_BYTES = new byte[0];
public void scanTable2(String tableName) throws PDException {
// java -jar hg-store-cli-3.6.0-SNAPSHOT.jar -scan 10.45.30.212:8989 "DEFAULT/case_112/g"
// g+ie
PDClient pdClient = storeClient.getPdClient();
List<Metapb.Partition> partitions = pdClient.getPartitions(0, graphName);
HgStoreSession session = storeClient.openSession(graphName);
int count = 0;
byte[] position = null;
HgMetricX hgMetricX = HgMetricX.ofStart();
for (Metapb.Partition partition : partitions) {
while (true) {
try (HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,
(int) (partition.getStartKey()),
(int) (partition.getEndKey()),
HgKvStore.SCAN_HASHCODE,
EMPTY_BYTES)) {
if (position != null) {
iterator.seek(position);
}
while (iterator.hasNext()) {
iterator.next();
count++;
if (count % 3000 == 0) {
if (iterator.hasNext()) {
iterator.next();
position = iterator.position();
System.out.println("count is " + count);
} else {
position = null;
}
break;
}
}
if (!iterator.hasNext()) {
position = null;
break;
}
}
}
}
hgMetricX.end();
log.info("*************************************************");
log.info("************* Scanning Completed **************");
log.info("Graph: {}", graphName);
log.info("Table: {}", tableName);
log.info("Keys: {}", count);
log.info("Total: {} seconds.", hgMetricX.past() / 1000);
log.info("*************************************************");
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli.scan;
import java.util.Arrays;
import java.util.List;
import org.apache.hugegraph.pd.client.PDClient;
import org.apache.hugegraph.pd.common.PDException;
import org.apache.hugegraph.pd.grpc.Metapb;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgKvStore;
import org.apache.hugegraph.store.HgScanQuery;
import org.apache.hugegraph.store.HgSessionManager;
import org.apache.hugegraph.store.HgStoreClient;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.cli.util.HgCliUtil;
import org.apache.hugegraph.store.cli.util.HgMetricX;
import org.apache.hugegraph.store.client.grpc.KvCloseableIterator;
import org.apache.hugegraph.store.client.util.HgStoreClientConfig;
import org.apache.hugegraph.store.client.util.MetricX;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com on 2022/2/14
*/
@Slf4j
public class HgStoreScanner {
private final HgStoreClient storeClient;
private final String graphName;
private long modNumber = 1_000_000;
private int max = 10_000_000;
public long getModNumber() {
return modNumber;
}
public void setModNumber(int modNumber) {
if (modNumber <= 0) {
return;
}
this.modNumber = modNumber;
}
public int getMax() {
return max;
}
public void setMax(int max) {
if (modNumber <= 0) {
return;
}
this.max = max;
}
public static HgStoreScanner of(HgStoreClient storeClient, String graph) {
return new HgStoreScanner(storeClient, graph);
}
private HgStoreScanner(HgStoreClient storeClient, String graph) {
this.storeClient = storeClient;
this.graphName = graph;
}
protected HgStoreSession getStoreSession() {
return HgSessionManager.getInstance().openSession(this.graphName);
}
protected HgStoreSession getStoreSession(String graphName) {
return HgSessionManager.getInstance().openSession(graphName);
}
public void scanTable(String tableName) {
log.info("Starting scan table [{}] of graph [{}] ...", tableName, graphName);
HgMetricX hgMetricX = HgMetricX.ofStart();
HgStoreSession session = getStoreSession();
int count = 0;
KvCloseableIterator<HgKvIterator<HgKvEntry>> iterator =
session.scanBatch2(HgScanQuery.tableOf(tableName));
long start = System.currentTimeMillis();
while (iterator.hasNext()) {
HgKvIterator<HgKvEntry> iterator2 = iterator.next();
while (iterator2.hasNext()) {
count++;
iterator2.next();
if (count % (modNumber) == 0) {
log.info("Scanning keys: " + count + " time is " + modNumber * 1000
/
(System.currentTimeMillis() -
start));
start = System.currentTimeMillis();
}
if (count == max) {
break;
}
}
}
iterator.close();
hgMetricX.end();
log.info("*************************************************");
log.info("************* Scanning Completed **************");
log.info("Graph: {}", graphName);
log.info("Table: {}", tableName);
log.info("Keys: {}", count);
log.info("Max: {}", max);
log.info("Waiting: {} seconds.", MetricX.getIteratorWait() / 1000);
log.info("Total: {} seconds.", hgMetricX.past() / 1000);
log.info("Iterator: [{}]", iterator.getClass().getSimpleName());
log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize());
log.info("*************************************************");
}
public void scanHash() {
String tableName = "g+i";
HgMetricX hgMetricX = HgMetricX.ofStart();
String graphName = "/DEFAULT/graphs/hugegraph1/";
HgStoreSession session = getStoreSession(graphName);
int count = 0;
String query =
"{\"conditions\":[{\"cls\":\"S\",\"el\":{\"key\":\"ID\",\"relation\":\"SCAN\"," +
"\"value\"" +
":{\"start\":\"61180\",\"end\":\"63365\",\"length\":0}}}]," +
"\"optimizedType\":\"NONE\",\"ids\":[]," +
"\"mustSortByInput\":true,\"resultType\":\"EDGE\",\"offset\":0," +
"\"actualOffset\":0,\"actualStoreOffset\":" +
"0,\"limit\":9223372036854775807,\"capacity\":-1,\"showHidden\":false," +
"\"showDeleting\":false," +
"\"showExpired\":false,\"olap\":false,\"withProperties\":false,\"olapPks\":[]}";
//HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,0,715827883,
// HgKvStore.SCAN_ANY,null);
//HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,61180,63365, 348, null);
//HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,0,65535, 348, null);
HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName);
while (iterator.hasNext()) {
count++;
//iterator.next();
// if (count % (modNumber) == 0) {
// log.info("Scanning keys: " + count);
HgCliUtil.println(Arrays.toString(iterator.next().key()));
// }
if (count == max) {
break;
}
}
hgMetricX.end();
log.info("*************************************************");
log.info("************* Scanning Completed **************");
log.info("Graph: {}", this.graphName);
log.info("Table: {}", tableName);
log.info("Keys: {}", count);
log.info("Max: {}", max);
log.info("Waiting: {} seconds.", MetricX.getIteratorWait() / 1000);
log.info("Total: {} seconds.", hgMetricX.past() / 1000);
log.info("Iterator: [{}]", iterator.getClass().getSimpleName());
log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize());
log.info("*************************************************");
}
public static final byte[] EMPTY_BYTES = new byte[0];
public void scanTable2(String tableName) throws PDException {
// java -jar hg-store-cli-3.6.0-SNAPSHOT.jar -scan 10.45.30.212:8989 "DEFAULT/case_112/g"
// g+ie
PDClient pdClient = storeClient.getPdClient();
List<Metapb.Partition> partitions = pdClient.getPartitions(0, graphName);
HgStoreSession session = storeClient.openSession(graphName);
int count = 0;
byte[] position = null;
HgMetricX hgMetricX = HgMetricX.ofStart();
for (Metapb.Partition partition : partitions) {
while (true) {
try (HgKvIterator<HgKvEntry> iterator = session.scanIterator(tableName,
(int) (partition.getStartKey()),
(int) (partition.getEndKey()),
HgKvStore.SCAN_HASHCODE,
EMPTY_BYTES)) {
if (position != null) {
iterator.seek(position);
}
while (iterator.hasNext()) {
iterator.next();
count++;
if (count % 3000 == 0) {
if (iterator.hasNext()) {
iterator.next();
position = iterator.position();
System.out.println("count is " + count);
} else {
position = null;
}
break;
}
}
if (!iterator.hasNext()) {
position = null;
break;
}
}
}
}
hgMetricX.end();
log.info("*************************************************");
log.info("************* Scanning Completed **************");
log.info("Graph: {}", graphName);
log.info("Table: {}", tableName);
log.info("Keys: {}", count);
log.info("Total: {} seconds.", hgMetricX.past() / 1000);
log.info("*************************************************");
}
}

482
hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgCliUtil.java
View File

@ -1,241 +1,241 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli.util;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
/**
* @author lynn.bond@hotmail.com on 2022/2/14
*/
public class HgCliUtil {
public final static String TABLE_NAME = "cli-table";
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String keyPrefix) {
return batchPut(session, keyPrefix, 100);
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String keyPrefix,
int loop) {
return batchPut(session, TABLE_NAME, keyPrefix, loop);
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
, String keyPrefix, int loop) {
return batchPut(session, tableName, keyPrefix, loop, 1, key -> toOwnerKey(key));
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
, String keyPrefix, int loop, int start) {
return batchPut(session, tableName, keyPrefix, loop, start, key -> toOwnerKey(key));
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
, String keyPrefix, int loop, Function<String, HgOwnerKey> f) {
return batchPut(session, tableName, keyPrefix, loop, 1, f);
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
, String keyPrefix, int loop, int start, Function<String, HgOwnerKey> f) {
Map<HgOwnerKey, byte[]> res = new LinkedHashMap<>();
int length = String.valueOf(loop).length();
session.beginTx();
for (int i = start; i <= loop; i++) {
HgOwnerKey key = f.apply(keyPrefix + "-" + padLeftZeros(String.valueOf(i), length));
byte[] value = toBytes(keyPrefix + "-V-" + i);
res.put(key, value);
session.put(tableName, key, value);
if ((i + 1) % 10000 == 0) {
println("commit: " + (i + 1));
session.commit();
session.beginTx();
}
}
if (session.isTx()) {
session.commit();
}
return res;
}
public static void printNum(List<HgKvEntry> list, String title) {
if (list == null) return;
println(title + " size: " + list.size());
}
public static void println(Iterator<HgKvEntry> iterator) {
if (iterator == null) return;
while (iterator.hasNext()) {
println(iterator.next());
}
}
public static void printOwner(List<HgOwnerKey> list) {
if (list == null) return;
for (HgOwnerKey entry : list) {
println(entry);
}
}
public static void println(List<HgKvEntry> list) {
if (list == null) return;
for (HgKvEntry entry : list) {
println(entry);
}
}
public static void println(List<HgKvEntry> list, int mod) {
if (list == null) return;
for (int i = 0; i < list.size(); i++) {
if (i % mod == 0) {
println(list.get(i));
}
}
}
public static void println(HgKvEntry kv) {
if (kv == null) {
System.out.println("null");
return;
}
println("[ " + toStr(kv.key()) + " : " + toStr(kv.value()) + " ]");
}
public static void println(HgOwnerKey key) {
if (key == null) {
System.out.println("null");
return;
}
println("[ " + toInt(key.getOwner()) + " : " + toStr(key.getKey()) + " ]");
}
public static void println(String str) {
System.out.println(str);
}
public static HgOwnerKey toOwnerKey(String key) {
return new HgOwnerKey(getOwner(key), toBytes(key));
}
public static HgOwnerKey toOwnerKey(byte[] key) {
return new HgOwnerKey(getOwner(key), key);
}
private static byte[] getOwner(String key) {
return getOwner(toBytes(key));
}
private static byte[] getOwner(byte[] key) {
return toBytes(Arrays.hashCode(key));
}
public static HgOwnerKey toAllPartitionKey(String key) {
return HgOwnerKey.of(HgStoreClientConst.ALL_PARTITION_OWNER, toBytes(key));
}
public static HgOwnerKey toOwnerKey(String owner, String key) {
return HgOwnerKey.of(toBytes(owner), toBytes(key));
}
public static String toStr(byte[] b) {
if (b == null) return "";
if (b.length == 0) return "";
return new String(b, StandardCharsets.UTF_8);
}
public static byte[] toBytes(String str) {
if (str == null) return null;
return str.getBytes(StandardCharsets.UTF_8);
}
public static byte[] toBytes(long l) {
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
buffer.putLong(l);
return buffer.array();
}
private static byte[] toBytes(final int i) {
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
buffer.putInt(i);
return buffer.array();
}
public static long toLong(byte[] bytes) {
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
buffer.put(bytes);
buffer.flip();//need flip
return buffer.getLong();
}
public static long toInt(byte[] bytes) {
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
buffer.put(bytes);
buffer.flip();//need flip
return buffer.getInt();
}
public static String padLeftZeros(String str, int n) {
return String.format("%1$" + n + "s", str).replace(' ', '0');
}
public static String toSuffix(int num, int length) {
return "-" + padLeftZeros(String.valueOf(num), length);
}
public static int amountOf(List list) {
if (list == null) {
return 0;
}
return list.size();
}
public static int amountOf(Iterator iterator) {
if (iterator == null) return 0;
int count = 0;
while (iterator.hasNext()) {
iterator.next();
count++;
}
return count;
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli.util;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgStoreSession;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
/**
* @author lynn.bond@hotmail.com on 2022/2/14
*/
public class HgCliUtil {
public final static String TABLE_NAME = "cli-table";
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String keyPrefix) {
return batchPut(session, keyPrefix, 100);
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String keyPrefix,
int loop) {
return batchPut(session, TABLE_NAME, keyPrefix, loop);
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
, String keyPrefix, int loop) {
return batchPut(session, tableName, keyPrefix, loop, 1, key -> toOwnerKey(key));
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
, String keyPrefix, int loop, int start) {
return batchPut(session, tableName, keyPrefix, loop, start, key -> toOwnerKey(key));
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
, String keyPrefix, int loop, Function<String, HgOwnerKey> f) {
return batchPut(session, tableName, keyPrefix, loop, 1, f);
}
public static Map<HgOwnerKey, byte[]> batchPut(HgStoreSession session, String tableName
, String keyPrefix, int loop, int start, Function<String, HgOwnerKey> f) {
Map<HgOwnerKey, byte[]> res = new LinkedHashMap<>();
int length = String.valueOf(loop).length();
session.beginTx();
for (int i = start; i <= loop; i++) {
HgOwnerKey key = f.apply(keyPrefix + "-" + padLeftZeros(String.valueOf(i), length));
byte[] value = toBytes(keyPrefix + "-V-" + i);
res.put(key, value);
session.put(tableName, key, value);
if ((i + 1) % 10000 == 0) {
println("commit: " + (i + 1));
session.commit();
session.beginTx();
}
}
if (session.isTx()) {
session.commit();
}
return res;
}
public static void printNum(List<HgKvEntry> list, String title) {
if (list == null) return;
println(title + " size: " + list.size());
}
public static void println(Iterator<HgKvEntry> iterator) {
if (iterator == null) return;
while (iterator.hasNext()) {
println(iterator.next());
}
}
public static void printOwner(List<HgOwnerKey> list) {
if (list == null) return;
for (HgOwnerKey entry : list) {
println(entry);
}
}
public static void println(List<HgKvEntry> list) {
if (list == null) return;
for (HgKvEntry entry : list) {
println(entry);
}
}
public static void println(List<HgKvEntry> list, int mod) {
if (list == null) return;
for (int i = 0; i < list.size(); i++) {
if (i % mod == 0) {
println(list.get(i));
}
}
}
public static void println(HgKvEntry kv) {
if (kv == null) {
System.out.println("null");
return;
}
println("[ " + toStr(kv.key()) + " : " + toStr(kv.value()) + " ]");
}
public static void println(HgOwnerKey key) {
if (key == null) {
System.out.println("null");
return;
}
println("[ " + toInt(key.getOwner()) + " : " + toStr(key.getKey()) + " ]");
}
public static void println(String str) {
System.out.println(str);
}
public static HgOwnerKey toOwnerKey(String key) {
return new HgOwnerKey(getOwner(key), toBytes(key));
}
public static HgOwnerKey toOwnerKey(byte[] key) {
return new HgOwnerKey(getOwner(key), key);
}
private static byte[] getOwner(String key) {
return getOwner(toBytes(key));
}
private static byte[] getOwner(byte[] key) {
return toBytes(Arrays.hashCode(key));
}
public static HgOwnerKey toAllPartitionKey(String key) {
return HgOwnerKey.of(HgStoreClientConst.ALL_PARTITION_OWNER, toBytes(key));
}
public static HgOwnerKey toOwnerKey(String owner, String key) {
return HgOwnerKey.of(toBytes(owner), toBytes(key));
}
public static String toStr(byte[] b) {
if (b == null) return "";
if (b.length == 0) return "";
return new String(b, StandardCharsets.UTF_8);
}
public static byte[] toBytes(String str) {
if (str == null) return null;
return str.getBytes(StandardCharsets.UTF_8);
}
public static byte[] toBytes(long l) {
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
buffer.putLong(l);
return buffer.array();
}
private static byte[] toBytes(final int i) {
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
buffer.putInt(i);
return buffer.array();
}
public static long toLong(byte[] bytes) {
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
buffer.put(bytes);
buffer.flip();//need flip
return buffer.getLong();
}
public static long toInt(byte[] bytes) {
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
buffer.put(bytes);
buffer.flip();//need flip
return buffer.getInt();
}
public static String padLeftZeros(String str, int n) {
return String.format("%1$" + n + "s", str).replace(' ', '0');
}
public static String toSuffix(int num, int length) {
return "-" + padLeftZeros(String.valueOf(num), length);
}
public static int amountOf(List list) {
if (list == null) {
return 0;
}
return list.size();
}
public static int amountOf(Iterator iterator) {
if (iterator == null) return 0;
int count = 0;
while (iterator.hasNext()) {
iterator.next();
count++;
}
return count;
}
}

122
hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgMetricX.java
View File

@ -1,61 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli.util;
/**
* @author lynn.bond@hotmail.com on 2022/1/29
*/
public class HgMetricX {
private long start;
private long end;
private long waitStart = System.currentTimeMillis();
private long waitTotal;
public static HgMetricX ofStart() {
return new HgMetricX(System.currentTimeMillis());
}
private HgMetricX(long start) {
this.start = start;
}
public long start() {
return this.start = System.currentTimeMillis();
}
public long end() {
return this.end = System.currentTimeMillis();
}
public long past() {
return this.end - this.start;
}
public long getWaitTotal() {
return this.waitTotal;
}
public void startWait() {
this.waitStart = System.currentTimeMillis();
}
public void appendWait() {
this.waitTotal += System.currentTimeMillis() - waitStart;
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.cli.util;
/**
* @author lynn.bond@hotmail.com on 2022/1/29
*/
public class HgMetricX {
private long start;
private long end;
private long waitStart = System.currentTimeMillis();
private long waitTotal;
public static HgMetricX ofStart() {
return new HgMetricX(System.currentTimeMillis());
}
private HgMetricX(long start) {
this.start = start;
}
public long start() {
return this.start = System.currentTimeMillis();
}
public long end() {
return this.end = System.currentTimeMillis();
}
public long past() {
return this.end - this.start;
}
public long getWaitTotal() {
return this.waitTotal;
}
public void startWait() {
this.waitStart = System.currentTimeMillis();
}
public void appendWait() {
this.waitTotal += System.currentTimeMillis() - waitStart;
}
}

68
hg-store-client/src/main/java/org/apache/hugegraph/store/HgKvIterator.java
View File

@ -1,34 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store;
import java.io.Closeable;
import java.util.Iterator;
/**
* @author lynn.bond@hotmail.com created on 2021/10/21
*/
public interface HgKvIterator<E> extends Iterator<E>, HgSeekAble, Closeable {
byte[] key();
byte[] value();
void close();
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store;
import java.io.Closeable;
import java.util.Iterator;
/**
* @author lynn.bond@hotmail.com created on 2021/10/21
*/
public interface HgKvIterator<E> extends Iterator<E>, HgSeekAble, Closeable {
byte[] key();
byte[] value();
void close();
}

664
hg-store-client/src/main/java/org/apache/hugegraph/store/HgScanQuery.java
View File

@ -1,332 +1,332 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import org.apache.hugegraph.store.client.util.HgAssert;
import org.apache.hugegraph.store.grpc.common.ScanOrderType;
/**
* @author lynn.bond@hotmail.com on 2022/3/4
* @version 0.5.0
*/
public interface HgScanQuery {
String getTable();
HgScanQuery.ScanMethod getScanMethod();
List<HgOwnerKey> getPrefixList();
Iterator<HgOwnerKey> getPrefixItr();
List<HgOwnerKey> getStartList();
List<HgOwnerKey> getEndList();
long getLimit();
long getPerKeyLimit();
long getPerKeyMax();
long getSkipDegree();
int getScanType();
ScanOrderType getOrderType();
boolean isOnlyKey();
byte[] getQuery();
enum ScanMethod {
ALL,
PREFIX,
RANGE
}
enum SortType {
UNSORTED,
SORT_BY_EDGE,
SORT_BY_VERTEX
}
ScanBuilder builder();
static HgScanQuery tableOf(String table) {
return ScanBuilder.tableOf(table).build();
}
static HgScanQuery rangeOf(String table, List<HgOwnerKey> startList, List<HgOwnerKey> endList) {
return ScanBuilder.rangeOf(table, startList, endList).build();
}
static HgScanQuery prefixOf(String table, List<HgOwnerKey> prefixList) {
return ScanBuilder.prefixOf(table, prefixList).build();
}
static HgScanQuery prefixOf(String table, List<HgOwnerKey> prefixList,
ScanOrderType orderType) {
return ScanBuilder.prefixOf(table, prefixList).setOrderType(orderType).build();
}
static HgScanQuery prefixIteratorOf(String table, Iterator<HgOwnerKey> prefixItr) {
return ScanBuilder.prefixIteratorOf(table, prefixItr).build();
}
static HgScanQuery prefixIteratorOf(String table, Iterator<HgOwnerKey> prefixItr,
ScanOrderType orderType) {
return ScanBuilder.prefixIteratorOf(table, prefixItr).setOrderType(orderType).build();
}
class ScanBuilder {
private final String table;
private final HgScanQuery.ScanMethod sanMethod;
private long limit = Integer.MAX_VALUE;
private long perKeyLimit = Integer.MAX_VALUE;
private long perKeyMax = Integer.MAX_VALUE;
private int scanType;
private ScanOrderType orderType;
private long skipDegree;
private boolean onlyKey;
private byte[] query;
private List<HgOwnerKey> prefixList;
private List<HgOwnerKey> startList;
private List<HgOwnerKey> endList;
private Iterator<HgOwnerKey> prefixItr;
public static ScanBuilder rangeOf(String table, List<HgOwnerKey> startList,
List<HgOwnerKey> endList) {
HgAssert.isArgumentValid(table, "table");
HgAssert.isArgumentValid(startList, "startList");
HgAssert.isArgumentValid(endList, "endList");
HgAssert.isTrue(startList.size() == endList.size()
, "The size of startList not equals endList's.");
ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.RANGE, table);
res.startList = startList;
res.endList = endList;
res.scanType = HgKvStore.SCAN_GTE_BEGIN | HgKvStore.SCAN_LTE_END;
return res;
}
public static ScanBuilder prefixOf(String table, List<HgOwnerKey> prefixList) {
HgAssert.isArgumentValid(table, "table");
HgAssert.isArgumentValid(prefixList, "prefixList");
ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.PREFIX, table);
res.prefixList = prefixList;
return res;
}
public static ScanBuilder prefixIteratorOf(String table, Iterator<HgOwnerKey> prefixItr) {
HgAssert.isArgumentValid(table, "table");
ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.PREFIX, table);
res.prefixItr = prefixItr;
return res;
}
public static ScanBuilder tableOf(String table) {
HgAssert.isArgumentValid(table, "table");
return new ScanBuilder(HgScanQuery.ScanMethod.ALL, table);
}
ScanBuilder(HgScanQuery.ScanMethod sanMethod, String table) {
this.table = table;
this.sanMethod = sanMethod;
this.orderType = ScanOrderType.ORDER_NONE;
}
public ScanBuilder setLimit(long limit) {
this.limit = limit;
return this;
}
public ScanBuilder setPerKeyLimit(long limit) {
this.perKeyLimit = limit;
return this;
}
public ScanBuilder setPerKeyMax(long max) {
this.perKeyMax = max;
return this;
}
public ScanBuilder setScanType(int scanType) {
this.scanType = scanType;
return this;
}
public ScanBuilder setOrderType(ScanOrderType orderType) {
this.orderType = orderType;
return this;
}
public ScanBuilder setQuery(byte[] query) {
this.query = query;
return this;
}
public ScanBuilder setSkipDegree(long skipDegree) {
this.skipDegree = skipDegree;
return this;
}
public ScanBuilder setOnlyKey(boolean onlyKey) {
this.onlyKey = onlyKey;
return this;
}
public HgScanQuery build() {
return this.new BatchScanQuery();
}
private class BatchScanQuery implements HgScanQuery {
@Override
public String getTable() {
return table;
}
@Override
public HgScanQuery.ScanMethod getScanMethod() {
return sanMethod;
}
@Override
public List<HgOwnerKey> getPrefixList() {
if (prefixList == null) {
return Collections.EMPTY_LIST;
} else {
return Collections.unmodifiableList(prefixList);
}
}
@Override
public Iterator<HgOwnerKey> getPrefixItr() {
return prefixItr;
}
@Override
public List<HgOwnerKey> getStartList() {
if (startList == null) {
return Collections.EMPTY_LIST;
} else {
return Collections.unmodifiableList(startList);
}
}
@Override
public List<HgOwnerKey> getEndList() {
if (endList == null) {
return Collections.EMPTY_LIST;
} else {
return Collections.unmodifiableList(endList);
}
}
@Override
public long getLimit() {
return limit;
}
@Override
public long getPerKeyLimit() {
return perKeyLimit;
}
@Override
public long getPerKeyMax() {
return perKeyMax;
}
@Override
public long getSkipDegree() {
return skipDegree;
}
@Override
public int getScanType() {
return scanType;
}
@Override
public ScanOrderType getOrderType() {
return orderType;
}
@Override
public boolean isOnlyKey() {
return onlyKey;
}
@Override
public byte[] getQuery() {
return query;
}
@Override
public ScanBuilder builder() {
return ScanBuilder.this;
}
@Override
public String toString() {
final StringBuffer sb = new StringBuffer("HgScanQuery{");
sb.append("table='").append(getTable()).append('\'');
sb.append(", scanMethod=").append(getScanMethod());
sb.append(", prefixList=").append(getPrefixList());
sb.append(", startList=").append(getStartList());
sb.append(", endList=").append(getEndList());
sb.append(", limit=").append(getLimit());
sb.append(", perKeyLimit=").append(getPerKeyLimit());
sb.append(", perKeyMax=").append(getPerKeyMax());
sb.append(", skipDegree=").append(getSkipDegree());
sb.append(", scanType=").append(getScanType());
sb.append(", orderType=").append(getOrderType());
sb.append(", onlyKey=").append(isOnlyKey());
sb.append(", query=");
if (query == null) {
sb.append("null");
} else {
sb.append('[');
for (int i = 0; i < query.length; ++i) {
sb.append(i == 0 ? "" : ", ").append(query[i]);
}
sb.append(']');
}
sb.append('}');
return sb.toString();
}
}
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import org.apache.hugegraph.store.client.util.HgAssert;
import org.apache.hugegraph.store.grpc.common.ScanOrderType;
/**
* @author lynn.bond@hotmail.com on 2022/3/4
* @version 0.5.0
*/
public interface HgScanQuery {
String getTable();
HgScanQuery.ScanMethod getScanMethod();
List<HgOwnerKey> getPrefixList();
Iterator<HgOwnerKey> getPrefixItr();
List<HgOwnerKey> getStartList();
List<HgOwnerKey> getEndList();
long getLimit();
long getPerKeyLimit();
long getPerKeyMax();
long getSkipDegree();
int getScanType();
ScanOrderType getOrderType();
boolean isOnlyKey();
byte[] getQuery();
enum ScanMethod {
ALL,
PREFIX,
RANGE
}
enum SortType {
UNSORTED,
SORT_BY_EDGE,
SORT_BY_VERTEX
}
ScanBuilder builder();
static HgScanQuery tableOf(String table) {
return ScanBuilder.tableOf(table).build();
}
static HgScanQuery rangeOf(String table, List<HgOwnerKey> startList, List<HgOwnerKey> endList) {
return ScanBuilder.rangeOf(table, startList, endList).build();
}
static HgScanQuery prefixOf(String table, List<HgOwnerKey> prefixList) {
return ScanBuilder.prefixOf(table, prefixList).build();
}
static HgScanQuery prefixOf(String table, List<HgOwnerKey> prefixList,
ScanOrderType orderType) {
return ScanBuilder.prefixOf(table, prefixList).setOrderType(orderType).build();
}
static HgScanQuery prefixIteratorOf(String table, Iterator<HgOwnerKey> prefixItr) {
return ScanBuilder.prefixIteratorOf(table, prefixItr).build();
}
static HgScanQuery prefixIteratorOf(String table, Iterator<HgOwnerKey> prefixItr,
ScanOrderType orderType) {
return ScanBuilder.prefixIteratorOf(table, prefixItr).setOrderType(orderType).build();
}
class ScanBuilder {
private final String table;
private final HgScanQuery.ScanMethod sanMethod;
private long limit = Integer.MAX_VALUE;
private long perKeyLimit = Integer.MAX_VALUE;
private long perKeyMax = Integer.MAX_VALUE;
private int scanType;
private ScanOrderType orderType;
private long skipDegree;
private boolean onlyKey;
private byte[] query;
private List<HgOwnerKey> prefixList;
private List<HgOwnerKey> startList;
private List<HgOwnerKey> endList;
private Iterator<HgOwnerKey> prefixItr;
public static ScanBuilder rangeOf(String table, List<HgOwnerKey> startList,
List<HgOwnerKey> endList) {
HgAssert.isArgumentValid(table, "table");
HgAssert.isArgumentValid(startList, "startList");
HgAssert.isArgumentValid(endList, "endList");
HgAssert.isTrue(startList.size() == endList.size()
, "The size of startList not equals endList's.");
ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.RANGE, table);
res.startList = startList;
res.endList = endList;
res.scanType = HgKvStore.SCAN_GTE_BEGIN | HgKvStore.SCAN_LTE_END;
return res;
}
public static ScanBuilder prefixOf(String table, List<HgOwnerKey> prefixList) {
HgAssert.isArgumentValid(table, "table");
HgAssert.isArgumentValid(prefixList, "prefixList");
ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.PREFIX, table);
res.prefixList = prefixList;
return res;
}
public static ScanBuilder prefixIteratorOf(String table, Iterator<HgOwnerKey> prefixItr) {
HgAssert.isArgumentValid(table, "table");
ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.PREFIX, table);
res.prefixItr = prefixItr;
return res;
}
public static ScanBuilder tableOf(String table) {
HgAssert.isArgumentValid(table, "table");
return new ScanBuilder(HgScanQuery.ScanMethod.ALL, table);
}
ScanBuilder(HgScanQuery.ScanMethod sanMethod, String table) {
this.table = table;
this.sanMethod = sanMethod;
this.orderType = ScanOrderType.ORDER_NONE;
}
public ScanBuilder setLimit(long limit) {
this.limit = limit;
return this;
}
public ScanBuilder setPerKeyLimit(long limit) {
this.perKeyLimit = limit;
return this;
}
public ScanBuilder setPerKeyMax(long max) {
this.perKeyMax = max;
return this;
}
public ScanBuilder setScanType(int scanType) {
this.scanType = scanType;
return this;
}
public ScanBuilder setOrderType(ScanOrderType orderType) {
this.orderType = orderType;
return this;
}
public ScanBuilder setQuery(byte[] query) {
this.query = query;
return this;
}
public ScanBuilder setSkipDegree(long skipDegree) {
this.skipDegree = skipDegree;
return this;
}
public ScanBuilder setOnlyKey(boolean onlyKey) {
this.onlyKey = onlyKey;
return this;
}
public HgScanQuery build() {
return this.new BatchScanQuery();
}
private class BatchScanQuery implements HgScanQuery {
@Override
public String getTable() {
return table;
}
@Override
public HgScanQuery.ScanMethod getScanMethod() {
return sanMethod;
}
@Override
public List<HgOwnerKey> getPrefixList() {
if (prefixList == null) {
return Collections.EMPTY_LIST;
} else {
return Collections.unmodifiableList(prefixList);
}
}
@Override
public Iterator<HgOwnerKey> getPrefixItr() {
return prefixItr;
}
@Override
public List<HgOwnerKey> getStartList() {
if (startList == null) {
return Collections.EMPTY_LIST;
} else {
return Collections.unmodifiableList(startList);
}
}
@Override
public List<HgOwnerKey> getEndList() {
if (endList == null) {
return Collections.EMPTY_LIST;
} else {
return Collections.unmodifiableList(endList);
}
}
@Override
public long getLimit() {
return limit;
}
@Override
public long getPerKeyLimit() {
return perKeyLimit;
}
@Override
public long getPerKeyMax() {
return perKeyMax;
}
@Override
public long getSkipDegree() {
return skipDegree;
}
@Override
public int getScanType() {
return scanType;
}
@Override
public ScanOrderType getOrderType() {
return orderType;
}
@Override
public boolean isOnlyKey() {
return onlyKey;
}
@Override
public byte[] getQuery() {
return query;
}
@Override
public ScanBuilder builder() {
return ScanBuilder.this;
}
@Override
public String toString() {
final StringBuffer sb = new StringBuffer("HgScanQuery{");
sb.append("table='").append(getTable()).append('\'');
sb.append(", scanMethod=").append(getScanMethod());
sb.append(", prefixList=").append(getPrefixList());
sb.append(", startList=").append(getStartList());
sb.append(", endList=").append(getEndList());
sb.append(", limit=").append(getLimit());
sb.append(", perKeyLimit=").append(getPerKeyLimit());
sb.append(", perKeyMax=").append(getPerKeyMax());
sb.append(", skipDegree=").append(getSkipDegree());
sb.append(", scanType=").append(getScanType());
sb.append(", orderType=").append(getOrderType());
sb.append(", onlyKey=").append(isOnlyKey());
sb.append(", query=");
if (query == null) {
sb.append("null");
} else {
sb.append('[');
for (int i = 0; i < query.length; ++i) {
sb.append(i == 0 ? "" : ", ").append(query[i]);
}
sb.append(']');
}
sb.append('}');
return sb.toString();
}
}
}
}

314
hg-store-client/src/main/java/org/apache/hugegraph/store/client/ShiftWorkIteratorProxy.java
View File

@ -1,157 +1,157 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client;
import java.util.LinkedList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Queue;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgKvPagingIterator;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2021/10/24
* @version 0.1.1
*/
@Slf4j
public class ShiftWorkIteratorProxy implements HgKvIterator {
private static final byte[] EMPTY_BYTES = new byte[0];
private HgKvPagingIterator<HgKvEntry> iterator;
private Queue<HgKvPagingIterator> queue = new LinkedList<>();
private HgKvEntry entry;
private final int limit;
private int count;
private int shiftCount;
ShiftWorkIteratorProxy(List<HgKvPagingIterator> iterators, int limit) {
this.queue = new LinkedList<>(iterators);
this.limit = limit <= 0 ? Integer.MAX_VALUE : limit;
}
private HgKvPagingIterator getIterator() {
if (this.queue.isEmpty()) return null;
HgKvPagingIterator buf = null;
while ((buf = this.queue.poll()) != null) {
if (buf.hasNext()) {
break;
}
}
if (buf == null) {
return null;
}
this.queue.add(buf);
return buf;
}
private void closeIterators() {
if (this.queue.isEmpty()) return;
HgKvPagingIterator buf;
while ((buf = this.queue.poll()) != null) {
buf.close();
}
}
private void setIterator() {
// if (++this.shiftCount >= this.iterator.getPageSize() / 2) {
if (++this.shiftCount >= this.iterator.getPageSize()) {
this.iterator = null;
this.shiftCount = 0;
}
}
private void doNext() {
}
@Override
public byte[] key() {
if (this.entry != null) {
return this.entry.key();
}
return null;
}
@Override
public byte[] value() {
if (this.entry != null) {
return this.entry.value();
}
return null;
}
@Override
public byte[] position() {
return this.iterator != null ? this.iterator.position() : EMPTY_BYTES;
}
@Override
public void seek(byte[] position) {
if (this.iterator != null) {
this.iterator.seek(position);
}
}
@Override
public boolean hasNext() {
if (this.count >= this.limit) {
return false;
}
if (this.iterator == null
|| !this.iterator.hasNext()) {
this.iterator = this.getIterator();
}
return this.iterator != null;
}
@Override
public Object next() {
if (this.iterator == null) {
hasNext();
}
if (this.iterator == null) {
throw new NoSuchElementException();
}
this.entry = this.iterator.next();
this.setIterator();
this.count++;
//log.info("next - > {}",this.entry);
return this.entry;
}
@Override
public void close() {
if (this.iterator != null) {
this.iterator.close();
}
this.closeIterators();
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client;
import java.util.LinkedList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Queue;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgKvPagingIterator;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2021/10/24
* @version 0.1.1
*/
@Slf4j
public class ShiftWorkIteratorProxy implements HgKvIterator {
private static final byte[] EMPTY_BYTES = new byte[0];
private HgKvPagingIterator<HgKvEntry> iterator;
private Queue<HgKvPagingIterator> queue = new LinkedList<>();
private HgKvEntry entry;
private final int limit;
private int count;
private int shiftCount;
ShiftWorkIteratorProxy(List<HgKvPagingIterator> iterators, int limit) {
this.queue = new LinkedList<>(iterators);
this.limit = limit <= 0 ? Integer.MAX_VALUE : limit;
}
private HgKvPagingIterator getIterator() {
if (this.queue.isEmpty()) return null;
HgKvPagingIterator buf = null;
while ((buf = this.queue.poll()) != null) {
if (buf.hasNext()) {
break;
}
}
if (buf == null) {
return null;
}
this.queue.add(buf);
return buf;
}
private void closeIterators() {
if (this.queue.isEmpty()) return;
HgKvPagingIterator buf;
while ((buf = this.queue.poll()) != null) {
buf.close();
}
}
private void setIterator() {
// if (++this.shiftCount >= this.iterator.getPageSize() / 2) {
if (++this.shiftCount >= this.iterator.getPageSize()) {
this.iterator = null;
this.shiftCount = 0;
}
}
private void doNext() {
}
@Override
public byte[] key() {
if (this.entry != null) {
return this.entry.key();
}
return null;
}
@Override
public byte[] value() {
if (this.entry != null) {
return this.entry.value();
}
return null;
}
@Override
public byte[] position() {
return this.iterator != null ? this.iterator.position() : EMPTY_BYTES;
}
@Override
public void seek(byte[] position) {
if (this.iterator != null) {
this.iterator.seek(position);
}
}
@Override
public boolean hasNext() {
if (this.count >= this.limit) {
return false;
}
if (this.iterator == null
|| !this.iterator.hasNext()) {
this.iterator = this.getIterator();
}
return this.iterator != null;
}
@Override
public Object next() {
if (this.iterator == null) {
hasNext();
}
if (this.iterator == null) {
throw new NoSuchElementException();
}
this.entry = this.iterator.next();
this.setIterator();
this.count++;
//log.info("next - > {}",this.entry);
return this.entry;
}
@Override
public void close() {
if (this.iterator != null) {
this.iterator.close();
}
this.closeIterators();
}
}

274
hg-store-client/src/main/java/org/apache/hugegraph/store/client/TopWorkIteratorProxy.java
View File

@ -1,137 +1,137 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client;
import java.util.LinkedList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Queue;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
/**
* @author lynn.bond@hotmail.com created on 2021/10/21
* @version 0.1.0
*/
class TopWorkIteratorProxy implements HgKvIterator {
private static final byte[] EMPTY_BYTES = new byte[0];
private HgKvIterator<HgKvEntry> iterator;
private final Queue<HgKvIterator> queue;
private HgKvEntry entry;
private final long limit;
private int count;
TopWorkIteratorProxy(List<HgKvIterator> iterators, long limit) {
this.queue = new LinkedList<>(iterators);
this.limit = limit <= 0 ? Integer.MAX_VALUE : limit;
}
private HgKvIterator getIterator() {
if (this.queue.isEmpty()) return null;
HgKvIterator buf = null;
while ((buf = this.queue.poll()) != null) {
if (buf.hasNext()) {
break;
}
}
if (buf == null) {
return null;
}
this.queue.add(buf);
return buf;
}
private void closeIterators() {
if (this.queue.isEmpty()) return;
HgKvIterator buf;
while ((buf = this.queue.poll()) != null) {
buf.close();
}
}
private void setIterator() {
this.iterator = null;
}
@Override
public byte[] key() {
if (this.entry != null) {
return this.entry.key();
}
return null;
}
@Override
public byte[] value() {
if (this.entry != null) {
return this.entry.value();
}
return null;
}
@Override
public byte[] position() {
return this.iterator != null ? this.iterator.position() : EMPTY_BYTES;
}
@Override
public void seek(byte[] position) {
if (this.iterator != null) this.iterator.seek(position);
}
@Override
public boolean hasNext() {
if (this.count >= this.limit) {
return false;
}
if (this.iterator == null) {
this.iterator = this.getIterator();
}
return this.iterator != null;
}
@Override
public Object next() {
if (this.iterator == null) {
hasNext();
}
if (this.iterator == null) {
throw new NoSuchElementException();
}
this.entry = this.iterator.next();
this.setIterator();
this.count++;
return this.entry;
}
@Override
public void close() {
if (this.iterator != null) this.iterator.close();
this.closeIterators();
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client;
import java.util.LinkedList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Queue;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
/**
* @author lynn.bond@hotmail.com created on 2021/10/21
* @version 0.1.0
*/
class TopWorkIteratorProxy implements HgKvIterator {
private static final byte[] EMPTY_BYTES = new byte[0];
private HgKvIterator<HgKvEntry> iterator;
private final Queue<HgKvIterator> queue;
private HgKvEntry entry;
private final long limit;
private int count;
TopWorkIteratorProxy(List<HgKvIterator> iterators, long limit) {
this.queue = new LinkedList<>(iterators);
this.limit = limit <= 0 ? Integer.MAX_VALUE : limit;
}
private HgKvIterator getIterator() {
if (this.queue.isEmpty()) return null;
HgKvIterator buf = null;
while ((buf = this.queue.poll()) != null) {
if (buf.hasNext()) {
break;
}
}
if (buf == null) {
return null;
}
this.queue.add(buf);
return buf;
}
private void closeIterators() {
if (this.queue.isEmpty()) return;
HgKvIterator buf;
while ((buf = this.queue.poll()) != null) {
buf.close();
}
}
private void setIterator() {
this.iterator = null;
}
@Override
public byte[] key() {
if (this.entry != null) {
return this.entry.key();
}
return null;
}
@Override
public byte[] value() {
if (this.entry != null) {
return this.entry.value();
}
return null;
}
@Override
public byte[] position() {
return this.iterator != null ? this.iterator.position() : EMPTY_BYTES;
}
@Override
public void seek(byte[] position) {
if (this.iterator != null) this.iterator.seek(position);
}
@Override
public boolean hasNext() {
if (this.count >= this.limit) {
return false;
}
if (this.iterator == null) {
this.iterator = this.getIterator();
}
return this.iterator != null;
}
@Override
public Object next() {
if (this.iterator == null) {
hasNext();
}
if (this.iterator == null) {
throw new NoSuchElementException();
}
this.entry = this.iterator.next();
this.setIterator();
this.count++;
return this.entry;
}
@Override
public void close() {
if (this.iterator != null) this.iterator.close();
this.closeIterators();
}
}

322
hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcKvIteratorImpl.java
View File

@ -1,161 +1,161 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.util.List;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgKvOrderedIterator;
import org.apache.hugegraph.store.HgKvPagingIterator;
import org.apache.hugegraph.store.HgPageSize;
import org.apache.hugegraph.store.HgSeekAble;
import org.apache.hugegraph.store.client.HgStoreNodeSession;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.store.client.util.HgStoreClientUtil;
import org.apache.hugegraph.store.grpc.common.Kv;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2021/10/20
* @version 0.2.1
*/
@Slf4j
class GrpcKvIteratorImpl implements HgKvPagingIterator<HgKvEntry>, HgKvOrderedIterator<HgKvEntry> {
private final byte[] emptyBytes = HgStoreClientConst.EMPTY_BYTES;
private final KvCloseableIterator<Kv> iterator;
private HgKvEntry element;
private final HgPageSize pageLimiter;
private final HgStoreNodeSession session;
public static HgKvIterator<HgKvEntry> of(HgStoreNodeSession nodeSession,
KvCloseableIterator<Kv> iterator) {
if (iterator instanceof HgPageSize) {
return of(nodeSession, iterator, (HgPageSize) iterator);
}
return new GrpcKvIteratorImpl(nodeSession, iterator, () -> 1);
}
public static HgKvIterator<HgKvEntry> of(HgStoreNodeSession nodeSession,
KvCloseableIterator<Kv> iterator,
HgPageSize pageLimiter) {
return new GrpcKvIteratorImpl(nodeSession, iterator, pageLimiter);
}
public static HgKvIterator<HgKvEntry> of(HgStoreNodeSession nodeSession, List<Kv> kvList) {
int pageSize = kvList.size();
return new GrpcKvIteratorImpl(nodeSession, new KvListIterator<Kv>(kvList), () -> pageSize);
}
private GrpcKvIteratorImpl(HgStoreNodeSession session, KvCloseableIterator<Kv> iterator,
HgPageSize pageLimiter) {
this.iterator = iterator;
this.pageLimiter = pageLimiter;
this.session = session;
}
@Override
public boolean hasNext() {
// if (log.isDebugEnabled()) {
// if (!this.iterator.hasNext() && !nodeSession.getGraphName().endsWith("/s")) {
// log.debug("[ANALYSIS GrpcKv hasNext-> FALSE] ");
// }
// }
return this.iterator.hasNext();
}
@Override
public HgKvEntry next() {
Kv kv = this.iterator.next();
this.element = new GrpcKvEntryImpl(kv.getKey().toByteArray(), kv.getValue().toByteArray(),
kv.getCode());
return this.element;
}
@Override
public byte[] key() {
if (this.element == null) {
return null;
}
return this.element.key();
}
@Override
public byte[] value() {
if (this.element == null) {
return null;
}
return this.element.value();
}
@Override
public byte[] position() {
if (this.element == null) {
return emptyBytes;
}
byte[] key = this.element.key();
if (key == null) {
return emptyBytes;
}
if (!(this.iterator instanceof HgSeekAble)) {
return emptyBytes;
}
byte[] upstream = ((HgSeekAble) this.iterator).position();
byte[] code = HgStoreClientUtil.toIntBytes(this.element.code());
byte[] result = new byte[upstream.length + Integer.BYTES + key.length];
System.arraycopy(upstream, 0, result, 0, upstream.length);
System.arraycopy(code, 0, result, upstream.length, Integer.BYTES);
System.arraycopy(key, 0, result, upstream.length + Integer.BYTES, key.length);
return result;
}
@Override
public void seek(byte[] position) {
if (this.iterator instanceof HgSeekAble) {
((HgSeekAble) this.iterator).seek(position);
}
}
@Override
public long getPageSize() {
return pageLimiter.getPageSize();
}
@Override
public boolean isPageEmpty() {
return !iterator.hasNext();
}
@Override
public int compareTo(HgKvOrderedIterator o) {
return Long.compare(this.getSequence(), o.getSequence());
}
@Override
public long getSequence() {
return this.session.getStoreNode().getNodeId().longValue();
}
@Override
public void close() {
this.iterator.close();
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.util.List;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgKvOrderedIterator;
import org.apache.hugegraph.store.HgKvPagingIterator;
import org.apache.hugegraph.store.HgPageSize;
import org.apache.hugegraph.store.HgSeekAble;
import org.apache.hugegraph.store.client.HgStoreNodeSession;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.store.client.util.HgStoreClientUtil;
import org.apache.hugegraph.store.grpc.common.Kv;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2021/10/20
* @version 0.2.1
*/
@Slf4j
class GrpcKvIteratorImpl implements HgKvPagingIterator<HgKvEntry>, HgKvOrderedIterator<HgKvEntry> {
private final byte[] emptyBytes = HgStoreClientConst.EMPTY_BYTES;
private final KvCloseableIterator<Kv> iterator;
private HgKvEntry element;
private final HgPageSize pageLimiter;
private final HgStoreNodeSession session;
public static HgKvIterator<HgKvEntry> of(HgStoreNodeSession nodeSession,
KvCloseableIterator<Kv> iterator) {
if (iterator instanceof HgPageSize) {
return of(nodeSession, iterator, (HgPageSize) iterator);
}
return new GrpcKvIteratorImpl(nodeSession, iterator, () -> 1);
}
public static HgKvIterator<HgKvEntry> of(HgStoreNodeSession nodeSession,
KvCloseableIterator<Kv> iterator,
HgPageSize pageLimiter) {
return new GrpcKvIteratorImpl(nodeSession, iterator, pageLimiter);
}
public static HgKvIterator<HgKvEntry> of(HgStoreNodeSession nodeSession, List<Kv> kvList) {
int pageSize = kvList.size();
return new GrpcKvIteratorImpl(nodeSession, new KvListIterator<Kv>(kvList), () -> pageSize);
}
private GrpcKvIteratorImpl(HgStoreNodeSession session, KvCloseableIterator<Kv> iterator,
HgPageSize pageLimiter) {
this.iterator = iterator;
this.pageLimiter = pageLimiter;
this.session = session;
}
@Override
public boolean hasNext() {
// if (log.isDebugEnabled()) {
// if (!this.iterator.hasNext() && !nodeSession.getGraphName().endsWith("/s")) {
// log.debug("[ANALYSIS GrpcKv hasNext-> FALSE] ");
// }
// }
return this.iterator.hasNext();
}
@Override
public HgKvEntry next() {
Kv kv = this.iterator.next();
this.element = new GrpcKvEntryImpl(kv.getKey().toByteArray(), kv.getValue().toByteArray(),
kv.getCode());
return this.element;
}
@Override
public byte[] key() {
if (this.element == null) {
return null;
}
return this.element.key();
}
@Override
public byte[] value() {
if (this.element == null) {
return null;
}
return this.element.value();
}
@Override
public byte[] position() {
if (this.element == null) {
return emptyBytes;
}
byte[] key = this.element.key();
if (key == null) {
return emptyBytes;
}
if (!(this.iterator instanceof HgSeekAble)) {
return emptyBytes;
}
byte[] upstream = ((HgSeekAble) this.iterator).position();
byte[] code = HgStoreClientUtil.toIntBytes(this.element.code());
byte[] result = new byte[upstream.length + Integer.BYTES + key.length];
System.arraycopy(upstream, 0, result, 0, upstream.length);
System.arraycopy(code, 0, result, upstream.length, Integer.BYTES);
System.arraycopy(key, 0, result, upstream.length + Integer.BYTES, key.length);
return result;
}
@Override
public void seek(byte[] position) {
if (this.iterator instanceof HgSeekAble) {
((HgSeekAble) this.iterator).seek(position);
}
}
@Override
public long getPageSize() {
return pageLimiter.getPageSize();
}
@Override
public boolean isPageEmpty() {
return !iterator.hasNext();
}
@Override
public int compareTo(HgKvOrderedIterator o) {
return Long.compare(this.getSequence(), o.getSequence());
}
@Override
public long getSequence() {
return this.session.getStoreNode().getNodeId().longValue();
}
@Override
public void close() {
this.iterator.close();
}
}

436
hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreStreamClient.java
View File

@ -1,218 +1,218 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import javax.annotation.concurrent.ThreadSafe;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgScanQuery;
import org.apache.hugegraph.store.client.HgStoreNodeSession;
import org.apache.hugegraph.store.grpc.common.Kv;
import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc;
import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamBlockingStub;
import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamStub;
import io.grpc.ManagedChannel;
import io.grpc.stub.AbstractAsyncStub;
import io.grpc.stub.AbstractBlockingStub;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2021/10/19
* @version 1.1.1 added synchronized in getChannel.
*/
@Slf4j
@ThreadSafe
public class GrpcStoreStreamClient extends AbstractGrpcClient {
public HgStoreStreamStub getStub(HgStoreNodeSession nodeSession) {
return (HgStoreStreamStub) getAsyncStub(nodeSession.getStoreNode().getAddress());
}
@Override
public AbstractAsyncStub getAsyncStub(ManagedChannel channel) {
return HgStoreStreamGrpc.newStub(channel);
}
private HgStoreStreamBlockingStub getBlockingStub(HgStoreNodeSession nodeSession) {
return (HgStoreStreamBlockingStub) getBlockingStub(nodeSession.getStoreNode().getAddress());
}
@Override
public AbstractBlockingStub getBlockingStub(ManagedChannel channel) {
return HgStoreStreamGrpc.newBlockingStub(channel);
}
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table, long limit,
byte[] query) {
return KvOneShotScanner.scanAll(nodeSession
, this.getBlockingStub(nodeSession)
, table
, limit
, query
);
}
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table,
long limit) {
return KvOneShotScanner.scanAll(nodeSession
, this.getBlockingStub(nodeSession)
, table
, limit
, null
);
}
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table,
HgOwnerKey prefix, long limit) {
return KvOneShotScanner.scanPrefix(nodeSession
, this.getBlockingStub(nodeSession)
, table
, prefix
, limit
, null
);
}
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table,
HgOwnerKey prefix, long limit,
byte[] query) {
return KvOneShotScanner.scanPrefix(nodeSession
, this.getBlockingStub(nodeSession)
, table
, prefix
, limit
, query
);
}
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table,
HgOwnerKey startKey,
HgOwnerKey endKey
, long limit
, int scanType
, byte[] query) {
return KvOneShotScanner.scanRange(nodeSession
, this.getBlockingStub(nodeSession)
, table
, startKey
, endKey
, limit
, scanType
, query
);
}
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
, String table
, long limit
, byte[] query) {
return KvPageScanner.scanAll(nodeSession
, this.getStub(nodeSession)
, table
, limit
, query
);
}
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
, String table
, long limit) {
return KvPageScanner.scanAll(nodeSession
, this.getStub(nodeSession)
, table
, limit
, null
);
}
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
, String table
, HgOwnerKey prefix
, long limit) {
return KvPageScanner.scanPrefix(nodeSession
, this.getStub(nodeSession)
, table
, prefix
, limit
, null
);
}
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
, String table
, HgOwnerKey prefix
, long limit
, byte[] query) {
return KvPageScanner.scanPrefix(nodeSession
, this.getStub(nodeSession)
, table
, prefix
, limit
, query
);
}
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
, String table
, HgOwnerKey startKey
, HgOwnerKey endKey
, long limit
, int scanType
, byte[] query) {
return KvPageScanner.scanRange(nodeSession
, this.getStub(nodeSession)
, table
, startKey
, endKey
, limit
, scanType
, query
);
}
KvCloseableIterator<Kv> doBatchScan(HgStoreNodeSession nodeSession, HgScanQuery scanQuery) {
return KvBatchScanner5.scan(nodeSession, this.getStub(nodeSession), scanQuery);
}
// 返回多个小的迭代器，允许上层并行处理
KvCloseableIterator<HgKvIterator<HgKvEntry>> doBatchScan3(HgStoreNodeSession nodeSession,
HgScanQuery scanQuery,
KvCloseableIterator iterator) {
KvBatchScanner.scan(this.getStub(nodeSession), nodeSession.getGraphName(), scanQuery,
iterator);
return iterator;
}
KvCloseableIterator<Kv> doBatchScanOneShot(HgStoreNodeSession nodeSession,
HgScanQuery scanQuery) {
return KvBatchOneShotScanner.scan(nodeSession, this.getBlockingStub(nodeSession),
scanQuery);
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import javax.annotation.concurrent.ThreadSafe;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgKvIterator;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgScanQuery;
import org.apache.hugegraph.store.client.HgStoreNodeSession;
import org.apache.hugegraph.store.grpc.common.Kv;
import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc;
import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamBlockingStub;
import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamStub;
import io.grpc.ManagedChannel;
import io.grpc.stub.AbstractAsyncStub;
import io.grpc.stub.AbstractBlockingStub;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2021/10/19
* @version 1.1.1 added synchronized in getChannel.
*/
@Slf4j
@ThreadSafe
public class GrpcStoreStreamClient extends AbstractGrpcClient {
public HgStoreStreamStub getStub(HgStoreNodeSession nodeSession) {
return (HgStoreStreamStub) getAsyncStub(nodeSession.getStoreNode().getAddress());
}
@Override
public AbstractAsyncStub getAsyncStub(ManagedChannel channel) {
return HgStoreStreamGrpc.newStub(channel);
}
private HgStoreStreamBlockingStub getBlockingStub(HgStoreNodeSession nodeSession) {
return (HgStoreStreamBlockingStub) getBlockingStub(nodeSession.getStoreNode().getAddress());
}
@Override
public AbstractBlockingStub getBlockingStub(ManagedChannel channel) {
return HgStoreStreamGrpc.newBlockingStub(channel);
}
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table, long limit,
byte[] query) {
return KvOneShotScanner.scanAll(nodeSession
, this.getBlockingStub(nodeSession)
, table
, limit
, query
);
}
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table,
long limit) {
return KvOneShotScanner.scanAll(nodeSession
, this.getBlockingStub(nodeSession)
, table
, limit
, null
);
}
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table,
HgOwnerKey prefix, long limit) {
return KvOneShotScanner.scanPrefix(nodeSession
, this.getBlockingStub(nodeSession)
, table
, prefix
, limit
, null
);
}
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table,
HgOwnerKey prefix, long limit,
byte[] query) {
return KvOneShotScanner.scanPrefix(nodeSession
, this.getBlockingStub(nodeSession)
, table
, prefix
, limit
, query
);
}
KvCloseableIterator<Kv> doScanOneShot(HgStoreNodeSession nodeSession, String table,
HgOwnerKey startKey,
HgOwnerKey endKey
, long limit
, int scanType
, byte[] query) {
return KvOneShotScanner.scanRange(nodeSession
, this.getBlockingStub(nodeSession)
, table
, startKey
, endKey
, limit
, scanType
, query
);
}
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
, String table
, long limit
, byte[] query) {
return KvPageScanner.scanAll(nodeSession
, this.getStub(nodeSession)
, table
, limit
, query
);
}
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
, String table
, long limit) {
return KvPageScanner.scanAll(nodeSession
, this.getStub(nodeSession)
, table
, limit
, null
);
}
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
, String table
, HgOwnerKey prefix
, long limit) {
return KvPageScanner.scanPrefix(nodeSession
, this.getStub(nodeSession)
, table
, prefix
, limit
, null
);
}
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
, String table
, HgOwnerKey prefix
, long limit
, byte[] query) {
return KvPageScanner.scanPrefix(nodeSession
, this.getStub(nodeSession)
, table
, prefix
, limit
, query
);
}
KvCloseableIterator<Kv> doScan(HgStoreNodeSession nodeSession
, String table
, HgOwnerKey startKey
, HgOwnerKey endKey
, long limit
, int scanType
, byte[] query) {
return KvPageScanner.scanRange(nodeSession
, this.getStub(nodeSession)
, table
, startKey
, endKey
, limit
, scanType
, query
);
}
KvCloseableIterator<Kv> doBatchScan(HgStoreNodeSession nodeSession, HgScanQuery scanQuery) {
return KvBatchScanner5.scan(nodeSession, this.getStub(nodeSession), scanQuery);
}
// 返回多个小的迭代器，允许上层并行处理
KvCloseableIterator<HgKvIterator<HgKvEntry>> doBatchScan3(HgStoreNodeSession nodeSession,
HgScanQuery scanQuery,
KvCloseableIterator iterator) {
KvBatchScanner.scan(this.getStub(nodeSession), nodeSession.getGraphName(), scanQuery,
iterator);
return iterator;
}
KvCloseableIterator<Kv> doBatchScanOneShot(HgStoreNodeSession nodeSession,
HgScanQuery scanQuery) {
return KvBatchOneShotScanner.scan(nodeSession, this.getBlockingStub(nodeSession),
scanQuery);
}
}

298
hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcUtil.java
View File

@ -1,149 +1,149 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.client.HgStoreNodeSession;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.store.grpc.common.Header;
import org.apache.hugegraph.store.grpc.common.Key;
import org.apache.hugegraph.store.grpc.common.Kv;
import org.apache.hugegraph.store.grpc.common.Tk;
import org.apache.hugegraph.store.grpc.common.Tkv;
import org.apache.hugegraph.store.grpc.common.Tp;
import org.apache.hugegraph.store.grpc.common.Tse;
import com.google.protobuf.ByteString;
import io.grpc.Status;
import io.grpc.StatusRuntimeException;
/**
* @author lynn.bond@hotmail.com on 2022/1/19
*/
final class GrpcUtil {
private static final ThreadLocal<Key.Builder> keyBuilder = new ThreadLocal<Key.Builder>();
static Header getHeader(HgStoreNodeSession nodeSession) {
return Header.newBuilder()
.setGraph(nodeSession.getGraphName())
.build();
}
static Tk toTk(String table, HgOwnerKey ownerKey) {
return Tk.newBuilder()
.setTable(table)
.setKey(ByteString.copyFrom(ownerKey.getKey()))
.setCode(ownerKey.getKeyCode())
.build();
}
static Key.Builder getOwnerKeyBuilder() {
Key.Builder builder = keyBuilder.get();
if (builder == null) {
builder = Key.newBuilder();
// TODO 线程级变量，寻找删除时机
keyBuilder.set(builder);
}
return builder;
}
static Key toKey(HgOwnerKey ownerKey, Key.Builder builder) {
if (ownerKey == null) {
return null;
}
return builder
.setKey(ByteString.copyFrom(ownerKey.getKey()))
.setCode(ownerKey.getKeyCode())
.build();
}
static Key toKey(HgOwnerKey ownerKey) {
if (ownerKey == null) {
return null;
}
Key.Builder builder = keyBuilder.get();
if (builder == null) {
builder = Key.newBuilder();
// TODO 线程级变量，寻找删除时机
keyBuilder.set(builder);
}
return builder
.setKey(ByteString.copyFrom(ownerKey.getKey()))
.setCode(ownerKey.getKeyCode())
.build();
}
static Tkv toTkv(String table, HgOwnerKey ownerKey, byte[] value) {
return Tkv.newBuilder()
.setTable(table)
.setKey(ByteString.copyFrom(ownerKey.getKey()))
.setValue(ByteString.copyFrom(value))
.setCode(ownerKey.getKeyCode())
.build();
}
static Tp toTp(String table, HgOwnerKey ownerKey) {
return Tp.newBuilder()
.setTable(table)
.setPrefix(ByteString.copyFrom(ownerKey.getKey()))
.setCode(ownerKey.getKeyCode())
.build();
}
static Tse toTse(String table, HgOwnerKey startKey, HgOwnerKey endKey) {
return Tse.newBuilder()
.setTable(table)
.setStart(toKey(startKey))
.setEnd(toKey(endKey))
.build();
}
static List<HgKvEntry> toList(List<Kv> kvList) {
if (kvList == null || kvList.isEmpty()) {
return HgStoreClientConst.EMPTY_LIST;
}
Iterator<Kv> iter = kvList.iterator();
List<HgKvEntry> resList = new ArrayList<>(kvList.size());
while (iter.hasNext()) {
Kv entry = iter.next();
resList.add(new GrpcKvEntryImpl(entry.getKey().toByteArray(),
entry.getValue().toByteArray(), entry.getCode()));
}
return resList;
}
static StatusRuntimeException toErr(String msg) {
return new StatusRuntimeException(Status.UNKNOWN.withDescription(msg));
}
static ByteString toBs(byte[] bytes) {
return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES);
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.hugegraph.store.HgKvEntry;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.client.HgStoreNodeSession;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.store.grpc.common.Header;
import org.apache.hugegraph.store.grpc.common.Key;
import org.apache.hugegraph.store.grpc.common.Kv;
import org.apache.hugegraph.store.grpc.common.Tk;
import org.apache.hugegraph.store.grpc.common.Tkv;
import org.apache.hugegraph.store.grpc.common.Tp;
import org.apache.hugegraph.store.grpc.common.Tse;
import com.google.protobuf.ByteString;
import io.grpc.Status;
import io.grpc.StatusRuntimeException;
/**
* @author lynn.bond@hotmail.com on 2022/1/19
*/
final class GrpcUtil {
private static final ThreadLocal<Key.Builder> keyBuilder = new ThreadLocal<Key.Builder>();
static Header getHeader(HgStoreNodeSession nodeSession) {
return Header.newBuilder()
.setGraph(nodeSession.getGraphName())
.build();
}
static Tk toTk(String table, HgOwnerKey ownerKey) {
return Tk.newBuilder()
.setTable(table)
.setKey(ByteString.copyFrom(ownerKey.getKey()))
.setCode(ownerKey.getKeyCode())
.build();
}
static Key.Builder getOwnerKeyBuilder() {
Key.Builder builder = keyBuilder.get();
if (builder == null) {
builder = Key.newBuilder();
// TODO 线程级变量，寻找删除时机
keyBuilder.set(builder);
}
return builder;
}
static Key toKey(HgOwnerKey ownerKey, Key.Builder builder) {
if (ownerKey == null) {
return null;
}
return builder
.setKey(ByteString.copyFrom(ownerKey.getKey()))
.setCode(ownerKey.getKeyCode())
.build();
}
static Key toKey(HgOwnerKey ownerKey) {
if (ownerKey == null) {
return null;
}
Key.Builder builder = keyBuilder.get();
if (builder == null) {
builder = Key.newBuilder();
// TODO 线程级变量，寻找删除时机
keyBuilder.set(builder);
}
return builder
.setKey(ByteString.copyFrom(ownerKey.getKey()))
.setCode(ownerKey.getKeyCode())
.build();
}
static Tkv toTkv(String table, HgOwnerKey ownerKey, byte[] value) {
return Tkv.newBuilder()
.setTable(table)
.setKey(ByteString.copyFrom(ownerKey.getKey()))
.setValue(ByteString.copyFrom(value))
.setCode(ownerKey.getKeyCode())
.build();
}
static Tp toTp(String table, HgOwnerKey ownerKey) {
return Tp.newBuilder()
.setTable(table)
.setPrefix(ByteString.copyFrom(ownerKey.getKey()))
.setCode(ownerKey.getKeyCode())
.build();
}
static Tse toTse(String table, HgOwnerKey startKey, HgOwnerKey endKey) {
return Tse.newBuilder()
.setTable(table)
.setStart(toKey(startKey))
.setEnd(toKey(endKey))
.build();
}
static List<HgKvEntry> toList(List<Kv> kvList) {
if (kvList == null || kvList.isEmpty()) {
return HgStoreClientConst.EMPTY_LIST;
}
Iterator<Kv> iter = kvList.iterator();
List<HgKvEntry> resList = new ArrayList<>(kvList.size());
while (iter.hasNext()) {
Kv entry = iter.next();
resList.add(new GrpcKvEntryImpl(entry.getKey().toByteArray(),
entry.getValue().toByteArray(), entry.getCode()));
}
return resList;
}
static StatusRuntimeException toErr(String msg) {
return new StatusRuntimeException(Status.UNKNOWN.withDescription(msg));
}
static ByteString toBs(byte[] bytes) {
return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES);
}
}

244
hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchOneShotScanner.java
View File

@ -1,123 +1,123 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.EMPTY_POSITION;
import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.createQueryReq;
import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.getHeader;
import java.util.Iterator;
import java.util.List;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.store.HgPageSize;
import org.apache.hugegraph.store.HgScanQuery;
import org.apache.hugegraph.store.HgSeekAble;
import org.apache.hugegraph.store.client.HgStoreNodeSession;
import org.apache.hugegraph.store.grpc.common.Kv;
import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc;
import org.apache.hugegraph.store.grpc.stream.ScanStreamBatchReq;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2022/04/08
*/
@Slf4j
@NotThreadSafe
class KvBatchOneShotScanner implements KvCloseableIterator<Kv>, HgPageSize, HgSeekAble {
private final HgStoreNodeSession nodeSession;
private final HgStoreStreamGrpc.HgStoreStreamBlockingStub stub;
private final HgScanQuery scanQuery;
private Iterator<Kv> iterator;
private List<Kv> list = null;
public static KvCloseableIterator scan(HgStoreNodeSession nodeSession,
HgStoreStreamGrpc.HgStoreStreamBlockingStub stub,
HgScanQuery scanQuery) {
return new KvBatchOneShotScanner(nodeSession, stub, scanQuery);
}
private KvBatchOneShotScanner(HgStoreNodeSession nodeSession,
HgStoreStreamGrpc.HgStoreStreamBlockingStub stub,
HgScanQuery scanQuery) {
this.nodeSession = nodeSession;
this.stub = stub;
this.scanQuery = scanQuery;
}
private ScanStreamBatchReq createReq() {
return ScanStreamBatchReq.newBuilder()
.setHeader(getHeader(this.nodeSession))
.setQueryRequest(createQueryReq(this.scanQuery, Integer.MAX_VALUE))
.build();
}
private Iterator<Kv> createIterator() {
this.list = this.stub.scanBatchOneShot(this.createReq()).getDataList();
return this.list.iterator();
}
/*** Iterator ***/
@Override
public boolean hasNext() {
if (this.iterator == null) {
this.iterator = this.createIterator();
}
return this.iterator.hasNext();
}
@Override
public Kv next() {
if (this.iterator == null) {
this.iterator = this.createIterator();
}
return this.iterator.next();
}
@Override
public long getPageSize() {
return Integer.MAX_VALUE;
}
@Override
public boolean isPageEmpty() {
return !this.iterator.hasNext();
}
@Override
public byte[] position() {
//TODO: to implement
return EMPTY_POSITION;
}
@Override
public void seek(byte[] position) {
//TODO: to implement
}
@Override
public void close() {
//Nothing to do
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.EMPTY_POSITION;
import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.createQueryReq;
import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.getHeader;
import java.util.Iterator;
import java.util.List;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.store.HgPageSize;
import org.apache.hugegraph.store.HgScanQuery;
import org.apache.hugegraph.store.HgSeekAble;
import org.apache.hugegraph.store.client.HgStoreNodeSession;
import org.apache.hugegraph.store.grpc.common.Kv;
import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc;
import org.apache.hugegraph.store.grpc.stream.ScanStreamBatchReq;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2022/04/08
*/
@Slf4j
@NotThreadSafe
class KvBatchOneShotScanner implements KvCloseableIterator<Kv>, HgPageSize, HgSeekAble {
private final HgStoreNodeSession nodeSession;
private final HgStoreStreamGrpc.HgStoreStreamBlockingStub stub;
private final HgScanQuery scanQuery;
private Iterator<Kv> iterator;
private List<Kv> list = null;
public static KvCloseableIterator scan(HgStoreNodeSession nodeSession,
HgStoreStreamGrpc.HgStoreStreamBlockingStub stub,
HgScanQuery scanQuery) {
return new KvBatchOneShotScanner(nodeSession, stub, scanQuery);
}
private KvBatchOneShotScanner(HgStoreNodeSession nodeSession,
HgStoreStreamGrpc.HgStoreStreamBlockingStub stub,
HgScanQuery scanQuery) {
this.nodeSession = nodeSession;
this.stub = stub;
this.scanQuery = scanQuery;
}
private ScanStreamBatchReq createReq() {
return ScanStreamBatchReq.newBuilder()
.setHeader(getHeader(this.nodeSession))
.setQueryRequest(createQueryReq(this.scanQuery, Integer.MAX_VALUE))
.build();
}
private Iterator<Kv> createIterator() {
this.list = this.stub.scanBatchOneShot(this.createReq()).getDataList();
return this.list.iterator();
}
/*** Iterator ***/
@Override
public boolean hasNext() {
if (this.iterator == null) {
this.iterator = this.createIterator();
}
return this.iterator.hasNext();
}
@Override
public Kv next() {
if (this.iterator == null) {
this.iterator = this.createIterator();
}
return this.iterator.next();
}
@Override
public long getPageSize() {
return Integer.MAX_VALUE;
}
@Override
public boolean isPageEmpty() {
return !this.iterator.hasNext();
}
@Override
public byte[] position() {
//TODO: to implement
return EMPTY_POSITION;
}
@Override
public void seek(byte[] position) {
//TODO: to implement
}
@Override
public void close() {
//Nothing to do
}
}

892
hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchScanner5.java
View File

@ -1,447 +1,447 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Supplier;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.store.HgPageSize;
import org.apache.hugegraph.store.HgScanQuery;
import org.apache.hugegraph.store.client.HgStoreNodeSession;
import org.apache.hugegraph.store.client.type.HgStoreClientException;
import org.apache.hugegraph.store.client.util.Base58;
import org.apache.hugegraph.store.client.util.HgStoreClientConfig;
import org.apache.hugegraph.store.grpc.common.Kv;
import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2022/03/23
* @version 5.0.0
*/
@Slf4j
@NotThreadSafe
class KvBatchScanner5 {
private final static HgStoreClientConfig storeClientConfig = HgStoreClientConfig.of();
//private final static int HAVE_NEXT_TIMEOUT_SECONDS = storeClientConfig
// .getNetKvScannerHaveNextTimeout();
private final static int HAVE_NEXT_TIMEOUT_SECONDS = 60;
private final static long PAGE_SIZE = storeClientConfig.getNetKvScannerPageSize();
private enum OrderState {
NEW(0),
WORKING(1),
COMPLETED(10);
int value;
OrderState(int value) {
this.value = value;
}
}
public static KvCloseableIterator scan(HgStoreNodeSession nodeSession,
HgStoreStreamGrpc.HgStoreStreamStub stub,
HgScanQuery scanQuery) {
return new OrderConsumer(new OrderBroker(stub, scanQuery, nodeSession));
}
/*** Broker ***/
private static class OrderBroker {
private final HgScanQuery scanQuery;
private final StreamObserver<ScanStreamBatchReq> requestObserver;
private final ScanStreamBatchReq.Builder reqBuilder;
private final ReentrantLock senderLock = new ReentrantLock();
private final AtomicBoolean serverFinished = new AtomicBoolean();
private final AtomicBoolean clientFinished = new AtomicBoolean();
private final ScanReceiptRequest.Builder receiptReqBuilder =
ScanReceiptRequest.newBuilder();
private final ScanCancelRequest cancelReq = ScanCancelRequest.newBuilder().build();
private final HgStoreNodeSession nodeSession;
private final OrderAgent agent;
public final OrderKeeper keeper = new OrderKeeper();
private OrderState state = OrderState.NEW;
private final AtomicLong receivedCount = new AtomicLong();
private final AtomicInteger receivedLastTimes = new AtomicInteger();
private final BlockingQueue<Integer> timesQueue = new LinkedBlockingQueue();
String brokerId = "";
OrderBroker(HgStoreStreamGrpc.HgStoreStreamStub stub,
HgScanQuery scanQuery,
HgStoreNodeSession nodeSession) {
if (log.isDebugEnabled()) {
if (scanQuery.getPrefixList() != null && scanQuery.getPrefixList().size() > 0) {
brokerId = Base58.encode(scanQuery.getPrefixList().get(0).getKey());
log.debug(
"[ANALYSIS START] [{}] firstKey: {}, keyLength: {}, table: {}, node: {}"
, brokerId
, scanQuery.getPrefixList().get(0)
, scanQuery.getPrefixList().size()
, scanQuery.getTable()
, nodeSession.getStoreNode().getAddress());
}
}
this.scanQuery = scanQuery;
this.reqBuilder = KvBatchUtil.getRequestBuilder(nodeSession);
this.nodeSession = nodeSession;
this.agent = new OrderAgent(brokerId);
this.requestObserver = stub.scanBatch(agent);
}
List<Kv> oneMore() {
if (this.state == OrderState.NEW) {
synchronized (this.state) {
if (this.state == OrderState.NEW) {
this.makeADeal();
this.state = OrderState.WORKING;
}
}
} else {
this.sendReceipt();
}
return this.keeper.pickUp();
}
void receipt(int times) {
this.timesQueue.offer(times);
receivedLastTimes.set(times);
}
void sendReceipt() {
Integer buf = this.timesQueue.poll();
if (buf == null) {
buf = this.receivedLastTimes.get();
}
AtomicInteger timesBuf = new AtomicInteger(buf);
if (!this.clientFinished.get()) {
this.send(() ->
getReqBuilder().setReceiptRequest(
this.receiptReqBuilder.setTimes(timesBuf.get()).build())
.build()
);
}
}
private void makeADeal() {
this.send(() -> getReqBuilder()
.setQueryRequest(KvBatchUtil.createQueryReq(scanQuery, PAGE_SIZE)).build()
);
}
private void finish(long tookAmt) {
this.clientFinished.set(true);
if (log.isDebugEnabled()) {
log.debug("[ANALYSIS END] [{}] times: {}, received: {}, took: {}"
, this.brokerId
, this.receivedLastTimes.get()
, this.receivedCount.get()
, tookAmt
);
}
if (this.receivedCount.get() != tookAmt) {
if (log.isDebugEnabled()) {
log.debug("[ANALYSIS END] [{}] times: {}, received: {}, took: {}"
, this.brokerId
, this.receivedLastTimes.get()
, this.receivedCount.get()
, tookAmt
);
}
}
synchronized (this.state) {
if (this.state.value < OrderState.COMPLETED.value) {
this.send(() -> getReqBuilder().setCancelRequest(this.cancelReq).build());
this.state = OrderState.COMPLETED;
}
}
}
private ScanStreamBatchReq.Builder getReqBuilder() {
return this.reqBuilder.clearQueryRequest();
}
private void send(Supplier<ScanStreamBatchReq> supplier) {
this.senderLock.lock();
try {
if (!this.serverFinished.get()) {
this.requestObserver.onNext(supplier.get());
}
Thread.yield();
} finally {
this.senderLock.unlock();
}
}
private class OrderAgent implements StreamObserver<KvPageRes> {
private final AtomicInteger count = new AtomicInteger(0);
private final AtomicBoolean over = new AtomicBoolean(false);
private final String agentId;
OrderAgent(String agentId) {
this.agentId = agentId;
}
@Override
public void onNext(KvPageRes value) {
if (log.isDebugEnabled()) {
log.debug("Scan [ {} ] [ {} ] times, received: [ {} ]"
, nodeSession.getStoreNode().getAddress(), value.getTimes(),
value.getDataList().size());
}
serverFinished.set(value.getOver());
List<Kv> buffer = value.getDataList();
count.addAndGet(buffer.size());
if (log.isDebugEnabled()) {
if (value.getOver()) {
log.debug("[ANALYSIS OVER] [{}] count: {}", agentId, count);
}
}
keeper.receive(buffer, value.getTimes());
this.over.set(value.getOver());
this.checkOver(value.getTimes());
}
private void checkOver(int times) {
if (this.over.get()) {
requestObserver.onCompleted();
keeper.done(times);
}
}
@Override
public void onError(Throwable t) {
log.error("received server onError event, Throwable:", t);
keeper.shout(t);
}
@Override
public void onCompleted() {
if (log.isDebugEnabled()) log.debug("received sever completed event.");
serverFinished.set(true);
}
}
/*** Inventory Keeper ***/
private class OrderKeeper {
private final BlockingQueue<Supplier<List<Kv>>> queue = new LinkedBlockingQueue<>();
private final ReentrantLock pickUpLock = new ReentrantLock();
private final AtomicBoolean done = new AtomicBoolean();
private final AtomicBoolean stop = new AtomicBoolean();
private int timesOfOver;
private int lastTimes;
private Throwable serverErr;
void receive(List<Kv> data, int times) {
receivedCount.addAndGet(data.size());
this.queue.offer(() -> data);
receipt(times);
this.lastTimes = times;
}
private List<Kv> pickUp() {
Supplier<List<Kv>> res;
pickUpLock.lock();
try {
if (this.done.get()) {
if (this.stop.get()) {
log.warn("Invoking pickUp method after OrderKeeper has bean closing.");
}
res = this.queue.poll();
if (res == null) {
res = () -> null;
}
} else {
res = this.queue.poll(HAVE_NEXT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
if (res == null) {
if (this.done.get()) {
res = () -> null;
} else {
throw HgStoreClientException.of(
"Timeout, max time: " + HAVE_NEXT_TIMEOUT_SECONDS +
" seconds" +
", isOver: " + this.done.get() +
", isStop: " + this.stop.get() +
", last-times: " + this.lastTimes +
", over-times: " + this.timesOfOver);
}
}
}
} catch (InterruptedException e) {
log.error(
"Failed to receive List<Kv> from queue because of interruption of " +
"current thread ["
+ Thread.currentThread().getName() + "]");
Thread.currentThread().interrupt();
throw HgStoreClientException.of(
"Failed to receive List<Kv> from queue, cause by:", e);
} finally {
pickUpLock.unlock();
}
checkServerErr();
return res.get();
}
void done(int times) {
this.timesOfOver = times;
this.done.set(true);
this.queue.offer(() -> null);
}
void shout(Throwable t) {
this.serverErr = t;
log.error("Failed to receive from sever", t);
this.queue.offer(() -> null);
}
private void checkServerErr() {
if (this.serverErr != null) {
throw HgStoreClientException.of(this.serverErr);
}
}
}
}
/* iterator */
private static class OrderConsumer implements KvCloseableIterator<Kv>, HgPageSize {
private final OrderBroker broker;
private Iterator<Kv> dataIterator;
private long tookCount = 0;
private final String consumerId;
OrderConsumer(OrderBroker broker) {
this.broker = broker;
consumerId = broker.brokerId;
}
private Iterator<Kv> getIterator() {
List<Kv> list = this.broker.oneMore();
if (log.isDebugEnabled()) {
if (list != null && list.isEmpty()) {
log.debug("[ANALYSIS EMPTY] [{}] , tookCount: {}", consumerId, tookCount);
}
}
if (list == null || list.isEmpty()) {
return null;
} else {
return list.iterator();
}
}
@Override
public void close() {
this.broker.finish(this.tookCount);
}
@Override
public long getPageSize() {
return PAGE_SIZE;
}
@Override
public boolean hasNext() {
if (this.dataIterator == null) {
this.dataIterator = this.getIterator();
} else {
if (this.dataIterator.hasNext()) {
return true;
} else {
this.dataIterator = this.getIterator();
}
}
if (this.dataIterator == null) {
if (log.isDebugEnabled()) {
log.debug("[ANALYSIS NULL -> FALSE] [{}] , tookCount: {}", consumerId,
tookCount);
}
return false;
} else {
if (log.isDebugEnabled()) {
if (!this.dataIterator.hasNext()) {
log.debug("[ANALYSIS hasNext -> FALSE] [{}] , tookCount: {}", consumerId,
tookCount);
}
}
return this.dataIterator.hasNext();
}
}
@Override
public Kv next() {
if (this.dataIterator == null) {
if (!this.hasNext()) {
throw new NoSuchElementException();
}
}
if (log.isDebugEnabled()) {
tookCount++;
if (tookCount % 10000 == 0) {
log.debug("[ANALYSIS NEXT] [{}] , tookCount: {}", consumerId, tookCount);
}
}
return this.dataIterator.next();
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Supplier;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.store.HgPageSize;
import org.apache.hugegraph.store.HgScanQuery;
import org.apache.hugegraph.store.client.HgStoreNodeSession;
import org.apache.hugegraph.store.client.type.HgStoreClientException;
import org.apache.hugegraph.store.client.util.Base58;
import org.apache.hugegraph.store.client.util.HgStoreClientConfig;
import org.apache.hugegraph.store.grpc.common.Kv;
import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2022/03/23
* @version 5.0.0
*/
@Slf4j
@NotThreadSafe
class KvBatchScanner5 {
private final static HgStoreClientConfig storeClientConfig = HgStoreClientConfig.of();
//private final static int HAVE_NEXT_TIMEOUT_SECONDS = storeClientConfig
// .getNetKvScannerHaveNextTimeout();
private final static int HAVE_NEXT_TIMEOUT_SECONDS = 60;
private final static long PAGE_SIZE = storeClientConfig.getNetKvScannerPageSize();
private enum OrderState {
NEW(0),
WORKING(1),
COMPLETED(10);
int value;
OrderState(int value) {
this.value = value;
}
}
public static KvCloseableIterator scan(HgStoreNodeSession nodeSession,
HgStoreStreamGrpc.HgStoreStreamStub stub,
HgScanQuery scanQuery) {
return new OrderConsumer(new OrderBroker(stub, scanQuery, nodeSession));
}
/*** Broker ***/
private static class OrderBroker {
private final HgScanQuery scanQuery;
private final StreamObserver<ScanStreamBatchReq> requestObserver;
private final ScanStreamBatchReq.Builder reqBuilder;
private final ReentrantLock senderLock = new ReentrantLock();
private final AtomicBoolean serverFinished = new AtomicBoolean();
private final AtomicBoolean clientFinished = new AtomicBoolean();
private final ScanReceiptRequest.Builder receiptReqBuilder =
ScanReceiptRequest.newBuilder();
private final ScanCancelRequest cancelReq = ScanCancelRequest.newBuilder().build();
private final HgStoreNodeSession nodeSession;
private final OrderAgent agent;
public final OrderKeeper keeper = new OrderKeeper();
private OrderState state = OrderState.NEW;
private final AtomicLong receivedCount = new AtomicLong();
private final AtomicInteger receivedLastTimes = new AtomicInteger();
private final BlockingQueue<Integer> timesQueue = new LinkedBlockingQueue();
String brokerId = "";
OrderBroker(HgStoreStreamGrpc.HgStoreStreamStub stub,
HgScanQuery scanQuery,
HgStoreNodeSession nodeSession) {
if (log.isDebugEnabled()) {
if (scanQuery.getPrefixList() != null && scanQuery.getPrefixList().size() > 0) {
brokerId = Base58.encode(scanQuery.getPrefixList().get(0).getKey());
log.debug(
"[ANALYSIS START] [{}] firstKey: {}, keyLength: {}, table: {}, node: {}"
, brokerId
, scanQuery.getPrefixList().get(0)
, scanQuery.getPrefixList().size()
, scanQuery.getTable()
, nodeSession.getStoreNode().getAddress());
}
}
this.scanQuery = scanQuery;
this.reqBuilder = KvBatchUtil.getRequestBuilder(nodeSession);
this.nodeSession = nodeSession;
this.agent = new OrderAgent(brokerId);
this.requestObserver = stub.scanBatch(agent);
}
List<Kv> oneMore() {
if (this.state == OrderState.NEW) {
synchronized (this.state) {
if (this.state == OrderState.NEW) {
this.makeADeal();
this.state = OrderState.WORKING;
}
}
} else {
this.sendReceipt();
}
return this.keeper.pickUp();
}
void receipt(int times) {
this.timesQueue.offer(times);
receivedLastTimes.set(times);
}
void sendReceipt() {
Integer buf = this.timesQueue.poll();
if (buf == null) {
buf = this.receivedLastTimes.get();
}
AtomicInteger timesBuf = new AtomicInteger(buf);
if (!this.clientFinished.get()) {
this.send(() ->
getReqBuilder().setReceiptRequest(
this.receiptReqBuilder.setTimes(timesBuf.get()).build())
.build()
);
}
}
private void makeADeal() {
this.send(() -> getReqBuilder()
.setQueryRequest(KvBatchUtil.createQueryReq(scanQuery, PAGE_SIZE)).build()
);
}
private void finish(long tookAmt) {
this.clientFinished.set(true);
if (log.isDebugEnabled()) {
log.debug("[ANALYSIS END] [{}] times: {}, received: {}, took: {}"
, this.brokerId
, this.receivedLastTimes.get()
, this.receivedCount.get()
, tookAmt
);
}
if (this.receivedCount.get() != tookAmt) {
if (log.isDebugEnabled()) {
log.debug("[ANALYSIS END] [{}] times: {}, received: {}, took: {}"
, this.brokerId
, this.receivedLastTimes.get()
, this.receivedCount.get()
, tookAmt
);
}
}
synchronized (this.state) {
if (this.state.value < OrderState.COMPLETED.value) {
this.send(() -> getReqBuilder().setCancelRequest(this.cancelReq).build());
this.state = OrderState.COMPLETED;
}
}
}
private ScanStreamBatchReq.Builder getReqBuilder() {
return this.reqBuilder.clearQueryRequest();
}
private void send(Supplier<ScanStreamBatchReq> supplier) {
this.senderLock.lock();
try {
if (!this.serverFinished.get()) {
this.requestObserver.onNext(supplier.get());
}
Thread.yield();
} finally {
this.senderLock.unlock();
}
}
private class OrderAgent implements StreamObserver<KvPageRes> {
private final AtomicInteger count = new AtomicInteger(0);
private final AtomicBoolean over = new AtomicBoolean(false);
private final String agentId;
OrderAgent(String agentId) {
this.agentId = agentId;
}
@Override
public void onNext(KvPageRes value) {
if (log.isDebugEnabled()) {
log.debug("Scan [ {} ] [ {} ] times, received: [ {} ]"
, nodeSession.getStoreNode().getAddress(), value.getTimes(),
value.getDataList().size());
}
serverFinished.set(value.getOver());
List<Kv> buffer = value.getDataList();
count.addAndGet(buffer.size());
if (log.isDebugEnabled()) {
if (value.getOver()) {
log.debug("[ANALYSIS OVER] [{}] count: {}", agentId, count);
}
}
keeper.receive(buffer, value.getTimes());
this.over.set(value.getOver());
this.checkOver(value.getTimes());
}
private void checkOver(int times) {
if (this.over.get()) {
requestObserver.onCompleted();
keeper.done(times);
}
}
@Override
public void onError(Throwable t) {
log.error("received server onError event, Throwable:", t);
keeper.shout(t);
}
@Override
public void onCompleted() {
if (log.isDebugEnabled()) log.debug("received sever completed event.");
serverFinished.set(true);
}
}
/*** Inventory Keeper ***/
private class OrderKeeper {
private final BlockingQueue<Supplier<List<Kv>>> queue = new LinkedBlockingQueue<>();
private final ReentrantLock pickUpLock = new ReentrantLock();
private final AtomicBoolean done = new AtomicBoolean();
private final AtomicBoolean stop = new AtomicBoolean();
private int timesOfOver;
private int lastTimes;
private Throwable serverErr;
void receive(List<Kv> data, int times) {
receivedCount.addAndGet(data.size());
this.queue.offer(() -> data);
receipt(times);
this.lastTimes = times;
}
private List<Kv> pickUp() {
Supplier<List<Kv>> res;
pickUpLock.lock();
try {
if (this.done.get()) {
if (this.stop.get()) {
log.warn("Invoking pickUp method after OrderKeeper has bean closing.");
}
res = this.queue.poll();
if (res == null) {
res = () -> null;
}
} else {
res = this.queue.poll(HAVE_NEXT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
if (res == null) {
if (this.done.get()) {
res = () -> null;
} else {
throw HgStoreClientException.of(
"Timeout, max time: " + HAVE_NEXT_TIMEOUT_SECONDS +
" seconds" +
", isOver: " + this.done.get() +
", isStop: " + this.stop.get() +
", last-times: " + this.lastTimes +
", over-times: " + this.timesOfOver);
}
}
}
} catch (InterruptedException e) {
log.error(
"Failed to receive List<Kv> from queue because of interruption of " +
"current thread ["
+ Thread.currentThread().getName() + "]");
Thread.currentThread().interrupt();
throw HgStoreClientException.of(
"Failed to receive List<Kv> from queue, cause by:", e);
} finally {
pickUpLock.unlock();
}
checkServerErr();
return res.get();
}
void done(int times) {
this.timesOfOver = times;
this.done.set(true);
this.queue.offer(() -> null);
}
void shout(Throwable t) {
this.serverErr = t;
log.error("Failed to receive from sever", t);
this.queue.offer(() -> null);
}
private void checkServerErr() {
if (this.serverErr != null) {
throw HgStoreClientException.of(this.serverErr);
}
}
}
}
/* iterator */
private static class OrderConsumer implements KvCloseableIterator<Kv>, HgPageSize {
private final OrderBroker broker;
private Iterator<Kv> dataIterator;
private long tookCount = 0;
private final String consumerId;
OrderConsumer(OrderBroker broker) {
this.broker = broker;
consumerId = broker.brokerId;
}
private Iterator<Kv> getIterator() {
List<Kv> list = this.broker.oneMore();
if (log.isDebugEnabled()) {
if (list != null && list.isEmpty()) {
log.debug("[ANALYSIS EMPTY] [{}] , tookCount: {}", consumerId, tookCount);
}
}
if (list == null || list.isEmpty()) {
return null;
} else {
return list.iterator();
}
}
@Override
public void close() {
this.broker.finish(this.tookCount);
}
@Override
public long getPageSize() {
return PAGE_SIZE;
}
@Override
public boolean hasNext() {
if (this.dataIterator == null) {
this.dataIterator = this.getIterator();
} else {
if (this.dataIterator.hasNext()) {
return true;
} else {
this.dataIterator = this.getIterator();
}
}
if (this.dataIterator == null) {
if (log.isDebugEnabled()) {
log.debug("[ANALYSIS NULL -> FALSE] [{}] , tookCount: {}", consumerId,
tookCount);
}
return false;
} else {
if (log.isDebugEnabled()) {
if (!this.dataIterator.hasNext()) {
log.debug("[ANALYSIS hasNext -> FALSE] [{}] , tookCount: {}", consumerId,
tookCount);
}
}
return this.dataIterator.hasNext();
}
}
@Override
public Kv next() {
if (this.dataIterator == null) {
if (!this.hasNext()) {
throw new NoSuchElementException();
}
}
if (log.isDebugEnabled()) {
tookCount++;
if (tookCount % 10000 == 0) {
log.debug("[ANALYSIS NEXT] [{}] , tookCount: {}", consumerId, tookCount);
}
}
return this.dataIterator.next();
}
}
}

302
hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchUtil.java
View File

@ -1,152 +1,152 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.util.List;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgScanQuery;
import org.apache.hugegraph.store.client.HgStoreNodeSession;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.store.grpc.common.Header;
import org.apache.hugegraph.store.grpc.common.ScanMethod;
import com.google.protobuf.ByteString;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2022/04/23
*/
@Slf4j
@NotThreadSafe
class KvBatchUtil {
static final byte[] EMPTY_POSITION = HgStoreClientConst.EMPTY_BYTES;
static ScanStreamBatchReq.Builder getRequestBuilder(HgStoreNodeSession nodeSession) {
return ScanStreamBatchReq.newBuilder().setHeader(getHeader(nodeSession));
}
static ScanQueryRequest createQueryReq(HgScanQuery scanQuery, long pageSize) {
ScanQueryRequest.Builder qb = ScanQueryRequest.newBuilder();
ScanCondition.Builder cb = ScanCondition.newBuilder();
qb.setLimit(getLimit(scanQuery.getLimit()));
qb.setPerKeyLimit(getLimit(scanQuery.getPerKeyLimit()));
qb.setPerKeyMax(getLimit(scanQuery.getPerKeyMax()));
switch (scanQuery.getScanMethod()) {
case ALL:
qb.setMethod(ScanMethod.ALL);
break;
case PREFIX:
qb.setMethod(ScanMethod.PREFIX);
addPrefixCondition(scanQuery, qb, cb);
break;
case RANGE:
qb.setMethod(ScanMethod.RANGE);
addRangeCondition(scanQuery, qb, cb);
break;
default:
throw new RuntimeException("Unsupported ScanType: " + scanQuery.getScanMethod());
}
qb.setTable(scanQuery.getTable());
qb.setPageSize(pageSize);
qb.setQuery(toBs(scanQuery.getQuery()));
qb.setScanType(scanQuery.getScanType());
qb.setOrderType(scanQuery.getOrderType());
qb.setSkipDegree(scanQuery.getSkipDegree());
return qb.build();
}
static long getLimit(long limit) {
return limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE : limit;
}
static Header getHeader(HgStoreNodeSession nodeSession) {
return Header.newBuilder().setGraph(nodeSession.getGraphName()).build();
}
static void addPrefixCondition(HgScanQuery scanQuery, ScanQueryRequest.Builder qb,
ScanCondition.Builder cb) {
List<HgOwnerKey> prefixList = scanQuery.getPrefixList();
if (prefixList == null || prefixList.isEmpty()) {
throw new RuntimeException(
"The start-list of ScanQuery shouldn't to be invalid in ScanMethod.PREFIX " +
"mode.");
}
prefixList.forEach((e) -> {
qb.addCondition(cb.clear()
.setPrefix(toBs(e.getKey()))
.setCode(e.getKeyCode())
.setSerialNo(e.getSerialNo())
.build()
);
});
}
static void addRangeCondition(HgScanQuery scanQuery, ScanQueryRequest.Builder qb,
ScanCondition.Builder cb) {
List<HgOwnerKey> startList = scanQuery.getStartList();
List<HgOwnerKey> endList = scanQuery.getEndList();
if (startList == null || startList.isEmpty()) {
throw new RuntimeException(
"The start-list of ScanQuery shouldn't to be invalid in ScanMethod.RANGE mode" +
".");
}
if (endList == null || endList.isEmpty()) {
throw new RuntimeException(
"The end-list of ScanQuery shouldn't to be invalid in ScanMethod.RANGE mode.");
}
if (startList.size() != endList.size()) {
throw new RuntimeException("The size of start-list not equals end-list's.");
}
for (int i = 0, s = startList.size(); i < s; i++) {
HgOwnerKey start = startList.get(i);
HgOwnerKey end = endList.get(i);
qb.addCondition(cb.clear().setCode(start.getKeyCode())
.setStart(toBs(start.getKey()))
.setEnd(toBs(end.getKey()))
.setSerialNo(start.getSerialNo())
.build()
);
}
}
static HgOwnerKey toOk(HgOwnerKey key) {
return key == null ? HgStoreClientConst.EMPTY_OWNER_KEY : key;
}
static ByteString toBs(byte[] bytes) {
return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES);
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.util.List;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgScanQuery;
import org.apache.hugegraph.store.client.HgStoreNodeSession;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.store.grpc.common.Header;
import org.apache.hugegraph.store.grpc.common.ScanMethod;
import com.google.protobuf.ByteString;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2022/04/23
*/
@Slf4j
@NotThreadSafe
class KvBatchUtil {
static final byte[] EMPTY_POSITION = HgStoreClientConst.EMPTY_BYTES;
static ScanStreamBatchReq.Builder getRequestBuilder(HgStoreNodeSession nodeSession) {
return ScanStreamBatchReq.newBuilder().setHeader(getHeader(nodeSession));
}
static ScanQueryRequest createQueryReq(HgScanQuery scanQuery, long pageSize) {
ScanQueryRequest.Builder qb = ScanQueryRequest.newBuilder();
ScanCondition.Builder cb = ScanCondition.newBuilder();
qb.setLimit(getLimit(scanQuery.getLimit()));
qb.setPerKeyLimit(getLimit(scanQuery.getPerKeyLimit()));
qb.setPerKeyMax(getLimit(scanQuery.getPerKeyMax()));
switch (scanQuery.getScanMethod()) {
case ALL:
qb.setMethod(ScanMethod.ALL);
break;
case PREFIX:
qb.setMethod(ScanMethod.PREFIX);
addPrefixCondition(scanQuery, qb, cb);
break;
case RANGE:
qb.setMethod(ScanMethod.RANGE);
addRangeCondition(scanQuery, qb, cb);
break;
default:
throw new RuntimeException("Unsupported ScanType: " + scanQuery.getScanMethod());
}
qb.setTable(scanQuery.getTable());
qb.setPageSize(pageSize);
qb.setQuery(toBs(scanQuery.getQuery()));
qb.setScanType(scanQuery.getScanType());
qb.setOrderType(scanQuery.getOrderType());
qb.setSkipDegree(scanQuery.getSkipDegree());
return qb.build();
}
static long getLimit(long limit) {
return limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE : limit;
}
static Header getHeader(HgStoreNodeSession nodeSession) {
return Header.newBuilder().setGraph(nodeSession.getGraphName()).build();
}
static void addPrefixCondition(HgScanQuery scanQuery, ScanQueryRequest.Builder qb,
ScanCondition.Builder cb) {
List<HgOwnerKey> prefixList = scanQuery.getPrefixList();
if (prefixList == null || prefixList.isEmpty()) {
throw new RuntimeException(
"The start-list of ScanQuery shouldn't to be invalid in ScanMethod.PREFIX " +
"mode.");
}
prefixList.forEach((e) -> {
qb.addCondition(cb.clear()
.setPrefix(toBs(e.getKey()))
.setCode(e.getKeyCode())
.setSerialNo(e.getSerialNo())
.build()
);
});
}
static void addRangeCondition(HgScanQuery scanQuery, ScanQueryRequest.Builder qb,
ScanCondition.Builder cb) {
List<HgOwnerKey> startList = scanQuery.getStartList();
List<HgOwnerKey> endList = scanQuery.getEndList();
if (startList == null || startList.isEmpty()) {
throw new RuntimeException(
"The start-list of ScanQuery shouldn't to be invalid in ScanMethod.RANGE mode" +
".");
}
if (endList == null || endList.isEmpty()) {
throw new RuntimeException(
"The end-list of ScanQuery shouldn't to be invalid in ScanMethod.RANGE mode.");
}
if (startList.size() != endList.size()) {
throw new RuntimeException("The size of start-list not equals end-list's.");
}
for (int i = 0, s = startList.size(); i < s; i++) {
HgOwnerKey start = startList.get(i);
HgOwnerKey end = endList.get(i);
qb.addCondition(cb.clear().setCode(start.getKeyCode())
.setStart(toBs(start.getKey()))
.setEnd(toBs(end.getKey()))
.setSerialNo(start.getSerialNo())
.build()
);
}
}
static HgOwnerKey toOk(HgOwnerKey key) {
return key == null ? HgStoreClientConst.EMPTY_OWNER_KEY : key;
}
static ByteString toBs(byte[] bytes) {
return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES);
}
}

56
hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvCloseableIterator.java
View File

@ -1,28 +1,28 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.io.Closeable;
import java.util.Iterator;
/**
* @author lynn.bond@hotmail.com on 2022/3/16
*/
public interface KvCloseableIterator<T> extends Iterator<T>, Closeable {
void close();
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.io.Closeable;
import java.util.Iterator;
/**
* @author lynn.bond@hotmail.com on 2022/3/16
*/
public interface KvCloseableIterator<T> extends Iterator<T>, Closeable {
void close();
}

96
hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvListIterator.java
View File

@ -1,48 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.util.Iterator;
import java.util.List;
/**
* @author lynn.bond@hotmail.com on 2022/4/6
*/
class KvListIterator<T> implements KvCloseableIterator {
private final Iterator<T> iterator;
KvListIterator(List<T> list) {
this.iterator = list.iterator();
}
@Override
public void close() {
/*Nothing to do.*/
}
@Override
public boolean hasNext() {
return this.iterator.hasNext();
}
@Override
public T next() {
return this.iterator.next();
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.util.Iterator;
import java.util.List;
/**
* @author lynn.bond@hotmail.com on 2022/4/6
*/
class KvListIterator<T> implements KvCloseableIterator {
private final Iterator<T> iterator;
KvListIterator(List<T> list) {
this.iterator = list.iterator();
}
@Override
public void close() {
/*Nothing to do.*/
}
@Override
public boolean hasNext() {
return this.iterator.hasNext();
}
@Override
public T next() {
return this.iterator.next();
}
}

424
hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvOneShotScanner.java
View File

@ -1,213 +1,213 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.util.Iterator;
import java.util.List;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.store.HgKvStore;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgPageSize;
import org.apache.hugegraph.store.HgSeekAble;
import org.apache.hugegraph.store.client.HgStoreNodeSession;
import org.apache.hugegraph.store.client.util.HgStoreClientConfig;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.store.client.util.HgStoreClientUtil;
import org.apache.hugegraph.store.grpc.common.Header;
import org.apache.hugegraph.store.grpc.common.Kv;
import org.apache.hugegraph.store.grpc.common.ScanMethod;
import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamBlockingStub;
import org.apache.hugegraph.store.grpc.stream.ScanStreamReq;
import com.google.protobuf.ByteString;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2021/12/1
*/
@Slf4j
@NotThreadSafe
class KvOneShotScanner implements KvCloseableIterator<Kv>, HgPageSize, HgSeekAble {
private static final HgStoreClientConfig storeClientConfig = HgStoreClientConfig.of();
private final HgStoreNodeSession session;
private final HgStoreStreamBlockingStub stub;
private final ScanStreamReq.Builder reqBuilder = ScanStreamReq.newBuilder();
private ScanStreamReq req;
private Iterator<Kv> iterator;
private List<Kv> list = null;
private final String table;
private final HgOwnerKey startKey;
private final HgOwnerKey endKey;
private final HgOwnerKey prefix;
private final ScanMethod scanMethod;
private final long limit;
private final int partition;
private final int scanType;
private final byte[] query;
private final int pageSize;
private boolean in = true;
private byte[] nodePosition = HgStoreClientConst.EMPTY_BYTES;
public static KvCloseableIterator<Kv> scanAll(HgStoreNodeSession session,
HgStoreStreamBlockingStub stub,
String table, long limit, byte[] query) {
return new KvOneShotScanner(ScanMethod.ALL, session, stub, table, null, null, null, limit,
-1, HgKvStore.SCAN_ANY,
query);
}
public static KvCloseableIterator<Kv> scanPrefix(HgStoreNodeSession session,
HgStoreStreamBlockingStub stub,
String table, HgOwnerKey prefix, long limit,
byte[] query) {
return new KvOneShotScanner(ScanMethod.PREFIX, session, stub, table, prefix, null, null,
limit,
prefix.getKeyCode(), HgKvStore.SCAN_PREFIX_BEGIN, query);
}
public static KvCloseableIterator<Kv> scanRange(HgStoreNodeSession nodeSession,
HgStoreStreamBlockingStub stub,
String table, HgOwnerKey startKey,
HgOwnerKey endKey, long limit,
int scanType, byte[] query) {
return new KvOneShotScanner(ScanMethod.RANGE, nodeSession, stub, table, null, startKey,
endKey, limit,
startKey.getKeyCode(), scanType, query);
}
private KvOneShotScanner(ScanMethod scanMethod, HgStoreNodeSession session,
HgStoreStreamBlockingStub stub,
String table, HgOwnerKey prefix, HgOwnerKey startKey,
HgOwnerKey endKey, long limit,
int partition, int scanType, byte[] query) {
this.scanMethod = scanMethod;
this.session = session;
this.stub = stub;
this.table = table;
this.startKey = toOk(startKey);
this.endKey = toOk(endKey);
this.prefix = toOk(prefix);
this.partition = partition;
this.scanType = scanType;
this.query = query != null ? query : HgStoreClientConst.EMPTY_BYTES;
this.limit = limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE :
limit; // <=0 means no limit
this.pageSize = storeClientConfig.getNetKvScannerPageSize();
}
private Header getHeader(HgStoreNodeSession nodeSession) {
return Header.newBuilder().setGraph(nodeSession.getGraphName()).build();
}
private void createReq() {
this.req = this.reqBuilder
.setHeader(this.getHeader(this.session))
.setMethod(this.scanMethod)
.setTable(this.table)
.setStart(toBs(this.startKey.getKey()))
.setEnd(toBs(this.endKey.getKey()))
.setLimit(this.limit)
.setPrefix(toBs(this.prefix.getKey()))
.setCode(this.partition)
.setScanType(this.scanType)
.setQuery(toBs(this.query))
.setPageSize(pageSize)
.setPosition(toBs(this.nodePosition))
.build();
}
private void init() {
if (this.iterator == null) {
this.createReq();
this.list = this.stub.scanOneShot(this.req).getDataList();
this.iterator = this.list.iterator();
}
}
@Override
public boolean hasNext() {
if (!this.in) {
return false;
}
if (this.iterator == null) {
this.init();
}
return this.iterator.hasNext();
}
@Override
public Kv next() {
if (this.iterator == null) {
this.init();
}
return this.iterator.next();
}
@Override
public long getPageSize() {
return this.limit;
}
@Override
public boolean isPageEmpty() {
return !this.iterator.hasNext();
}
@Override
public byte[] position() {
return HgStoreClientUtil.toBytes(this.session.getStoreNode().getNodeId().longValue());
}
@Override
public void seek(byte[] position) {
if (position == null || position.length < Long.BYTES) {
return;
}
byte[] nodeIdBytes = new byte[Long.BYTES];
System.arraycopy(position, 0, nodeIdBytes, 0, Long.BYTES);
long nodeId = this.session.getStoreNode().getNodeId().longValue();
long pId = HgStoreClientUtil.toLong(nodeIdBytes);
this.in = nodeId >= pId;
if (this.in && nodeId == pId) {
this.nodePosition = new byte[position.length - Long.BYTES];
System.arraycopy(position, Long.BYTES, this.nodePosition, 0, this.nodePosition.length);
} else {
this.nodePosition = HgStoreClientConst.EMPTY_BYTES;
}
}
static HgOwnerKey toOk(HgOwnerKey key) {
return key == null ? HgStoreClientConst.EMPTY_OWNER_KEY : key;
}
static ByteString toBs(byte[] bytes) {
return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES);
}
@Override
public void close() {
//TODO: implements
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.util.Iterator;
import java.util.List;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.store.HgKvStore;
import org.apache.hugegraph.store.HgOwnerKey;
import org.apache.hugegraph.store.HgPageSize;
import org.apache.hugegraph.store.HgSeekAble;
import org.apache.hugegraph.store.client.HgStoreNodeSession;
import org.apache.hugegraph.store.client.util.HgStoreClientConfig;
import org.apache.hugegraph.store.client.util.HgStoreClientConst;
import org.apache.hugegraph.store.client.util.HgStoreClientUtil;
import org.apache.hugegraph.store.grpc.common.Header;
import org.apache.hugegraph.store.grpc.common.Kv;
import org.apache.hugegraph.store.grpc.common.ScanMethod;
import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamBlockingStub;
import org.apache.hugegraph.store.grpc.stream.ScanStreamReq;
import com.google.protobuf.ByteString;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2021/12/1
*/
@Slf4j
@NotThreadSafe
class KvOneShotScanner implements KvCloseableIterator<Kv>, HgPageSize, HgSeekAble {
private static final HgStoreClientConfig storeClientConfig = HgStoreClientConfig.of();
private final HgStoreNodeSession session;
private final HgStoreStreamBlockingStub stub;
private final ScanStreamReq.Builder reqBuilder = ScanStreamReq.newBuilder();
private ScanStreamReq req;
private Iterator<Kv> iterator;
private List<Kv> list = null;
private final String table;
private final HgOwnerKey startKey;
private final HgOwnerKey endKey;
private final HgOwnerKey prefix;
private final ScanMethod scanMethod;
private final long limit;
private final int partition;
private final int scanType;
private final byte[] query;
private final int pageSize;
private boolean in = true;
private byte[] nodePosition = HgStoreClientConst.EMPTY_BYTES;
public static KvCloseableIterator<Kv> scanAll(HgStoreNodeSession session,
HgStoreStreamBlockingStub stub,
String table, long limit, byte[] query) {
return new KvOneShotScanner(ScanMethod.ALL, session, stub, table, null, null, null, limit,
-1, HgKvStore.SCAN_ANY,
query);
}
public static KvCloseableIterator<Kv> scanPrefix(HgStoreNodeSession session,
HgStoreStreamBlockingStub stub,
String table, HgOwnerKey prefix, long limit,
byte[] query) {
return new KvOneShotScanner(ScanMethod.PREFIX, session, stub, table, prefix, null, null,
limit,
prefix.getKeyCode(), HgKvStore.SCAN_PREFIX_BEGIN, query);
}
public static KvCloseableIterator<Kv> scanRange(HgStoreNodeSession nodeSession,
HgStoreStreamBlockingStub stub,
String table, HgOwnerKey startKey,
HgOwnerKey endKey, long limit,
int scanType, byte[] query) {
return new KvOneShotScanner(ScanMethod.RANGE, nodeSession, stub, table, null, startKey,
endKey, limit,
startKey.getKeyCode(), scanType, query);
}
private KvOneShotScanner(ScanMethod scanMethod, HgStoreNodeSession session,
HgStoreStreamBlockingStub stub,
String table, HgOwnerKey prefix, HgOwnerKey startKey,
HgOwnerKey endKey, long limit,
int partition, int scanType, byte[] query) {
this.scanMethod = scanMethod;
this.session = session;
this.stub = stub;
this.table = table;
this.startKey = toOk(startKey);
this.endKey = toOk(endKey);
this.prefix = toOk(prefix);
this.partition = partition;
this.scanType = scanType;
this.query = query != null ? query : HgStoreClientConst.EMPTY_BYTES;
this.limit = limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE :
limit; // <=0 means no limit
this.pageSize = storeClientConfig.getNetKvScannerPageSize();
}
private Header getHeader(HgStoreNodeSession nodeSession) {
return Header.newBuilder().setGraph(nodeSession.getGraphName()).build();
}
private void createReq() {
this.req = this.reqBuilder
.setHeader(this.getHeader(this.session))
.setMethod(this.scanMethod)
.setTable(this.table)
.setStart(toBs(this.startKey.getKey()))
.setEnd(toBs(this.endKey.getKey()))
.setLimit(this.limit)
.setPrefix(toBs(this.prefix.getKey()))
.setCode(this.partition)
.setScanType(this.scanType)
.setQuery(toBs(this.query))
.setPageSize(pageSize)
.setPosition(toBs(this.nodePosition))
.build();
}
private void init() {
if (this.iterator == null) {
this.createReq();
this.list = this.stub.scanOneShot(this.req).getDataList();
this.iterator = this.list.iterator();
}
}
@Override
public boolean hasNext() {
if (!this.in) {
return false;
}
if (this.iterator == null) {
this.init();
}
return this.iterator.hasNext();
}
@Override
public Kv next() {
if (this.iterator == null) {
this.init();
}
return this.iterator.next();
}
@Override
public long getPageSize() {
return this.limit;
}
@Override
public boolean isPageEmpty() {
return !this.iterator.hasNext();
}
@Override
public byte[] position() {
return HgStoreClientUtil.toBytes(this.session.getStoreNode().getNodeId().longValue());
}
@Override
public void seek(byte[] position) {
if (position == null || position.length < Long.BYTES) {
return;
}
byte[] nodeIdBytes = new byte[Long.BYTES];
System.arraycopy(position, 0, nodeIdBytes, 0, Long.BYTES);
long nodeId = this.session.getStoreNode().getNodeId().longValue();
long pId = HgStoreClientUtil.toLong(nodeIdBytes);
this.in = nodeId >= pId;
if (this.in && nodeId == pId) {
this.nodePosition = new byte[position.length - Long.BYTES];
System.arraycopy(position, Long.BYTES, this.nodePosition, 0, this.nodePosition.length);
} else {
this.nodePosition = HgStoreClientConst.EMPTY_BYTES;
}
}
static HgOwnerKey toOk(HgOwnerKey key) {
return key == null ? HgStoreClientConst.EMPTY_OWNER_KEY : key;
}
static ByteString toBs(byte[] bytes) {
return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES);
}
@Override
public void close() {
//TODO: implements
}
}

136
hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/SeekAbleIterator.java
View File

@ -1,69 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.util.Iterator;
import java.util.function.Consumer;
import java.util.function.Supplier;
import org.apache.hugegraph.store.HgSeekAble;
import org.apache.hugegraph.store.client.util.HgAssert;
/**
* @author lynn.bond@hotmail.com on 2022/3/11
*/
class SeekAbleIterator<E> implements Iterator, HgSeekAble {
private final Iterator<E> iterator;
private final Consumer<byte[]> seeker;
private final Supplier<byte[]> positioner;
public static <E> SeekAbleIterator of(Iterator<E> iterator, Supplier<byte[]> positioner,
Consumer<byte[]> seeker) {
HgAssert.isArgumentNotNull(iterator, "iterator");
HgAssert.isArgumentNotNull(positioner, "positioner");
HgAssert.isArgumentNotNull(seeker, "seeker");
return new SeekAbleIterator(iterator, positioner, seeker);
}
private SeekAbleIterator(Iterator<E> iterator, Supplier<byte[]> positioner,
Consumer<byte[]> seeker) {
this.iterator = iterator;
this.positioner = positioner;
this.seeker = seeker;
}
@Override
public byte[] position() {
return this.positioner.get();
}
@Override
public void seek(byte[] position) {
this.seeker.accept(position);
}
@Override
public boolean hasNext() {
return this.iterator.hasNext();
}
@Override
public E next() {
return this.iterator.next();
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.grpc;
import java.util.Iterator;
import java.util.function.Consumer;
import java.util.function.Supplier;
import org.apache.hugegraph.store.HgSeekAble;
import org.apache.hugegraph.store.client.util.HgAssert;
/**
* @author lynn.bond@hotmail.com on 2022/3/11
*/
class SeekAbleIterator<E> implements Iterator, HgSeekAble {
private final Iterator<E> iterator;
private final Consumer<byte[]> seeker;
private final Supplier<byte[]> positioner;
public static <E> SeekAbleIterator of(Iterator<E> iterator, Supplier<byte[]> positioner,
Consumer<byte[]> seeker) {
HgAssert.isArgumentNotNull(iterator, "iterator");
HgAssert.isArgumentNotNull(positioner, "positioner");
HgAssert.isArgumentNotNull(seeker, "seeker");
return new SeekAbleIterator(iterator, positioner, seeker);
}
private SeekAbleIterator(Iterator<E> iterator, Supplier<byte[]> positioner,
Consumer<byte[]> seeker) {
this.iterator = iterator;
this.positioner = positioner;
this.seeker = seeker;
}
@Override
public byte[] position() {
return this.positioner.get();
}
@Override
public void seek(byte[] position) {
this.seeker.accept(position);
}
@Override
public boolean hasNext() {
return this.iterator.hasNext();
}
@Override
public E next() {
return this.iterator.next();
}
}

354
hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/Base58.java
View File

@ -1,177 +1,177 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.util;
import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
/**
* Created with IntelliJ IDEA.
* User: noah
* Date: 8/2/13
* Time: 10:36 AM
* To change this template use File | Settings | File Templates.
*/
public class Base58 {
public static final char[] ALPHABET =
"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray();
private static final int[] INDEXES = new int[128];
static {
for (int i = 0; i < INDEXES.length; i++) {
INDEXES[i] = -1;
}
for (int i = 0; i < ALPHABET.length; i++) {
INDEXES[ALPHABET[i]] = i;
}
}
/**
* Encodes the given bytes in base58. No checksum is appended.
*/
public static String encode(byte[] input) {
if (input.length == 0) {
return "";
}
input = copyOfRange(input, 0, input.length);
// Count leading zeroes.
int zeroCount = 0;
while (zeroCount < input.length && input[zeroCount] == 0) {
++zeroCount;
}
// The actual encoding.
byte[] temp = new byte[input.length * 2];
int j = temp.length;
int startAt = zeroCount;
while (startAt < input.length) {
byte mod = divmod58(input, startAt);
if (input[startAt] == 0) {
++startAt;
}
temp[--j] = (byte) ALPHABET[mod];
}
// Strip extra '1' if there are some after decoding.
while (j < temp.length && temp[j] == ALPHABET[0]) {
++j;
}
// Add as many leading '1' as there were leading zeros.
while (--zeroCount >= 0) {
temp[--j] = (byte) ALPHABET[0];
}
byte[] output = copyOfRange(temp, j, temp.length);
return new String(output, StandardCharsets.US_ASCII);
}
public static byte[] decode(String input) throws IllegalArgumentException {
if (input.length() == 0) {
return new byte[0];
}
byte[] input58 = new byte[input.length()];
// Transform the String to a base58 byte sequence
for (int i = 0; i < input.length(); ++i) {
char c = input.charAt(i);
int digit58 = -1;
if (c >= 0 && c < 128) {
digit58 = INDEXES[c];
}
if (digit58 < 0) {
throw new IllegalArgumentException("Illegal character " + c + " at " + i);
}
input58[i] = (byte) digit58;
}
// Count leading zeroes
int zeroCount = 0;
while (zeroCount < input58.length && input58[zeroCount] == 0) {
++zeroCount;
}
// The encoding
byte[] temp = new byte[input.length()];
int j = temp.length;
int startAt = zeroCount;
while (startAt < input58.length) {
byte mod = divmod256(input58, startAt);
if (input58[startAt] == 0) {
++startAt;
}
temp[--j] = mod;
}
// Do no add extra leading zeroes, move j to first non null byte.
while (j < temp.length && temp[j] == 0) {
++j;
}
return copyOfRange(temp, j - zeroCount, temp.length);
}
public static BigInteger decodeToBigInteger(String input) throws IllegalArgumentException {
return new BigInteger(1, decode(input));
}
//
// number -> number / 58, returns number % 58
//
private static byte divmod58(byte[] number, int startAt) {
int remainder = 0;
for (int i = startAt; i < number.length; i++) {
int digit256 = (int) number[i] & 0xFF;
int temp = remainder * 256 + digit256;
number[i] = (byte) (temp / 58);
remainder = temp % 58;
}
return (byte) remainder;
}
//
// number -> number / 256, returns number % 256
//
private static byte divmod256(byte[] number58, int startAt) {
int remainder = 0;
for (int i = startAt; i < number58.length; i++) {
int digit58 = (int) number58[i] & 0xFF;
int temp = remainder * 58 + digit58;
number58[i] = (byte) (temp / 256);
remainder = temp % 256;
}
return (byte) remainder;
}
private static byte[] copyOfRange(byte[] source, int from, int to) {
byte[] range = new byte[to - from];
System.arraycopy(source, from, range, 0, range.length);
return range;
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.util;
import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
/**
* Created with IntelliJ IDEA.
* User: noah
* Date: 8/2/13
* Time: 10:36 AM
* To change this template use File | Settings | File Templates.
*/
public class Base58 {
public static final char[] ALPHABET =
"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray();
private static final int[] INDEXES = new int[128];
static {
for (int i = 0; i < INDEXES.length; i++) {
INDEXES[i] = -1;
}
for (int i = 0; i < ALPHABET.length; i++) {
INDEXES[ALPHABET[i]] = i;
}
}
/**
* Encodes the given bytes in base58. No checksum is appended.
*/
public static String encode(byte[] input) {
if (input.length == 0) {
return "";
}
input = copyOfRange(input, 0, input.length);
// Count leading zeroes.
int zeroCount = 0;
while (zeroCount < input.length && input[zeroCount] == 0) {
++zeroCount;
}
// The actual encoding.
byte[] temp = new byte[input.length * 2];
int j = temp.length;
int startAt = zeroCount;
while (startAt < input.length) {
byte mod = divmod58(input, startAt);
if (input[startAt] == 0) {
++startAt;
}
temp[--j] = (byte) ALPHABET[mod];
}
// Strip extra '1' if there are some after decoding.
while (j < temp.length && temp[j] == ALPHABET[0]) {
++j;
}
// Add as many leading '1' as there were leading zeros.
while (--zeroCount >= 0) {
temp[--j] = (byte) ALPHABET[0];
}
byte[] output = copyOfRange(temp, j, temp.length);
return new String(output, StandardCharsets.US_ASCII);
}
public static byte[] decode(String input) throws IllegalArgumentException {
if (input.length() == 0) {
return new byte[0];
}
byte[] input58 = new byte[input.length()];
// Transform the String to a base58 byte sequence
for (int i = 0; i < input.length(); ++i) {
char c = input.charAt(i);
int digit58 = -1;
if (c >= 0 && c < 128) {
digit58 = INDEXES[c];
}
if (digit58 < 0) {
throw new IllegalArgumentException("Illegal character " + c + " at " + i);
}
input58[i] = (byte) digit58;
}
// Count leading zeroes
int zeroCount = 0;
while (zeroCount < input58.length && input58[zeroCount] == 0) {
++zeroCount;
}
// The encoding
byte[] temp = new byte[input.length()];
int j = temp.length;
int startAt = zeroCount;
while (startAt < input58.length) {
byte mod = divmod256(input58, startAt);
if (input58[startAt] == 0) {
++startAt;
}
temp[--j] = mod;
}
// Do no add extra leading zeroes, move j to first non null byte.
while (j < temp.length && temp[j] == 0) {
++j;
}
return copyOfRange(temp, j - zeroCount, temp.length);
}
public static BigInteger decodeToBigInteger(String input) throws IllegalArgumentException {
return new BigInteger(1, decode(input));
}
//
// number -> number / 58, returns number % 58
//
private static byte divmod58(byte[] number, int startAt) {
int remainder = 0;
for (int i = startAt; i < number.length; i++) {
int digit256 = (int) number[i] & 0xFF;
int temp = remainder * 256 + digit256;
number[i] = (byte) (temp / 58);
remainder = temp % 58;
}
return (byte) remainder;
}
//
// number -> number / 256, returns number % 256
//
private static byte divmod256(byte[] number58, int startAt) {
int remainder = 0;
for (int i = startAt; i < number58.length; i++) {
int digit58 = (int) number58[i] & 0xFF;
int temp = remainder * 58 + digit58;
number58[i] = (byte) (temp / 256);
remainder = temp % 256;
}
return (byte) remainder;
}
private static byte[] copyOfRange(byte[] source, int from, int to) {
byte[] range = new byte[to - from];
System.arraycopy(source, from, range, 0, range.length);
return range;
}
}

320
hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgBufferProxy.java
View File

@ -1,160 +1,160 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.util;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
import java.util.function.Supplier;
import javax.annotation.CheckForNull;
import org.apache.hugegraph.store.client.type.HgStoreClientException;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com on 2022/3/15
* @version 0.1.0
*/
@Slf4j
public final class HgBufferProxy<T> {
private final BlockingQueue<Supplier<T>> queue;
private final AtomicBoolean closed = new AtomicBoolean(false);
private final ReentrantLock lock = new ReentrantLock();
private final Runnable task;
private Throwable err;
public static HgBufferProxy of(Runnable task) {
HgAssert.isArgumentNotNull(task, "task");
return new HgBufferProxy(task);
}
private HgBufferProxy(Runnable task) {
this.task = task;
this.queue = new LinkedBlockingQueue<>();
}
public void send(T t) {
if (t == null) {
throw new IllegalArgumentException("the argument t is null");
}
if (this.closed.get()) {
return;
}
this.lock.lock();
try {
this.queue.offer(() -> t);
} finally {
lock.unlock();
}
}
private void apply() {
this.lock.lock();
try {
if (!this.closed.get()) {
this.task.run();
Thread.yield();
}
} finally {
this.lock.unlock();
}
}
/**
* return an item from the chan
*
* @return null when the chan has been closed
* @throws RuntimeException
*/
@CheckForNull
public T receive(int time, Consumer<Integer> callback) {
Supplier<T> s;
if (this.closed.get()) {
s = this.queue.poll();
this.checkErr();
return s != null ? s.get() : null;
}
if (this.queue.size() <= 1) {
this.apply();
}
lock.lock();
try {
if (this.isClosed()) {
s = this.queue.poll();
this.checkErr();
return s != null ? s.get() : null;
}
} finally {
lock.unlock();
}
try {
s = this.queue.poll(time, TimeUnit.SECONDS);
} catch (Throwable t) {
log.error("failed to receive a item from channel, cause by: ", t);
throw HgStoreClientException.of(t);
}
if (s == null) {
if (this.closed.get()) {
s = this.queue.poll();
} else {
if (callback == null) {
throw new RuntimeException("timeout, max time: " + time + " seconds");
} else {
callback.accept(time);
}
}
}
this.checkErr();
return s != null ? s.get() : null;
}
public boolean isClosed() {
return this.closed.get();
}
/**
* @throws RuntimeException when fail to close the chan
*/
public void close() {
if (this.closed.get()) {
return;
}
lock.lock();
this.closed.set(true);
try {
this.queue.offer(() -> null);
} finally {
lock.unlock();
}
}
public void setError(Throwable streamErr) {
this.err = streamErr;
}
private void checkErr() {
if (this.err != null) {
throw HgStoreClientException.of(this.err);
}
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.util;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
import java.util.function.Supplier;
import javax.annotation.CheckForNull;
import org.apache.hugegraph.store.client.type.HgStoreClientException;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com on 2022/3/15
* @version 0.1.0
*/
@Slf4j
public final class HgBufferProxy<T> {
private final BlockingQueue<Supplier<T>> queue;
private final AtomicBoolean closed = new AtomicBoolean(false);
private final ReentrantLock lock = new ReentrantLock();
private final Runnable task;
private Throwable err;
public static HgBufferProxy of(Runnable task) {
HgAssert.isArgumentNotNull(task, "task");
return new HgBufferProxy(task);
}
private HgBufferProxy(Runnable task) {
this.task = task;
this.queue = new LinkedBlockingQueue<>();
}
public void send(T t) {
if (t == null) {
throw new IllegalArgumentException("the argument t is null");
}
if (this.closed.get()) {
return;
}
this.lock.lock();
try {
this.queue.offer(() -> t);
} finally {
lock.unlock();
}
}
private void apply() {
this.lock.lock();
try {
if (!this.closed.get()) {
this.task.run();
Thread.yield();
}
} finally {
this.lock.unlock();
}
}
/**
* return an item from the chan
*
* @return null when the chan has been closed
* @throws RuntimeException
*/
@CheckForNull
public T receive(int time, Consumer<Integer> callback) {
Supplier<T> s;
if (this.closed.get()) {
s = this.queue.poll();
this.checkErr();
return s != null ? s.get() : null;
}
if (this.queue.size() <= 1) {
this.apply();
}
lock.lock();
try {
if (this.isClosed()) {
s = this.queue.poll();
this.checkErr();
return s != null ? s.get() : null;
}
} finally {
lock.unlock();
}
try {
s = this.queue.poll(time, TimeUnit.SECONDS);
} catch (Throwable t) {
log.error("failed to receive a item from channel, cause by: ", t);
throw HgStoreClientException.of(t);
}
if (s == null) {
if (this.closed.get()) {
s = this.queue.poll();
} else {
if (callback == null) {
throw new RuntimeException("timeout, max time: " + time + " seconds");
} else {
callback.accept(time);
}
}
}
this.checkErr();
return s != null ? s.get() : null;
}
public boolean isClosed() {
return this.closed.get();
}
/**
* @throws RuntimeException when fail to close the chan
*/
public void close() {
if (this.closed.get()) {
return;
}
lock.lock();
this.closed.set(true);
try {
this.queue.offer(() -> null);
} finally {
lock.unlock();
}
}
public void setError(Throwable streamErr) {
this.err = streamErr;
}
private void checkErr() {
if (this.err != null) {
throw HgStoreClientException.of(this.err);
}
}
}

88
hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgUuid.java
View File

@ -1,44 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.util;
import java.nio.ByteBuffer;
import java.util.UUID;
/**
* @author lynn.bond@hotmail.com
*/
public final class HgUuid {
private static String encode(UUID uuid) {
ByteBuffer bb = ByteBuffer.wrap(new byte[16]);
bb.putLong(uuid.getMostSignificantBits());
bb.putLong(uuid.getLeastSignificantBits());
return Base58.encode(bb.array());
}
/**
* Get a UUID in Base58 FORM
*
* @return
*/
public static String newUUID() {
return encode(UUID.randomUUID());
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.util;
import java.nio.ByteBuffer;
import java.util.UUID;
/**
* @author lynn.bond@hotmail.com
*/
public final class HgUuid {
private static String encode(UUID uuid) {
ByteBuffer bb = ByteBuffer.wrap(new byte[16]);
bb.putLong(uuid.getMostSignificantBits());
bb.putLong(uuid.getLeastSignificantBits());
return Base58.encode(bb.array());
}
/**
* Get a UUID in Base58 FORM
*
* @return
*/
public static String newUUID() {
return encode(UUID.randomUUID());
}
}

206
hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/MetricX.java
View File

@ -1,103 +1,103 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.util;
import java.util.concurrent.atomic.AtomicLong;
/**
* @author lynn.bond@hotmail.com on 2022/1/29
*/
public class MetricX {
public static AtomicLong iteratorSum = new AtomicLong();
public static AtomicLong iteratorCount = new AtomicLong();
public static AtomicLong iteratorMax = new AtomicLong();
public AtomicLong failureCount = new AtomicLong();
private long start;
private long end;
public static MetricX ofStart() {
return new MetricX(System.currentTimeMillis());
}
private MetricX(long start) {
this.start = start;
}
public long start() {
return this.start = System.currentTimeMillis();
}
public long end() {
return this.end = System.currentTimeMillis();
}
public long past() {
return this.end - this.start;
}
public static void plusIteratorWait(long nanoSeconds) {
iteratorSum.addAndGet(nanoSeconds);
iteratorCount.getAndIncrement();
if (iteratorMax.get() < nanoSeconds) {
iteratorMax.set(nanoSeconds);
}
}
/**
* amount of waiting
*
* @return millisecond
*/
public static long getIteratorWait() {
return iteratorSum.get() / 1_000_000;
}
/**
* average of waiting
*
* @return millisecond
*/
public static long getIteratorWaitAvg() {
if (iteratorCount.get() == 0) return -1;
return getIteratorWait() / iteratorCount.get();
}
/**
* maximum of waiting
*
* @return millisecond
*/
public static long getIteratorWaitMax() {
return iteratorMax.get() / 1_000_000;
}
public static long getIteratorCount() {
return iteratorCount.get();
}
public void countFail() {
this.failureCount.getAndIncrement();
}
public long getFailureCount() {
return this.failureCount.get();
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.client.util;
import java.util.concurrent.atomic.AtomicLong;
/**
* @author lynn.bond@hotmail.com on 2022/1/29
*/
public class MetricX {
public static AtomicLong iteratorSum = new AtomicLong();
public static AtomicLong iteratorCount = new AtomicLong();
public static AtomicLong iteratorMax = new AtomicLong();
public AtomicLong failureCount = new AtomicLong();
private long start;
private long end;
public static MetricX ofStart() {
return new MetricX(System.currentTimeMillis());
}
private MetricX(long start) {
this.start = start;
}
public long start() {
return this.start = System.currentTimeMillis();
}
public long end() {
return this.end = System.currentTimeMillis();
}
public long past() {
return this.end - this.start;
}
public static void plusIteratorWait(long nanoSeconds) {
iteratorSum.addAndGet(nanoSeconds);
iteratorCount.getAndIncrement();
if (iteratorMax.get() < nanoSeconds) {
iteratorMax.set(nanoSeconds);
}
}
/**
* amount of waiting
*
* @return millisecond
*/
public static long getIteratorWait() {
return iteratorSum.get() / 1_000_000;
}
/**
* average of waiting
*
* @return millisecond
*/
public static long getIteratorWaitAvg() {
if (iteratorCount.get() == 0) return -1;
return getIteratorWait() / iteratorCount.get();
}
/**
* maximum of waiting
*
* @return millisecond
*/
public static long getIteratorWaitMax() {
return iteratorMax.get() / 1_000_000;
}
public static long getIteratorCount() {
return iteratorCount.get();
}
public void countFail() {
this.failureCount.getAndIncrement();
}
public long getFailureCount() {
return this.failureCount.get();
}
}

204
hg-store-client/src/main/resources/log4j2.xml
View File

@ -1,102 +1,102 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<!-- Config will be auto loaded every 60s -->
<configuration status="error" monitorInterval="60">
<properties>
<property name="LOG_PATH">logs</property>
<property name="FILE_NAME">hg-store-client</property>
</properties>
<appenders>
<Console name="console" target="SYSTEM_OUT">
<ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
<!--JsonLayout compact="false" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout-->
</Console>
<!-- Normal server log config -->
<RollingRandomAccessFile name="file" fileName="${LOG_PATH}/${FILE_NAME}.log"
filePattern="${LOG_PATH}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log"
bufferedIO="false" immediateFlush="true">
<ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
<!--JsonLayout compact="true" eventEol="true" complete="true" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout-->
<!-- Trigger after exceeding 1day or 50MB -->
<Policies>
<SizeBasedTriggeringPolicy size="50MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<!-- Keep 5 files per day & auto Delete after over 2GB or 100 files -->
<DefaultRolloverStrategy max="5">
<Delete basePath="${LOG_PATH}" maxDepth="2">
<IfFileName glob="*/*.log"/>
<!-- Limit log amount & size -->
<IfAny>
<IfAccumulatedFileSize exceeds="2GB"/>
<IfAccumulatedFileCount exceeds="100"/>
</IfAny>
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
<!-- Separate & compress audit log, buffer size is 512KB -->
<RollingRandomAccessFile name="audit" fileName="${LOG_PATH}/audit-${FILE_NAME}.log"
filePattern="${LOG_PATH}/$${date:yyyy-MM}/audit-${FILE_NAME}-%d{yyyy-MM-dd-HH}-%i.gz"
bufferedIO="true" bufferSize="524288" immediateFlush="false">
<ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
<!-- Use simple format for audit log to speed up -->
<!-- PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} - %m%n"/ -->
<JsonLayout compact="true" eventEol="true" complete="false" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout>
<!-- Trigger after exceeding 1hour or 500MB -->
<Policies>
<SizeBasedTriggeringPolicy size="500MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<!-- Keep 2 files per hour & auto Delete [after 60 days] or [over 5GB or 500 files] -->
<DefaultRolloverStrategy max="2">
<Delete basePath="${LOG_PATH}" maxDepth="2">
<IfFileName glob="*/*.gz"/>
<IfLastModified age="60d"/>
<IfAny>
<IfAccumulatedFileSize exceeds="5GB"/>
<IfAccumulatedFileCount exceeds="500"/>
</IfAny>
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
</appenders>
<loggers>
<logger name="io.grpc.netty" level="ERROR" additivity="false">
<appender-ref ref="file"/>
</logger>
<root level="ERROR">
<appender-ref ref="console"/>
<appender-ref ref="file"/>
</root>
</loggers>
</configuration>
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<!-- Config will be auto loaded every 60s -->
<configuration status="error" monitorInterval="60">
<properties>
<property name="LOG_PATH">logs</property>
<property name="FILE_NAME">hg-store-client</property>
</properties>
<appenders>
<Console name="console" target="SYSTEM_OUT">
<ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
<!--JsonLayout compact="false" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout-->
</Console>
<!-- Normal server log config -->
<RollingRandomAccessFile name="file" fileName="${LOG_PATH}/${FILE_NAME}.log"
filePattern="${LOG_PATH}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log"
bufferedIO="false" immediateFlush="true">
<ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
<!--JsonLayout compact="true" eventEol="true" complete="true" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout-->
<!-- Trigger after exceeding 1day or 50MB -->
<Policies>
<SizeBasedTriggeringPolicy size="50MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<!-- Keep 5 files per day & auto Delete after over 2GB or 100 files -->
<DefaultRolloverStrategy max="5">
<Delete basePath="${LOG_PATH}" maxDepth="2">
<IfFileName glob="*/*.log"/>
<!-- Limit log amount & size -->
<IfAny>
<IfAccumulatedFileSize exceeds="2GB"/>
<IfAccumulatedFileCount exceeds="100"/>
</IfAny>
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
<!-- Separate & compress audit log, buffer size is 512KB -->
<RollingRandomAccessFile name="audit" fileName="${LOG_PATH}/audit-${FILE_NAME}.log"
filePattern="${LOG_PATH}/$${date:yyyy-MM}/audit-${FILE_NAME}-%d{yyyy-MM-dd-HH}-%i.gz"
bufferedIO="true" bufferSize="524288" immediateFlush="false">
<ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
<!-- Use simple format for audit log to speed up -->
<!-- PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} - %m%n"/ -->
<JsonLayout compact="true" eventEol="true" complete="false" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout>
<!-- Trigger after exceeding 1hour or 500MB -->
<Policies>
<SizeBasedTriggeringPolicy size="500MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<!-- Keep 2 files per hour & auto Delete [after 60 days] or [over 5GB or 500 files] -->
<DefaultRolloverStrategy max="2">
<Delete basePath="${LOG_PATH}" maxDepth="2">
<IfFileName glob="*/*.gz"/>
<IfLastModified age="60d"/>
<IfAny>
<IfAccumulatedFileSize exceeds="5GB"/>
<IfAccumulatedFileCount exceeds="500"/>
</IfAny>
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
</appenders>
<loggers>
<logger name="io.grpc.netty" level="ERROR" additivity="false">
<appender-ref ref="file"/>
</logger>
<root level="ERROR">
<appender-ref ref="console"/>
<appender-ref ref="file"/>
</root>
</loggers>
</configuration>

2248
hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerTest.java
View File

File diff suppressed because it is too large Load Diff

204
hg-store-client/src/test/resources/log4j2.xml
View File

@ -1,102 +1,102 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<!-- Config will be auto loaded every 60s -->
<configuration status="error" monitorInterval="60">
<properties>
<property name="LOG_PATH">logs</property>
<property name="FILE_NAME">hg-store-client</property>
</properties>
<appenders>
<Console name="console" target="SYSTEM_OUT">
<ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
<!--JsonLayout compact="false" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout-->
</Console>
<!-- Normal server log config -->
<RollingRandomAccessFile name="file" fileName="${LOG_PATH}/${FILE_NAME}.log"
filePattern="${LOG_PATH}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log"
bufferedIO="false" immediateFlush="true">
<ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
<!--JsonLayout compact="true" eventEol="true" complete="true" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout-->
<!-- Trigger after exceeding 1day or 50MB -->
<Policies>
<SizeBasedTriggeringPolicy size="50MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<!-- Keep 5 files per day & auto Delete after over 2GB or 100 files -->
<DefaultRolloverStrategy max="5">
<Delete basePath="${LOG_PATH}" maxDepth="2">
<IfFileName glob="*/*.log"/>
<!-- Limit log amount & size -->
<IfAny>
<IfAccumulatedFileSize exceeds="2GB"/>
<IfAccumulatedFileCount exceeds="100"/>
</IfAny>
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
<!-- Separate & compress audit log, buffer size is 512KB -->
<RollingRandomAccessFile name="audit" fileName="${LOG_PATH}/audit-${FILE_NAME}.log"
filePattern="${LOG_PATH}/$${date:yyyy-MM}/audit-${FILE_NAME}-%d{yyyy-MM-dd-HH}-%i.gz"
bufferedIO="true" bufferSize="524288" immediateFlush="false">
<ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
<!-- Use simple format for audit log to speed up -->
<!-- PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} - %m%n"/ -->
<JsonLayout compact="true" eventEol="true" complete="false" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout>
<!-- Trigger after exceeding 1hour or 500MB -->
<Policies>
<SizeBasedTriggeringPolicy size="500MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<!-- Keep 2 files per hour & auto Delete [after 60 days] or [over 5GB or 500 files] -->
<DefaultRolloverStrategy max="2">
<Delete basePath="${LOG_PATH}" maxDepth="2">
<IfFileName glob="*/*.gz"/>
<IfLastModified age="60d"/>
<IfAny>
<IfAccumulatedFileSize exceeds="5GB"/>
<IfAccumulatedFileCount exceeds="500"/>
</IfAny>
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
</appenders>
<loggers>
<logger name="io.grpc.netty" level="ERROR" additivity="false">
<appender-ref ref="file"/>
</logger>
<root level="INFO">
<appender-ref ref="console"/>
<appender-ref ref="file"/>
</root>
</loggers>
</configuration>
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<!-- Config will be auto loaded every 60s -->
<configuration status="error" monitorInterval="60">
<properties>
<property name="LOG_PATH">logs</property>
<property name="FILE_NAME">hg-store-client</property>
</properties>
<appenders>
<Console name="console" target="SYSTEM_OUT">
<ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
<!--JsonLayout compact="false" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout-->
</Console>
<!-- Normal server log config -->
<RollingRandomAccessFile name="file" fileName="${LOG_PATH}/${FILE_NAME}.log"
filePattern="${LOG_PATH}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log"
bufferedIO="false" immediateFlush="true">
<ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
<!--JsonLayout compact="true" eventEol="true" complete="true" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout-->
<!-- Trigger after exceeding 1day or 50MB -->
<Policies>
<SizeBasedTriggeringPolicy size="50MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<!-- Keep 5 files per day & auto Delete after over 2GB or 100 files -->
<DefaultRolloverStrategy max="5">
<Delete basePath="${LOG_PATH}" maxDepth="2">
<IfFileName glob="*/*.log"/>
<!-- Limit log amount & size -->
<IfAny>
<IfAccumulatedFileSize exceeds="2GB"/>
<IfAccumulatedFileCount exceeds="100"/>
</IfAny>
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
<!-- Separate & compress audit log, buffer size is 512KB -->
<RollingRandomAccessFile name="audit" fileName="${LOG_PATH}/audit-${FILE_NAME}.log"
filePattern="${LOG_PATH}/$${date:yyyy-MM}/audit-${FILE_NAME}-%d{yyyy-MM-dd-HH}-%i.gz"
bufferedIO="true" bufferSize="524288" immediateFlush="false">
<ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
<!-- Use simple format for audit log to speed up -->
<!-- PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} - %m%n"/ -->
<JsonLayout compact="true" eventEol="true" complete="false" locationInfo="true">
<KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/>
</JsonLayout>
<!-- Trigger after exceeding 1hour or 500MB -->
<Policies>
<SizeBasedTriggeringPolicy size="500MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true"/>
</Policies>
<!-- Keep 2 files per hour & auto Delete [after 60 days] or [over 5GB or 500 files] -->
<DefaultRolloverStrategy max="2">
<Delete basePath="${LOG_PATH}" maxDepth="2">
<IfFileName glob="*/*.gz"/>
<IfLastModified age="60d"/>
<IfAny>
<IfAccumulatedFileSize exceeds="5GB"/>
<IfAccumulatedFileCount exceeds="500"/>
</IfAny>
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
</appenders>
<loggers>
<logger name="io.grpc.netty" level="ERROR" additivity="false">
<appender-ref ref="file"/>
</logger>
<root level="INFO">
<appender-ref ref="console"/>
<appender-ref ref="file"/>
</root>
</loggers>
</configuration>

422
hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchGrpcClosure.java
View File

@ -1,212 +1,212 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Function;
import org.apache.commons.collections.CollectionUtils;
import org.apache.hugegraph.store.grpc.common.ResCode;
import org.apache.hugegraph.store.grpc.common.ResStatus;
import org.apache.hugegraph.store.grpc.session.FeedbackRes;
import org.apache.hugegraph.store.grpc.session.PartitionFaultResponse;
import org.apache.hugegraph.store.grpc.session.PartitionFaultType;
import org.apache.hugegraph.store.grpc.session.PartitionLeader;
import org.apache.hugegraph.store.raft.RaftClosure;
import org.apache.hugegraph.store.util.HgRaftError;
import com.alipay.sofa.jraft.Status;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
/**
* 批量处理的grpc回调封装类
*
* @param <V>
*/
@Slf4j
class BatchGrpcClosure<V> {
private final CountDownLatch countDownLatch;
private final List<Status> errorStatus;
private final List<V> results;
private final Map<Integer, Long> leaderMap;
public BatchGrpcClosure(int count) {
countDownLatch = new CountDownLatch(count);
errorStatus = Collections.synchronizedList(new ArrayList<>());
results = Collections.synchronizedList(new ArrayList<>());
leaderMap = new ConcurrentHashMap<>();
}
public RaftClosure newRaftClosure() {
return new GrpcClosure<V>() {
@Override
public void run(Status status) {
countDownLatch.countDown();
if (status.isOk()) {
results.add(this.getResult());
} else {
leaderMap.putAll(this.getLeaderMap());
errorStatus.add(status);
}
}
};
}
public RaftClosure newRaftClosure(Consumer<Status> ok) {
return new GrpcClosure<V>() {
@Override
public void run(Status status) {
countDownLatch.countDown();
if (status.isOk()) {
results.add(this.getResult());
} else {
leaderMap.putAll(this.getLeaderMap());
errorStatus.add(status);
}
ok.accept(status);
}
};
}
/**
* 不使用计数器latch
*
* @return
*/
public RaftClosure newClosureNoLatch() {
return new GrpcClosure<V>() {
@Override
public void run(Status status) {
if (status.isOk()) {
results.add(this.getResult());
} else {
leaderMap.putAll(this.getLeaderMap());
errorStatus.add(status);
}
}
};
}
public PartitionFaultResponse getErrorResponse() {
PartitionFaultResponse errorResponse;
if (leaderMap.size() > 0) {
PartitionFaultResponse.Builder partitionFault =
PartitionFaultResponse.newBuilder().setFaultType(
PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LEADER);
leaderMap.forEach((k, v) -> {
partitionFault.addPartitionLeaders(PartitionLeader.newBuilder()
.setPartitionId(k)
.setLeaderId(v).build());
});
errorResponse = partitionFault.build();
} else {
PartitionFaultType faultType = PartitionFaultType.PARTITION_FAULT_TYPE_UNKNOWN;
switch (HgRaftError.forNumber(errorStatus.get(0).getCode())) {
case NOT_LEADER:
faultType = PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LEADER;
break;
case WAIT_LEADER_TIMEOUT:
faultType = PartitionFaultType.PARTITION_FAULT_TYPE_WAIT_LEADER_TIMEOUT;
break;
case NOT_LOCAL:
faultType = PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LOCAL;
break;
default:
log.error("Unmatchable errorStatus: " + errorStatus);
}
errorResponse = PartitionFaultResponse.newBuilder().setFaultType(faultType).build();
}
return errorResponse;
}
public String getErrorMsg() {
StringBuilder builder = new StringBuilder();
errorStatus.forEach(status -> {
if (!status.isOk()) {
builder.append(status.getErrorMsg());
builder.append("\n");
}
});
return builder.toString();
}
/**
* 等待raft执行结束，返回结果给grpc
*/
public void waitFinish(StreamObserver<V> observer, Function<List<V>, V> ok, long timeout) {
try {
countDownLatch.await(timeout, TimeUnit.MILLISECONDS);
if (errorStatus.isEmpty()) { // 没有错误时，合并结果
observer.onNext(ok.apply(results));
} else {
observer.onNext((V) FeedbackRes.newBuilder()
.setStatus(ResStatus.newBuilder()
.setCode(ResCode.RES_CODE_FAIL)
.setMsg(getErrorMsg()))
.setPartitionFaultResponse(this.getErrorResponse())
.build());
}
} catch (InterruptedException e) {
log.error("waitFinish exception: ", e);
observer.onNext((V) FeedbackRes.newBuilder()
.setStatus(ResStatus.newBuilder()
.setCode(ResCode.RES_CODE_FAIL)
.setMsg(e.getLocalizedMessage())
.build()).build());
}
observer.onCompleted();
}
/**
* 从多个结果中选择一个错误的结果返回，如果没有错误，返回第一个
*/
public FeedbackRes selectError(List<FeedbackRes> results) {
if (!CollectionUtils.isEmpty(results)) {
AtomicReference<FeedbackRes> res = new AtomicReference<>(results.get(0));
results.forEach(e -> {
try {
if (e.getStatus().getCode() != ResCode.RES_CODE_OK) {
res.set(e);
}
} catch (Exception ex) {
log.error("{}", ex);
}
});
return res.get();
} else {
return FeedbackRes.newBuilder()
.setStatus(ResStatus.newBuilder()
.setCode(ResCode.RES_CODE_OK).build())
.build();
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Function;
import org.apache.commons.collections.CollectionUtils;
import org.apache.hugegraph.store.grpc.common.ResCode;
import org.apache.hugegraph.store.grpc.common.ResStatus;
import org.apache.hugegraph.store.grpc.session.FeedbackRes;
import org.apache.hugegraph.store.grpc.session.PartitionFaultResponse;
import org.apache.hugegraph.store.grpc.session.PartitionFaultType;
import org.apache.hugegraph.store.grpc.session.PartitionLeader;
import org.apache.hugegraph.store.raft.RaftClosure;
import org.apache.hugegraph.store.util.HgRaftError;
import com.alipay.sofa.jraft.Status;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
/**
* 批量处理的grpc回调封装类
*
* @param <V>
*/
@Slf4j
class BatchGrpcClosure<V> {
private final CountDownLatch countDownLatch;
private final List<Status> errorStatus;
private final List<V> results;
private final Map<Integer, Long> leaderMap;
public BatchGrpcClosure(int count) {
countDownLatch = new CountDownLatch(count);
errorStatus = Collections.synchronizedList(new ArrayList<>());
results = Collections.synchronizedList(new ArrayList<>());
leaderMap = new ConcurrentHashMap<>();
}
public RaftClosure newRaftClosure() {
return new GrpcClosure<V>() {
@Override
public void run(Status status) {
countDownLatch.countDown();
if (status.isOk()) {
results.add(this.getResult());
} else {
leaderMap.putAll(this.getLeaderMap());
errorStatus.add(status);
}
}
};
}
public RaftClosure newRaftClosure(Consumer<Status> ok) {
return new GrpcClosure<V>() {
@Override
public void run(Status status) {
countDownLatch.countDown();
if (status.isOk()) {
results.add(this.getResult());
} else {
leaderMap.putAll(this.getLeaderMap());
errorStatus.add(status);
}
ok.accept(status);
}
};
}
/**
* 不使用计数器latch
*
* @return
*/
public RaftClosure newClosureNoLatch() {
return new GrpcClosure<V>() {
@Override
public void run(Status status) {
if (status.isOk()) {
results.add(this.getResult());
} else {
leaderMap.putAll(this.getLeaderMap());
errorStatus.add(status);
}
}
};
}
public PartitionFaultResponse getErrorResponse() {
PartitionFaultResponse errorResponse;
if (leaderMap.size() > 0) {
PartitionFaultResponse.Builder partitionFault =
PartitionFaultResponse.newBuilder().setFaultType(
PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LEADER);
leaderMap.forEach((k, v) -> {
partitionFault.addPartitionLeaders(PartitionLeader.newBuilder()
.setPartitionId(k)
.setLeaderId(v).build());
});
errorResponse = partitionFault.build();
} else {
PartitionFaultType faultType = PartitionFaultType.PARTITION_FAULT_TYPE_UNKNOWN;
switch (HgRaftError.forNumber(errorStatus.get(0).getCode())) {
case NOT_LEADER:
faultType = PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LEADER;
break;
case WAIT_LEADER_TIMEOUT:
faultType = PartitionFaultType.PARTITION_FAULT_TYPE_WAIT_LEADER_TIMEOUT;
break;
case NOT_LOCAL:
faultType = PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LOCAL;
break;
default:
log.error("Unmatchable errorStatus: " + errorStatus);
}
errorResponse = PartitionFaultResponse.newBuilder().setFaultType(faultType).build();
}
return errorResponse;
}
public String getErrorMsg() {
StringBuilder builder = new StringBuilder();
errorStatus.forEach(status -> {
if (!status.isOk()) {
builder.append(status.getErrorMsg());
builder.append("\n");
}
});
return builder.toString();
}
/**
* 等待raft执行结束，返回结果给grpc
*/
public void waitFinish(StreamObserver<V> observer, Function<List<V>, V> ok, long timeout) {
try {
countDownLatch.await(timeout, TimeUnit.MILLISECONDS);
if (errorStatus.isEmpty()) { // 没有错误时，合并结果
observer.onNext(ok.apply(results));
} else {
observer.onNext((V) FeedbackRes.newBuilder()
.setStatus(ResStatus.newBuilder()
.setCode(ResCode.RES_CODE_FAIL)
.setMsg(getErrorMsg()))
.setPartitionFaultResponse(this.getErrorResponse())
.build());
}
} catch (InterruptedException e) {
log.error("waitFinish exception: ", e);
observer.onNext((V) FeedbackRes.newBuilder()
.setStatus(ResStatus.newBuilder()
.setCode(ResCode.RES_CODE_FAIL)
.setMsg(e.getLocalizedMessage())
.build()).build());
}
observer.onCompleted();
}
/**
* 从多个结果中选择一个错误的结果返回，如果没有错误，返回第一个
*/
public FeedbackRes selectError(List<FeedbackRes> results) {
if (!CollectionUtils.isEmpty(results)) {
AtomicReference<FeedbackRes> res = new AtomicReference<>(results.get(0));
results.forEach(e -> {
try {
if (e.getStatus().getCode() != ResCode.RES_CODE_OK) {
res.set(e);
}
} catch (Exception ex) {
log.error("{}", ex);
}
});
return res.get();
} else {
return FeedbackRes.newBuilder()
.setStatus(ResStatus.newBuilder()
.setCode(ResCode.RES_CODE_OK).build())
.build();
}
}
}

320
hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchScanIterator.java
View File

@ -1,160 +1,160 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import java.util.NoSuchElementException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.pd.common.KVPair;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.node.util.HgAssert;
import org.apache.hugegraph.store.node.util.HgStoreConst;
/**
* @author lynn.bond@hotmail.com on 2022/2/28
* @version 0.3.0 added limit support
*/
@NotThreadSafe
public final class BatchScanIterator implements ScanIterator {
private final Supplier<KVPair<QueryCondition, ScanIterator>> batchSupplier;
private final Supplier<Long> limitSupplier;
private ScanIterator iterator;
private boolean hasNext = false;
private long curCount;
private long curLimit;
private final AtomicBoolean closed = new AtomicBoolean();
public static BatchScanIterator of(
Supplier<KVPair<QueryCondition, ScanIterator>> iteratorSupplier,
Supplier<Long> limitSupplier) {
HgAssert.isArgumentNotNull(iteratorSupplier, "iteratorSupplier");
HgAssert.isArgumentNotNull(limitSupplier, "limitSupplier");
return new BatchScanIterator(iteratorSupplier, limitSupplier);
}
private BatchScanIterator(Supplier<KVPair<QueryCondition, ScanIterator>> iteratorSupplier,
Supplier<Long> limitSupplier) {
this.batchSupplier = iteratorSupplier;
this.limitSupplier = limitSupplier;
}
private ScanIterator getIterator() {
ScanIterator buf;
int count = 0;
this.curCount = 0L;
do {
buf = this.batchSupplier.get().getValue();
if (buf == null) {
break;
}
if (!buf.hasNext()) {
buf.close();
buf = null;
}
if (++count == Integer.MAX_VALUE) {
throw new RuntimeException("Do loop times more than Integer.MAX_VALUE");
}
} while (buf == null);
if (buf != null) {
Long limit = this.limitSupplier.get();
if (limit == null || limit <= 0) {
this.curLimit = Integer.MAX_VALUE;
} else {
this.curLimit = limit;
}
}
return buf;
}
@Override
public boolean hasNext() {
if (this.iterator == null) {
this.iterator = this.getIterator();
} else if (!this.iterator.hasNext()) {
this.iterator.close();
this.iterator = this.getIterator();
} else if (this.curCount == this.curLimit) {
this.iterator.close();
this.iterator = this.getIterator();
}
if (this.iterator == null) {
return false;
} else {
this.hasNext = true;
return true;
}
}
@Override
public <T> T next() {
if (this.hasNext) {
this.hasNext = false;
} else {
if (!this.hasNext()) {
throw new NoSuchElementException();
}
}
this.curCount++;
return this.iterator.next();
}
@Override
public void close() {
if (!this.closed.getAndSet(true)) {
if (this.iterator != null) {
this.iterator.close();
}
}
}
@Override
public boolean isValid() {
throw new UnsupportedOperationException();
}
@Override
public long count() {
throw new UnsupportedOperationException();
}
@Override
public byte[] position() {
if (this.iterator != null) {
return this.iterator.position();
}
return HgStoreConst.EMPTY_BYTES;
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import java.util.NoSuchElementException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.pd.common.KVPair;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.node.util.HgAssert;
import org.apache.hugegraph.store.node.util.HgStoreConst;
/**
* @author lynn.bond@hotmail.com on 2022/2/28
* @version 0.3.0 added limit support
*/
@NotThreadSafe
public final class BatchScanIterator implements ScanIterator {
private final Supplier<KVPair<QueryCondition, ScanIterator>> batchSupplier;
private final Supplier<Long> limitSupplier;
private ScanIterator iterator;
private boolean hasNext = false;
private long curCount;
private long curLimit;
private final AtomicBoolean closed = new AtomicBoolean();
public static BatchScanIterator of(
Supplier<KVPair<QueryCondition, ScanIterator>> iteratorSupplier,
Supplier<Long> limitSupplier) {
HgAssert.isArgumentNotNull(iteratorSupplier, "iteratorSupplier");
HgAssert.isArgumentNotNull(limitSupplier, "limitSupplier");
return new BatchScanIterator(iteratorSupplier, limitSupplier);
}
private BatchScanIterator(Supplier<KVPair<QueryCondition, ScanIterator>> iteratorSupplier,
Supplier<Long> limitSupplier) {
this.batchSupplier = iteratorSupplier;
this.limitSupplier = limitSupplier;
}
private ScanIterator getIterator() {
ScanIterator buf;
int count = 0;
this.curCount = 0L;
do {
buf = this.batchSupplier.get().getValue();
if (buf == null) {
break;
}
if (!buf.hasNext()) {
buf.close();
buf = null;
}
if (++count == Integer.MAX_VALUE) {
throw new RuntimeException("Do loop times more than Integer.MAX_VALUE");
}
} while (buf == null);
if (buf != null) {
Long limit = this.limitSupplier.get();
if (limit == null || limit <= 0) {
this.curLimit = Integer.MAX_VALUE;
} else {
this.curLimit = limit;
}
}
return buf;
}
@Override
public boolean hasNext() {
if (this.iterator == null) {
this.iterator = this.getIterator();
} else if (!this.iterator.hasNext()) {
this.iterator.close();
this.iterator = this.getIterator();
} else if (this.curCount == this.curLimit) {
this.iterator.close();
this.iterator = this.getIterator();
}
if (this.iterator == null) {
return false;
} else {
this.hasNext = true;
return true;
}
}
@Override
public <T> T next() {
if (this.hasNext) {
this.hasNext = false;
} else {
if (!this.hasNext()) {
throw new NoSuchElementException();
}
}
this.curCount++;
return this.iterator.next();
}
@Override
public void close() {
if (!this.closed.getAndSet(true)) {
if (this.iterator != null) {
this.iterator.close();
}
}
}
@Override
public boolean isValid() {
throw new UnsupportedOperationException();
}
@Override
public long count() {
throw new UnsupportedOperationException();
}
@Override
public byte[] position() {
if (this.iterator != null) {
return this.iterator.position();
}
return HgStoreConst.EMPTY_BYTES;
}
}

92
hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GRpcServerConfig.java
View File

@ -1,46 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import org.apache.hugegraph.store.node.AppConfig;
import org.apache.hugegraph.store.node.util.HgExecutorUtil;
import org.lognet.springboot.grpc.GRpcServerBuilderConfigurer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import io.grpc.ServerBuilder;
/**
* @author lynn.bond@hotmail.com on 2022/3/4
*/
@Component
public class GRpcServerConfig extends GRpcServerBuilderConfigurer {
public final static String EXECUTOR_NAME = "hg-grpc";
@Autowired
private AppConfig appConfig;
@Override
public void configure(ServerBuilder<?> serverBuilder) {
AppConfig.ThreadPoolGrpc grpc = appConfig.getThreadPoolGrpc();
serverBuilder.executor(
HgExecutorUtil.createExecutor(EXECUTOR_NAME, grpc.getCore(), grpc.getMax(),
grpc.getQueue())
);
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import org.apache.hugegraph.store.node.AppConfig;
import org.apache.hugegraph.store.node.util.HgExecutorUtil;
import org.lognet.springboot.grpc.GRpcServerBuilderConfigurer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import io.grpc.ServerBuilder;
/**
* @author lynn.bond@hotmail.com on 2022/3/4
*/
@Component
public class GRpcServerConfig extends GRpcServerBuilderConfigurer {
public final static String EXECUTOR_NAME = "hg-grpc";
@Autowired
private AppConfig appConfig;
@Override
public void configure(ServerBuilder<?> serverBuilder) {
AppConfig.ThreadPoolGrpc grpc = appConfig.getThreadPoolGrpc();
serverBuilder.executor(
HgExecutorUtil.createExecutor(EXECUTOR_NAME, grpc.getCore(), grpc.getMax(),
grpc.getQueue())
);
}
}

142
hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcClosure.java
View File

@ -1,72 +1,72 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hugegraph.store.grpc.session.FeedbackRes;
import org.apache.hugegraph.store.raft.RaftClosure;
import io.grpc.stub.StreamObserver;
/**
* @author lynn.bond@hotmail.com on 2022/1/27
*/
abstract class GrpcClosure<V> implements RaftClosure {
private V result;
private final Map<Integer, Long> leaderMap = new HashMap<>();
public V getResult() {
return result;
}
public Map<Integer, Long> getLeaderMap() {
return leaderMap;
}
@Override
public void onLeaderChanged(Integer partId, Long storeId) {
leaderMap.put(partId, storeId);
}
public void setResult(V result) {
this.result = result;
}
/**
* 设置输出结果给raftClosure，对于Follower来说，raftClosure为空
*/
public static <V> void setResult(RaftClosure raftClosure, V result) {
GrpcClosure closure = (GrpcClosure) raftClosure;
if (closure != null) {
closure.setResult(result);
}
}
public static <V> RaftClosure newRaftClosure(StreamObserver<V> observer) {
BatchGrpcClosure<V> wrap = new BatchGrpcClosure<V>(0);
return wrap.newRaftClosure(s -> {
wrap.waitFinish(observer, r -> {
return (V) wrap.selectError((List<FeedbackRes>) r);
}, 0);
});
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hugegraph.store.grpc.session.FeedbackRes;
import org.apache.hugegraph.store.raft.RaftClosure;
import io.grpc.stub.StreamObserver;
/**
* @author lynn.bond@hotmail.com on 2022/1/27
*/
abstract class GrpcClosure<V> implements RaftClosure {
private V result;
private final Map<Integer, Long> leaderMap = new HashMap<>();
public V getResult() {
return result;
}
public Map<Integer, Long> getLeaderMap() {
return leaderMap;
}
@Override
public void onLeaderChanged(Integer partId, Long storeId) {
leaderMap.put(partId, storeId);
}
public void setResult(V result) {
this.result = result;
}
/**
* 设置输出结果给raftClosure，对于Follower来说，raftClosure为空
*/
public static <V> void setResult(RaftClosure raftClosure, V result) {
GrpcClosure closure = (GrpcClosure) raftClosure;
if (closure != null) {
closure.setResult(result);
}
}
public static <V> RaftClosure newRaftClosure(StreamObserver<V> observer) {
BatchGrpcClosure<V> wrap = new BatchGrpcClosure<V>(0);
return wrap.newRaftClosure(s -> {
wrap.waitFinish(observer, r -> {
return (V) wrap.selectError((List<FeedbackRes>) r);
}, 0);
});
}
}

816
hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java
View File

@ -1,409 +1,409 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import java.util.List;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.grpc.common.Kv;
import org.apache.hugegraph.store.grpc.stream.KvPageRes;
import org.apache.hugegraph.store.grpc.stream.ScanCondition;
import org.apache.hugegraph.store.grpc.stream.ScanQueryRequest;
import org.apache.hugegraph.store.grpc.stream.ScanStreamBatchReq;
import org.apache.hugegraph.store.node.util.Base58;
import org.apache.hugegraph.store.node.util.HgAssert;
import org.apache.hugegraph.store.node.util.HgGrpc;
import org.apache.hugegraph.store.node.util.HgStoreConst;
import org.apache.hugegraph.store.node.util.HgStoreNodeUtil;
import com.google.protobuf.ByteString;
import io.grpc.Status;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2022/03/27
* @version 3.6.0
*/
@Slf4j
public class ScanBatchResponse3 {
private final static long DEFAULT_PACKAGE_SIZE = 10_000;
private final static int MAX_NOT_RECEIPT = 10;
public static StreamObserver of(StreamObserver<KvPageRes> responseObserver,
HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) {
HgAssert.isArgumentNotNull(responseObserver, "responseObserver");
HgAssert.isArgumentNotNull(wrapper, "wrapper");
return new Broker(responseObserver, wrapper, executor);
}
private enum OrderState {
NEW(0),
WORKING(1);//, PAUSE(2), COMPLETE(10);
int value;
OrderState(int value) {
this.value = value;
}
}
/*** Broker ***/
private static class Broker implements StreamObserver<ScanStreamBatchReq> {
private final StreamObserver<KvPageRes> responseObserver;
private final HgStoreWrapperEx wrapper;
private final ThreadPoolExecutor executor;
private String graph;
private final OrderManager manager = new OrderManager();
Broker(StreamObserver<KvPageRes> responseObserver, HgStoreWrapperEx wrapper,
ThreadPoolExecutor executor) {
this.responseObserver = responseObserver;
this.wrapper = wrapper;
this.executor = executor;
}
@Override
public void onNext(ScanStreamBatchReq request) {
this.handleHeader(request);
switch (request.getQueryCase()) {
case QUERY_REQUEST:
this.makeADeal(request.getQueryRequest());
break;
case RECEIPT_REQUEST:
this.manager.receipt(request.getReceiptRequest().getTimes());
break;
case CANCEL_REQUEST:
this.manager.finished();
break;
default:
responseObserver.onError(
HgGrpc.toErr("Unsupported sub-request: [ " + request + " ]"));
}
}
@Override
public void onError(Throwable t) {
log.warn(t.getMessage());
this.manager.breakdown();
}
@Override
public void onCompleted() {
this.manager.finished();
}
private void handleHeader(ScanStreamBatchReq request) {
if (this.graph == null) {
this.graph = request.getHeader().getGraph();
}
}
private void makeADeal(ScanQueryRequest request) {
String deliverId = "";
if (log.isDebugEnabled()) {
List<ScanCondition> conditions = request.getConditionList();
if (conditions.size() > 0) {
ScanCondition c = conditions.get(0);
if (c.getPrefix() != null && c.getPrefix().size() > 0) {
deliverId = Base58.encode(c.getPrefix().toByteArray());
log.info("[ANALYSIS DEAL] [{}] prefixLength: {}", deliverId,
conditions.size());
}
}
}
OrderDeliverer deliverer = new OrderDeliverer(deliverId, this.responseObserver);
OrderWorker worker = new OrderWorker(
request.getLimit(),
request.getPageSize(),
ScanUtil.getIterator(this.graph, request, this.wrapper),
deliverer,
this.executor);
this.manager.deal(worker, deliverer);
}
}
@NotThreadSafe
private static class OrderManager {
OrderState state = OrderState.NEW;
OrderWorker worker;
OrderDeliverer deliverer;
synchronized void deal(OrderWorker worker, OrderDeliverer deliverer) {
if (log.isDebugEnabled()) log.debug("Receiving query request.");
if (this.state == OrderState.NEW) {
this.worker = worker;
this.deliverer = deliverer;
this.worker.hereWeGo();
this.state = OrderState.WORKING;
}
}
synchronized void receipt(int receiptTimes) {
if (log.isDebugEnabled()) log.debug("Receiving receipt request.");
this.worker.setReceipt(receiptTimes);
}
synchronized void finished() {
if (log.isDebugEnabled()) log.debug("Receiving finished request.");
/* if (this.state.value > OrderState.NEW.value
&& this.state.value < OrderState.COMPLETE.value) {
this.state = OrderState.COMPLETE;
}*/
this.breakdown();
}
synchronized void breakdown() {
if (this.worker != null) {
this.worker.breakdown();
}
}
}
private static class OrderDeliverer {
private final StreamObserver<KvPageRes> responseObserver;
private final AtomicBoolean finishFlag = new AtomicBoolean();
private final String delivererId;
private final AtomicLong count = new AtomicLong();
OrderDeliverer(String delivererId, StreamObserver<KvPageRes> responseObserver) {
this.responseObserver = responseObserver;
this.delivererId = delivererId;
}
void deliver(KvPageRes.Builder dataBuilder, int times, boolean isOver) {
if (this.finishFlag.get()) {
return;
}
count.addAndGet(dataBuilder.getDataCount());
this.responseObserver.onNext(dataBuilder.setOver(isOver).setTimes(times).build());
if (log.isDebugEnabled()) log.debug("deliver times : {}, over: {}", times, isOver);
if (isOver) {
if (log.isDebugEnabled()) {
if (delivererId != null && !delivererId.isEmpty()) {
log.debug("[ANALYSIS OVER] [{}] count: {}, times: {}", delivererId, count,
times);
}
}
this.finish();
}
}
void finish() {
if (!finishFlag.getAndSet(true)) {
this.responseObserver.onCompleted();
}
}
void error(String msg) {
if (!finishFlag.getAndSet(true)) {
this.responseObserver.onError(HgGrpc.toErr(msg));
}
}
void error(String msg, Throwable t) {
if (!finishFlag.getAndSet(true)) {
this.responseObserver.onError(HgGrpc.toErr(Status.INTERNAL,
msg, t));
}
}
}
/*** Worker ***/
private static class OrderWorker {
private final ScanIterator iterator;
private final OrderDeliverer deliverer;
private final AtomicBoolean pauseFlag = new AtomicBoolean();
private final AtomicBoolean completeFlag = new AtomicBoolean();
private final ReentrantLock workingLock = new ReentrantLock();
private final AtomicBoolean isWorking = new AtomicBoolean();
private final AtomicBoolean breakdown = new AtomicBoolean();
private final AtomicInteger receiptTimes = new AtomicInteger();
private final AtomicInteger curTimes = new AtomicInteger();
private final ThreadPoolExecutor executor;
private final long limit;
private long packageSize;
private long counter;
OrderWorker(long limit, long packageSize, ScanIterator iterator, OrderDeliverer deliverer,
ThreadPoolExecutor executor) {
this.limit = limit;
this.packageSize = packageSize;
this.iterator = iterator;
this.deliverer = deliverer;
this.executor = executor;
if (this.packageSize <= 0) {
this.packageSize = DEFAULT_PACKAGE_SIZE;
log.warn(
"As page-Size is less than or equals 0, default package-size was " +
"effective.[ {} ]",
DEFAULT_PACKAGE_SIZE);
}
}
void hereWeGo() {
if (this.completeFlag.get()) {
log.warn("job complete.");
return;
}
if (this.isWorking.get()) {
log.warn("has been working.");
return;
}
if (this.workingLock.isLocked()) {
log.warn("working now");
return;
}
executor.execute(() -> working());
Thread.yield();
}
void setReceipt(int times) {
this.receiptTimes.set(times);
this.continueWorking();
}
boolean checkContinue() {
return (this.curTimes.get() - this.receiptTimes.get() < MAX_NOT_RECEIPT);
}
void continueWorking() {
if (this.checkContinue()) {
synchronized (this.iterator) {
this.iterator.notify();
}
}
}
void breakdown() {
this.breakdown.set(true);
synchronized (this.iterator) {
this.iterator.notify();
}
}
private void working() {
if (this.isWorking.getAndSet(true)) {
return;
}
this.workingLock.lock();
try {
synchronized (this.iterator) {
KvPageRes.Builder dataBuilder = KvPageRes.newBuilder();
Kv.Builder kvBuilder = Kv.newBuilder();
long packageCount = 0;
while (iterator.hasNext()) {
if (++this.counter > limit) {
this.completeFlag.set(true);
break;
}
if (++packageCount > packageSize) {
if (this.breakdown.get()) {
break;
}
deliverer.deliver(dataBuilder, curTimes.incrementAndGet(), false);
Thread.yield();
if (!this.checkContinue()) {
long start = System.currentTimeMillis();
iterator.wait(
HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS *
1000);
if (System.currentTimeMillis() - start
>=
HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * 1000) {
throw new TimeoutException("Waiting continue more than "
+
HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS +
" seconds.");
}
if (this.breakdown.get()) {
break;
}
}
packageCount = 1;
dataBuilder = KvPageRes.newBuilder();
}
RocksDBSession.BackendColumn col = iterator.next();
dataBuilder.addData(kvBuilder
.setKey(ByteString.copyFrom(col.name))
.setValue(ByteString.copyFrom(col.value))
.setCode(HgStoreNodeUtil.toInt(
iterator.position()))
//position == partition-id.
);
}
this.completeFlag.set(true);
deliverer.deliver(dataBuilder, curTimes.incrementAndGet(), true);
}
} catch (InterruptedException e) {
log.error("Interrupted waiting of iterator, canceled while.", e);
this.deliverer.error("Failed to finish scanning, cause by InterruptedException.");
} catch (TimeoutException t) {
log.info(t.getMessage());
this.deliverer.error("Sever waiting exceeded ["
+ HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS +
"] seconds.");
} catch (Throwable t) {
log.error("Failed to do while for scanning, cause by:", t);
this.deliverer.error("Failed to finish scanning ", t);
} finally {
this.workingLock.unlock();
this.iterator.close();
}
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import java.util.List;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.rocksdb.access.ScanIterator;
import org.apache.hugegraph.store.grpc.common.Kv;
import org.apache.hugegraph.store.grpc.stream.KvPageRes;
import org.apache.hugegraph.store.grpc.stream.ScanCondition;
import org.apache.hugegraph.store.grpc.stream.ScanQueryRequest;
import org.apache.hugegraph.store.grpc.stream.ScanStreamBatchReq;
import org.apache.hugegraph.store.node.util.Base58;
import org.apache.hugegraph.store.node.util.HgAssert;
import org.apache.hugegraph.store.node.util.HgGrpc;
import org.apache.hugegraph.store.node.util.HgStoreConst;
import org.apache.hugegraph.store.node.util.HgStoreNodeUtil;
import com.google.protobuf.ByteString;
import io.grpc.Status;
import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com created on 2022/03/27
* @version 3.6.0
*/
@Slf4j
public class ScanBatchResponse3 {
private final static long DEFAULT_PACKAGE_SIZE = 10_000;
private final static int MAX_NOT_RECEIPT = 10;
public static StreamObserver of(StreamObserver<KvPageRes> responseObserver,
HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) {
HgAssert.isArgumentNotNull(responseObserver, "responseObserver");
HgAssert.isArgumentNotNull(wrapper, "wrapper");
return new Broker(responseObserver, wrapper, executor);
}
private enum OrderState {
NEW(0),
WORKING(1);//, PAUSE(2), COMPLETE(10);
int value;
OrderState(int value) {
this.value = value;
}
}
/*** Broker ***/
private static class Broker implements StreamObserver<ScanStreamBatchReq> {
private final StreamObserver<KvPageRes> responseObserver;
private final HgStoreWrapperEx wrapper;
private final ThreadPoolExecutor executor;
private String graph;
private final OrderManager manager = new OrderManager();
Broker(StreamObserver<KvPageRes> responseObserver, HgStoreWrapperEx wrapper,
ThreadPoolExecutor executor) {
this.responseObserver = responseObserver;
this.wrapper = wrapper;
this.executor = executor;
}
@Override
public void onNext(ScanStreamBatchReq request) {
this.handleHeader(request);
switch (request.getQueryCase()) {
case QUERY_REQUEST:
this.makeADeal(request.getQueryRequest());
break;
case RECEIPT_REQUEST:
this.manager.receipt(request.getReceiptRequest().getTimes());
break;
case CANCEL_REQUEST:
this.manager.finished();
break;
default:
responseObserver.onError(
HgGrpc.toErr("Unsupported sub-request: [ " + request + " ]"));
}
}
@Override
public void onError(Throwable t) {
log.warn(t.getMessage());
this.manager.breakdown();
}
@Override
public void onCompleted() {
this.manager.finished();
}
private void handleHeader(ScanStreamBatchReq request) {
if (this.graph == null) {
this.graph = request.getHeader().getGraph();
}
}
private void makeADeal(ScanQueryRequest request) {
String deliverId = "";
if (log.isDebugEnabled()) {
List<ScanCondition> conditions = request.getConditionList();
if (conditions.size() > 0) {
ScanCondition c = conditions.get(0);
if (c.getPrefix() != null && c.getPrefix().size() > 0) {
deliverId = Base58.encode(c.getPrefix().toByteArray());
log.info("[ANALYSIS DEAL] [{}] prefixLength: {}", deliverId,
conditions.size());
}
}
}
OrderDeliverer deliverer = new OrderDeliverer(deliverId, this.responseObserver);
OrderWorker worker = new OrderWorker(
request.getLimit(),
request.getPageSize(),
ScanUtil.getIterator(this.graph, request, this.wrapper),
deliverer,
this.executor);
this.manager.deal(worker, deliverer);
}
}
@NotThreadSafe
private static class OrderManager {
OrderState state = OrderState.NEW;
OrderWorker worker;
OrderDeliverer deliverer;
synchronized void deal(OrderWorker worker, OrderDeliverer deliverer) {
if (log.isDebugEnabled()) log.debug("Receiving query request.");
if (this.state == OrderState.NEW) {
this.worker = worker;
this.deliverer = deliverer;
this.worker.hereWeGo();
this.state = OrderState.WORKING;
}
}
synchronized void receipt(int receiptTimes) {
if (log.isDebugEnabled()) log.debug("Receiving receipt request.");
this.worker.setReceipt(receiptTimes);
}
synchronized void finished() {
if (log.isDebugEnabled()) log.debug("Receiving finished request.");
/* if (this.state.value > OrderState.NEW.value
&& this.state.value < OrderState.COMPLETE.value) {
this.state = OrderState.COMPLETE;
}*/
this.breakdown();
}
synchronized void breakdown() {
if (this.worker != null) {
this.worker.breakdown();
}
}
}
private static class OrderDeliverer {
private final StreamObserver<KvPageRes> responseObserver;
private final AtomicBoolean finishFlag = new AtomicBoolean();
private final String delivererId;
private final AtomicLong count = new AtomicLong();
OrderDeliverer(String delivererId, StreamObserver<KvPageRes> responseObserver) {
this.responseObserver = responseObserver;
this.delivererId = delivererId;
}
void deliver(KvPageRes.Builder dataBuilder, int times, boolean isOver) {
if (this.finishFlag.get()) {
return;
}
count.addAndGet(dataBuilder.getDataCount());
this.responseObserver.onNext(dataBuilder.setOver(isOver).setTimes(times).build());
if (log.isDebugEnabled()) log.debug("deliver times : {}, over: {}", times, isOver);
if (isOver) {
if (log.isDebugEnabled()) {
if (delivererId != null && !delivererId.isEmpty()) {
log.debug("[ANALYSIS OVER] [{}] count: {}, times: {}", delivererId, count,
times);
}
}
this.finish();
}
}
void finish() {
if (!finishFlag.getAndSet(true)) {
this.responseObserver.onCompleted();
}
}
void error(String msg) {
if (!finishFlag.getAndSet(true)) {
this.responseObserver.onError(HgGrpc.toErr(msg));
}
}
void error(String msg, Throwable t) {
if (!finishFlag.getAndSet(true)) {
this.responseObserver.onError(HgGrpc.toErr(Status.INTERNAL,
msg, t));
}
}
}
/*** Worker ***/
private static class OrderWorker {
private final ScanIterator iterator;
private final OrderDeliverer deliverer;
private final AtomicBoolean pauseFlag = new AtomicBoolean();
private final AtomicBoolean completeFlag = new AtomicBoolean();
private final ReentrantLock workingLock = new ReentrantLock();
private final AtomicBoolean isWorking = new AtomicBoolean();
private final AtomicBoolean breakdown = new AtomicBoolean();
private final AtomicInteger receiptTimes = new AtomicInteger();
private final AtomicInteger curTimes = new AtomicInteger();
private final ThreadPoolExecutor executor;
private final long limit;
private long packageSize;
private long counter;
OrderWorker(long limit, long packageSize, ScanIterator iterator, OrderDeliverer deliverer,
ThreadPoolExecutor executor) {
this.limit = limit;
this.packageSize = packageSize;
this.iterator = iterator;
this.deliverer = deliverer;
this.executor = executor;
if (this.packageSize <= 0) {
this.packageSize = DEFAULT_PACKAGE_SIZE;
log.warn(
"As page-Size is less than or equals 0, default package-size was " +
"effective.[ {} ]",
DEFAULT_PACKAGE_SIZE);
}
}
void hereWeGo() {
if (this.completeFlag.get()) {
log.warn("job complete.");
return;
}
if (this.isWorking.get()) {
log.warn("has been working.");
return;
}
if (this.workingLock.isLocked()) {
log.warn("working now");
return;
}
executor.execute(() -> working());
Thread.yield();
}
void setReceipt(int times) {
this.receiptTimes.set(times);
this.continueWorking();
}
boolean checkContinue() {
return (this.curTimes.get() - this.receiptTimes.get() < MAX_NOT_RECEIPT);
}
void continueWorking() {
if (this.checkContinue()) {
synchronized (this.iterator) {
this.iterator.notify();
}
}
}
void breakdown() {
this.breakdown.set(true);
synchronized (this.iterator) {
this.iterator.notify();
}
}
private void working() {
if (this.isWorking.getAndSet(true)) {
return;
}
this.workingLock.lock();
try {
synchronized (this.iterator) {
KvPageRes.Builder dataBuilder = KvPageRes.newBuilder();
Kv.Builder kvBuilder = Kv.newBuilder();
long packageCount = 0;
while (iterator.hasNext()) {
if (++this.counter > limit) {
this.completeFlag.set(true);
break;
}
if (++packageCount > packageSize) {
if (this.breakdown.get()) {
break;
}
deliverer.deliver(dataBuilder, curTimes.incrementAndGet(), false);
Thread.yield();
if (!this.checkContinue()) {
long start = System.currentTimeMillis();
iterator.wait(
HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS *
1000);
if (System.currentTimeMillis() - start
>=
HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * 1000) {
throw new TimeoutException("Waiting continue more than "
+
HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS +
" seconds.");
}
if (this.breakdown.get()) {
break;
}
}
packageCount = 1;
dataBuilder = KvPageRes.newBuilder();
}
RocksDBSession.BackendColumn col = iterator.next();
dataBuilder.addData(kvBuilder
.setKey(ByteString.copyFrom(col.name))
.setValue(ByteString.copyFrom(col.value))
.setCode(HgStoreNodeUtil.toInt(
iterator.position()))
//position == partition-id.
);
}
this.completeFlag.set(true);
deliverer.deliver(dataBuilder, curTimes.incrementAndGet(), true);
}
} catch (InterruptedException e) {
log.error("Interrupted waiting of iterator, canceled while.", e);
this.deliverer.error("Failed to finish scanning, cause by InterruptedException.");
} catch (TimeoutException t) {
log.info(t.getMessage());
this.deliverer.error("Sever waiting exceeded ["
+ HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS +
"] seconds.");
} catch (Throwable t) {
log.error("Failed to do while for scanning, cause by:", t);
this.deliverer.error("Failed to finish scanning ", t);
} finally {
this.workingLock.unlock();
this.iterator.close();
}
}
}
}

208
hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanQuery.java
View File

@ -1,104 +1,104 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import java.util.Arrays;
import org.apache.hugegraph.store.grpc.common.ScanMethod;
/**
* @author lynn.bond@hotmail.com on 2022/2/28
*/
class ScanQuery implements QueryCondition {
String graph;
String table;
ScanMethod method;
byte[] start;
byte[] end;
byte[] prefix;
int keyCode;
int scanType;
byte[] query;
byte[] position;
int serialNo;
@Override
public byte[] getStart() {
return this.start;
}
@Override
public byte[] getEnd() {
return this.end;
}
@Override
public byte[] getPrefix() {
return this.prefix;
}
@Override
public int getKeyCode() {
return this.keyCode;
}
@Override
public int getScanType() {
return this.scanType;
}
@Override
public byte[] getQuery() {
return this.query;
}
@Override
public byte[] getPosition() {
return this.position;
}
@Override
public int getSerialNo() {
return this.serialNo;
}
static ScanQuery of() {
return new ScanQuery();
}
private ScanQuery() {
}
@Override
public String toString() {
return "ScanQuery{" +
"graph='" + graph + '\'' +
", table='" + table + '\'' +
", method=" + method +
", start=" + Arrays.toString(start) +
", end=" + Arrays.toString(end) +
", prefix=" + Arrays.toString(prefix) +
", partition=" + keyCode +
", scanType=" + scanType +
", serialNo=" + serialNo +
", query=" + Arrays.toString(query) +
", position=" + Arrays.toString(position) +
'}';
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.grpc;
import java.util.Arrays;
import org.apache.hugegraph.store.grpc.common.ScanMethod;
/**
* @author lynn.bond@hotmail.com on 2022/2/28
*/
class ScanQuery implements QueryCondition {
String graph;
String table;
ScanMethod method;
byte[] start;
byte[] end;
byte[] prefix;
int keyCode;
int scanType;
byte[] query;
byte[] position;
int serialNo;
@Override
public byte[] getStart() {
return this.start;
}
@Override
public byte[] getEnd() {
return this.end;
}
@Override
public byte[] getPrefix() {
return this.prefix;
}
@Override
public int getKeyCode() {
return this.keyCode;
}
@Override
public int getScanType() {
return this.scanType;
}
@Override
public byte[] getQuery() {
return this.query;
}
@Override
public byte[] getPosition() {
return this.position;
}
@Override
public int getSerialNo() {
return this.serialNo;
}
static ScanQuery of() {
return new ScanQuery();
}
private ScanQuery() {
}
@Override
public String toString() {
return "ScanQuery{" +
"graph='" + graph + '\'' +
", table='" + table + '\'' +
", method=" + method +
", start=" + Arrays.toString(start) +
", end=" + Arrays.toString(end) +
", prefix=" + Arrays.toString(prefix) +
", partition=" + keyCode +
", scanType=" + scanType +
", serialNo=" + serialNo +
", query=" + Arrays.toString(query) +
", position=" + Arrays.toString(position) +
'}';
}
}

188
hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/GRpcExMetrics.java
View File

@ -1,94 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.metrics;
import java.util.concurrent.ThreadPoolExecutor;
import org.apache.hugegraph.store.node.grpc.GRpcServerConfig;
import org.apache.hugegraph.store.node.util.HgExecutorUtil;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
/**
* @author lynn.bond@hotmail.com on 2022/3/8
*/
public class GRpcExMetrics {
public final static String PREFIX = "grpc";
private static MeterRegistry registry;
private final static ExecutorWrapper wrapper = new ExecutorWrapper();
private GRpcExMetrics() {
}
public synchronized static void init(MeterRegistry meterRegistry) {
if (registry == null) {
registry = meterRegistry;
registerMeters();
}
}
private static void registerMeters() {
registerExecutor();
}
private static void registerExecutor() {
Gauge.builder(PREFIX + ".executor.pool.size", wrapper, (e) -> e.getPoolSize())
.description("The current number of threads in the pool.")
.register(registry);
Gauge.builder(PREFIX + ".executor.core.pool.size", wrapper, (e) -> e.getCorePoolSize())
.description(
"The largest number of threads that have ever simultaneously been in the " +
"pool.")
.register(registry);
Gauge.builder(PREFIX + ".executor.active.count", wrapper, (e) -> e.getActiveCount())
.description("The approximate number of threads that are actively executing tasks.")
.register(registry);
}
private static class ExecutorWrapper {
ThreadPoolExecutor pool;
void init() {
if (this.pool == null) {
pool = HgExecutorUtil.getThreadPoolExecutor(GRpcServerConfig.EXECUTOR_NAME);
}
}
double getPoolSize() {
init();
return this.pool == null ? 0d : this.pool.getPoolSize();
}
int getCorePoolSize() {
init();
return this.pool == null ? 0 : this.pool.getCorePoolSize();
}
int getActiveCount() {
init();
return this.pool == null ? 0 : this.pool.getActiveCount();
}
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.metrics;
import java.util.concurrent.ThreadPoolExecutor;
import org.apache.hugegraph.store.node.grpc.GRpcServerConfig;
import org.apache.hugegraph.store.node.util.HgExecutorUtil;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
/**
* @author lynn.bond@hotmail.com on 2022/3/8
*/
public class GRpcExMetrics {
public final static String PREFIX = "grpc";
private static MeterRegistry registry;
private final static ExecutorWrapper wrapper = new ExecutorWrapper();
private GRpcExMetrics() {
}
public synchronized static void init(MeterRegistry meterRegistry) {
if (registry == null) {
registry = meterRegistry;
registerMeters();
}
}
private static void registerMeters() {
registerExecutor();
}
private static void registerExecutor() {
Gauge.builder(PREFIX + ".executor.pool.size", wrapper, (e) -> e.getPoolSize())
.description("The current number of threads in the pool.")
.register(registry);
Gauge.builder(PREFIX + ".executor.core.pool.size", wrapper, (e) -> e.getCorePoolSize())
.description(
"The largest number of threads that have ever simultaneously been in the " +
"pool.")
.register(registry);
Gauge.builder(PREFIX + ".executor.active.count", wrapper, (e) -> e.getActiveCount())
.description("The approximate number of threads that are actively executing tasks.")
.register(registry);
}
private static class ExecutorWrapper {
ThreadPoolExecutor pool;
void init() {
if (this.pool == null) {
pool = HgExecutorUtil.getThreadPoolExecutor(GRpcServerConfig.EXECUTOR_NAME);
}
}
double getPoolSize() {
init();
return this.pool == null ? 0d : this.pool.getPoolSize();
}
int getCorePoolSize() {
init();
return this.pool == null ? 0 : this.pool.getCorePoolSize();
}
int getActiveCount() {
init();
return this.pool == null ? 0 : this.pool.getActiveCount();
}
}
}

620
hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/JRaftMetrics.java
View File

@ -1,310 +1,310 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.metrics;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.node.util.HgRegexUtil;
import com.alipay.sofa.jraft.core.NodeMetrics;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Snapshot;
import com.codahale.metrics.Timer;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tag;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com on 2022/1/4
*/
@Slf4j
public class JRaftMetrics {
public final static String PREFIX = "jraft";
public static final String LABELS = "quantile";
public static final String LABEL_50 = "0.5";
public static final String LABEL_75 = "0.75";
public static final String LABEL_95 = "0.95";
public static final String LABEL_98 = "0.98";
public static final String LABEL_99 = "0.99";
public static final String LABEL_999 = "0.999";
private static MeterRegistry registry;
private final static HgStoreEngine storeEngine = HgStoreEngine.getInstance();
private final static AtomicInteger groups = new AtomicInteger(0);
private final static Tag handleDataTag = Tag.of("handle", "data");
// private final static Tag handleTxTag = Tag.of("handle", "tx"); //reservation
private final static Set<String> groupSet = new HashSet<>();
private final static String REGEX_REFINE_REPLICATOR = "(replicator)(.+?:\\d+)(.*)";
private JRaftMetrics() {
}
public synchronized static void init(MeterRegistry meterRegistry) {
if (registry == null) {
registry = meterRegistry;
registerMeters();
}
}
private static void registerMeters() {
Gauge.builder(PREFIX + ".groups", JRaftMetrics::updateGroups)
.description("Number of raft-groups, which handled the data of graph.")
.tags(Collections.singleton(handleDataTag))
.register(registry);
}
private static int updateGroups() {
int buf = getGroups();
if (buf != groups.get()) {
groups.set(buf);
registerNodeMetrics();
}
return buf;
}
private static int getGroups() {
return storeEngine.getRaftGroupCount();
}
private static Map<String, NodeMetrics> getRaftGroupMetrics() {
Map<String, NodeMetrics> map = storeEngine.getNodeMetrics();
if (map == null) {
return Collections.emptyMap();
}
return map;
}
private static void registerNodeMetrics() {
Map<String, NodeMetrics> map = getRaftGroupMetrics();
synchronized (groupSet) {
map.forEach((group, metrics) -> {
if (!groupSet.add(group)) return;
metrics.getMetricRegistry().getGauges()
.forEach((k, v) -> registerGauge(group, k, v));
metrics.getMetricRegistry().getMeters()
.forEach((k, v) -> registerMeter(group, k, v));
metrics.getMetricRegistry().getCounters()
.forEach((k, v) -> registerCounter(group, k, v));
metrics.getMetricRegistry().getTimers()
.forEach((k, v) -> registerTimer(group, k, v));
metrics.getMetricRegistry().getHistograms()
.forEach((k, v) -> registerHistogram(group, k, v));
});
}
}
private static class HistogramWrapper {
private final com.codahale.metrics.Histogram histogram;
private Snapshot snapshot;
private long ts = System.currentTimeMillis();
HistogramWrapper(com.codahale.metrics.Histogram histogram) {
this.histogram = histogram;
this.snapshot = this.histogram.getSnapshot();
}
Snapshot getSnapshot() {
if (System.currentTimeMillis() - this.ts > 30_000) {
this.snapshot = this.histogram.getSnapshot();
this.ts = System.currentTimeMillis();
}
return this.snapshot;
}
}
private static HistogramWrapper toWrapper(com.codahale.metrics.Histogram histogram) {
return new HistogramWrapper(histogram);
}
private static String refineMetrics(String name, List<Tag> tags) {
if (name == null || name.isEmpty()) return name;
List<String> buf = HgRegexUtil.toGroupValues(REGEX_REFINE_REPLICATOR, name);
String res = null;
/*Extracted name of replicator into a tag.*/
if (buf != null && buf.size() == 4) {
res = buf.get(1) + buf.get(3);
String value = buf.get(2);
if (value != null && value.startsWith("-")) {
value = value.substring(1);
}
tags.add(Tag.of("replicator", value));
} else {
res = name;
}
return res;
}
private static void registerHistogram(String group, String name,
com.codahale.metrics.Histogram histogram) {
if (histogram == null) return;
List<Tag> tags = new LinkedList<>();
tags.add(handleDataTag);
tags.add(Tag.of("group", group));
name = refineMetrics(name, tags);
String baseName = PREFIX + "." + name.toLowerCase();
HistogramWrapper wrapper = toWrapper(histogram);
Gauge.builder(baseName + ".median", wrapper, (d) -> d.getSnapshot().getMedian())
.tags(tags).register(registry);
Gauge.builder(baseName + ".min", wrapper, (d) -> d.getSnapshot().getMin())
.tags(tags).register(registry);
Gauge.builder(baseName + ".max", wrapper, (d) -> d.getSnapshot().getMax())
.tags(tags).register(registry);
Gauge.builder(baseName + ".mean", wrapper, (d) -> d.getSnapshot().getMean())
.tags(tags).register(registry);
baseName = baseName + ".summary";
Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().getMedian())
.tags(tags).tag(LABELS, LABEL_50).register(registry);
Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get75thPercentile())
.tags(tags).tag(LABELS, LABEL_75).register(registry);
Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get95thPercentile())
.tags(tags).tag(LABELS, LABEL_95).register(registry);
Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get98thPercentile())
.tags(tags).tag(LABELS, LABEL_98).register(registry);
Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get99thPercentile())
.tags(tags).tag(LABELS, LABEL_99).register(registry);
Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get999thPercentile())
.tags(tags).tag(LABELS, LABEL_999).register(registry);
Gauge.builder(baseName + ".sum", wrapper,
(d) -> Arrays.stream(d.getSnapshot().getValues()).sum())
.tags(tags).register(registry);
Gauge.builder(baseName + ".count", wrapper, (d) -> d.getSnapshot().size())
.tags(tags).register(registry);
}
private static void registerTimer(String group, String name, com.codahale.metrics.Timer timer) {
List<Tag> tags = new LinkedList<>();
tags.add(handleDataTag);
tags.add(Tag.of("group", group));
name = refineMetrics(name, tags);
String baseName = PREFIX + "." + name.toLowerCase();
Gauge.builder(baseName + ".count", timer, Timer::getCount)
.tags(tags).register(registry);
Gauge.builder(baseName + ".timer", timer, Timer::getCount)
.tags(tags).tag("rate", "1m").register(registry);
Gauge.builder(baseName + ".timer", timer, Timer::getCount)
.tags(tags).tag("rate", "5m").register(registry);
Gauge.builder(baseName + ".timer", timer, Timer::getCount)
.tags(tags).tag("rate", "15m").register(registry);
Gauge.builder(baseName + ".timer", timer, Timer::getCount)
.tags(tags).tag("rate", "mean").register(registry);
}
private static void registerMeter(String group, String name, com.codahale.metrics.Meter meter) {
List<Tag> tags = new LinkedList<>();
tags.add(handleDataTag);
tags.add(Tag.of("group", group));
name = refineMetrics(name, tags);
String baseName = PREFIX + "." + name.toLowerCase();
Gauge.builder(baseName + ".count", meter, Meter::getCount)
.tags(tags)
.register(registry);
Gauge.builder(baseName + ".rate", meter, Meter::getCount)
.tags(tags).tag("rate", "1m")
.register(registry);
Gauge.builder(baseName + ".rate", meter, Meter::getCount)
.tags(tags).tag("rate", "5m")
.register(registry);
Gauge.builder(baseName + ".rate", meter, Meter::getCount)
.tags(tags).tag("rate", "15m")
.register(registry);
Gauge.builder(baseName + ".rate", meter, Meter::getCount)
.tags(tags).tag("rate", "mean")
.register(registry);
}
private static void registerCounter(String group, String name,
com.codahale.metrics.Counter counter) {
List<Tag> tags = new LinkedList<>();
tags.add(handleDataTag);
tags.add(Tag.of("group", group));
name = refineMetrics(name, tags);
name = name.toLowerCase();
//Adapted a counter to be a gauge.
Gauge.builder(PREFIX + "." + name + ".count", counter, Counter::getCount)
.tags(tags).register(registry);
}
private static void registerGauge(String group, String name,
com.codahale.metrics.Gauge<?> gauge) {
List<Tag> tags = new LinkedList<>();
tags.add(handleDataTag);
tags.add(Tag.of("group", group));
name = refineMetrics(name, tags);
name = name.toLowerCase();
if (gauge.getValue() instanceof Number) {
Gauge.builder(PREFIX + "." + name, gauge, (g) -> ((Number) g.getValue()).doubleValue())
.tags(tags).register(registry);
} else {
Gauge.builder(PREFIX + "." + name, () -> 1.0)
.tags(tags)
.tag("str.gauge", String.valueOf(gauge.getValue())).register(registry);
}
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.metrics;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.node.util.HgRegexUtil;
import com.alipay.sofa.jraft.core.NodeMetrics;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Snapshot;
import com.codahale.metrics.Timer;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tag;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com on 2022/1/4
*/
@Slf4j
public class JRaftMetrics {
public final static String PREFIX = "jraft";
public static final String LABELS = "quantile";
public static final String LABEL_50 = "0.5";
public static final String LABEL_75 = "0.75";
public static final String LABEL_95 = "0.95";
public static final String LABEL_98 = "0.98";
public static final String LABEL_99 = "0.99";
public static final String LABEL_999 = "0.999";
private static MeterRegistry registry;
private final static HgStoreEngine storeEngine = HgStoreEngine.getInstance();
private final static AtomicInteger groups = new AtomicInteger(0);
private final static Tag handleDataTag = Tag.of("handle", "data");
// private final static Tag handleTxTag = Tag.of("handle", "tx"); //reservation
private final static Set<String> groupSet = new HashSet<>();
private final static String REGEX_REFINE_REPLICATOR = "(replicator)(.+?:\\d+)(.*)";
private JRaftMetrics() {
}
public synchronized static void init(MeterRegistry meterRegistry) {
if (registry == null) {
registry = meterRegistry;
registerMeters();
}
}
private static void registerMeters() {
Gauge.builder(PREFIX + ".groups", JRaftMetrics::updateGroups)
.description("Number of raft-groups, which handled the data of graph.")
.tags(Collections.singleton(handleDataTag))
.register(registry);
}
private static int updateGroups() {
int buf = getGroups();
if (buf != groups.get()) {
groups.set(buf);
registerNodeMetrics();
}
return buf;
}
private static int getGroups() {
return storeEngine.getRaftGroupCount();
}
private static Map<String, NodeMetrics> getRaftGroupMetrics() {
Map<String, NodeMetrics> map = storeEngine.getNodeMetrics();
if (map == null) {
return Collections.emptyMap();
}
return map;
}
private static void registerNodeMetrics() {
Map<String, NodeMetrics> map = getRaftGroupMetrics();
synchronized (groupSet) {
map.forEach((group, metrics) -> {
if (!groupSet.add(group)) return;
metrics.getMetricRegistry().getGauges()
.forEach((k, v) -> registerGauge(group, k, v));
metrics.getMetricRegistry().getMeters()
.forEach((k, v) -> registerMeter(group, k, v));
metrics.getMetricRegistry().getCounters()
.forEach((k, v) -> registerCounter(group, k, v));
metrics.getMetricRegistry().getTimers()
.forEach((k, v) -> registerTimer(group, k, v));
metrics.getMetricRegistry().getHistograms()
.forEach((k, v) -> registerHistogram(group, k, v));
});
}
}
private static class HistogramWrapper {
private final com.codahale.metrics.Histogram histogram;
private Snapshot snapshot;
private long ts = System.currentTimeMillis();
HistogramWrapper(com.codahale.metrics.Histogram histogram) {
this.histogram = histogram;
this.snapshot = this.histogram.getSnapshot();
}
Snapshot getSnapshot() {
if (System.currentTimeMillis() - this.ts > 30_000) {
this.snapshot = this.histogram.getSnapshot();
this.ts = System.currentTimeMillis();
}
return this.snapshot;
}
}
private static HistogramWrapper toWrapper(com.codahale.metrics.Histogram histogram) {
return new HistogramWrapper(histogram);
}
private static String refineMetrics(String name, List<Tag> tags) {
if (name == null || name.isEmpty()) return name;
List<String> buf = HgRegexUtil.toGroupValues(REGEX_REFINE_REPLICATOR, name);
String res = null;
/*Extracted name of replicator into a tag.*/
if (buf != null && buf.size() == 4) {
res = buf.get(1) + buf.get(3);
String value = buf.get(2);
if (value != null && value.startsWith("-")) {
value = value.substring(1);
}
tags.add(Tag.of("replicator", value));
} else {
res = name;
}
return res;
}
private static void registerHistogram(String group, String name,
com.codahale.metrics.Histogram histogram) {
if (histogram == null) return;
List<Tag> tags = new LinkedList<>();
tags.add(handleDataTag);
tags.add(Tag.of("group", group));
name = refineMetrics(name, tags);
String baseName = PREFIX + "." + name.toLowerCase();
HistogramWrapper wrapper = toWrapper(histogram);
Gauge.builder(baseName + ".median", wrapper, (d) -> d.getSnapshot().getMedian())
.tags(tags).register(registry);
Gauge.builder(baseName + ".min", wrapper, (d) -> d.getSnapshot().getMin())
.tags(tags).register(registry);
Gauge.builder(baseName + ".max", wrapper, (d) -> d.getSnapshot().getMax())
.tags(tags).register(registry);
Gauge.builder(baseName + ".mean", wrapper, (d) -> d.getSnapshot().getMean())
.tags(tags).register(registry);
baseName = baseName + ".summary";
Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().getMedian())
.tags(tags).tag(LABELS, LABEL_50).register(registry);
Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get75thPercentile())
.tags(tags).tag(LABELS, LABEL_75).register(registry);
Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get95thPercentile())
.tags(tags).tag(LABELS, LABEL_95).register(registry);
Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get98thPercentile())
.tags(tags).tag(LABELS, LABEL_98).register(registry);
Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get99thPercentile())
.tags(tags).tag(LABELS, LABEL_99).register(registry);
Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get999thPercentile())
.tags(tags).tag(LABELS, LABEL_999).register(registry);
Gauge.builder(baseName + ".sum", wrapper,
(d) -> Arrays.stream(d.getSnapshot().getValues()).sum())
.tags(tags).register(registry);
Gauge.builder(baseName + ".count", wrapper, (d) -> d.getSnapshot().size())
.tags(tags).register(registry);
}
private static void registerTimer(String group, String name, com.codahale.metrics.Timer timer) {
List<Tag> tags = new LinkedList<>();
tags.add(handleDataTag);
tags.add(Tag.of("group", group));
name = refineMetrics(name, tags);
String baseName = PREFIX + "." + name.toLowerCase();
Gauge.builder(baseName + ".count", timer, Timer::getCount)
.tags(tags).register(registry);
Gauge.builder(baseName + ".timer", timer, Timer::getCount)
.tags(tags).tag("rate", "1m").register(registry);
Gauge.builder(baseName + ".timer", timer, Timer::getCount)
.tags(tags).tag("rate", "5m").register(registry);
Gauge.builder(baseName + ".timer", timer, Timer::getCount)
.tags(tags).tag("rate", "15m").register(registry);
Gauge.builder(baseName + ".timer", timer, Timer::getCount)
.tags(tags).tag("rate", "mean").register(registry);
}
private static void registerMeter(String group, String name, com.codahale.metrics.Meter meter) {
List<Tag> tags = new LinkedList<>();
tags.add(handleDataTag);
tags.add(Tag.of("group", group));
name = refineMetrics(name, tags);
String baseName = PREFIX + "." + name.toLowerCase();
Gauge.builder(baseName + ".count", meter, Meter::getCount)
.tags(tags)
.register(registry);
Gauge.builder(baseName + ".rate", meter, Meter::getCount)
.tags(tags).tag("rate", "1m")
.register(registry);
Gauge.builder(baseName + ".rate", meter, Meter::getCount)
.tags(tags).tag("rate", "5m")
.register(registry);
Gauge.builder(baseName + ".rate", meter, Meter::getCount)
.tags(tags).tag("rate", "15m")
.register(registry);
Gauge.builder(baseName + ".rate", meter, Meter::getCount)
.tags(tags).tag("rate", "mean")
.register(registry);
}
private static void registerCounter(String group, String name,
com.codahale.metrics.Counter counter) {
List<Tag> tags = new LinkedList<>();
tags.add(handleDataTag);
tags.add(Tag.of("group", group));
name = refineMetrics(name, tags);
name = name.toLowerCase();
//Adapted a counter to be a gauge.
Gauge.builder(PREFIX + "." + name + ".count", counter, Counter::getCount)
.tags(tags).register(registry);
}
private static void registerGauge(String group, String name,
com.codahale.metrics.Gauge<?> gauge) {
List<Tag> tags = new LinkedList<>();
tags.add(handleDataTag);
tags.add(Tag.of("group", group));
name = refineMetrics(name, tags);
name = name.toLowerCase();
if (gauge.getValue() instanceof Number) {
Gauge.builder(PREFIX + "." + name, gauge, (g) -> ((Number) g.getValue()).doubleValue())
.tags(tags).register(registry);
} else {
Gauge.builder(PREFIX + "." + name, () -> 1.0)
.tags(tags)
.tag("str.gauge", String.valueOf(gauge.getValue())).register(registry);
}
}
}

134
hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsMetrics.java
View File

@ -1,67 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.metrics;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
/**
* @author lynn.bond@hotmail.com on 2022/3/1
* @version 0.1.0
*/
public class ProcfsMetrics {
private static MeterRegistry registry;
public final static String PREFIX = "process_memory";
private final static ProcfsSmaps smaps = new ProcfsSmaps();
private ProcfsMetrics() {
}
public synchronized static void init(MeterRegistry meterRegistry) {
if (registry == null) {
registry = meterRegistry;
registerMeters();
}
}
private static void registerMeters() {
registerProcessGauge();
}
private static void registerProcessGauge() {
Gauge.builder(PREFIX + ".rss.bytes", () -> smaps.get(ProcfsSmaps.KEY.RSS))
.register(registry);
Gauge.builder(PREFIX + ".pss.bytes", () -> smaps.get(ProcfsSmaps.KEY.PSS))
.register(registry);
Gauge.builder(PREFIX + ".vss.bytes", () -> smaps.get(ProcfsSmaps.KEY.VSS))
.register(registry);
Gauge.builder(PREFIX + ".swap.bytes", () -> smaps.get(ProcfsSmaps.KEY.SWAP))
.register(registry);
Gauge.builder(PREFIX + ".swappss.bytes", () -> smaps.get(ProcfsSmaps.KEY.SWAPPSS))
.register(registry);
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.metrics;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
/**
* @author lynn.bond@hotmail.com on 2022/3/1
* @version 0.1.0
*/
public class ProcfsMetrics {
private static MeterRegistry registry;
public final static String PREFIX = "process_memory";
private final static ProcfsSmaps smaps = new ProcfsSmaps();
private ProcfsMetrics() {
}
public synchronized static void init(MeterRegistry meterRegistry) {
if (registry == null) {
registry = meterRegistry;
registerMeters();
}
}
private static void registerMeters() {
registerProcessGauge();
}
private static void registerProcessGauge() {
Gauge.builder(PREFIX + ".rss.bytes", () -> smaps.get(ProcfsSmaps.KEY.RSS))
.register(registry);
Gauge.builder(PREFIX + ".pss.bytes", () -> smaps.get(ProcfsSmaps.KEY.PSS))
.register(registry);
Gauge.builder(PREFIX + ".vss.bytes", () -> smaps.get(ProcfsSmaps.KEY.VSS))
.register(registry);
Gauge.builder(PREFIX + ".swap.bytes", () -> smaps.get(ProcfsSmaps.KEY.SWAP))
.register(registry);
Gauge.builder(PREFIX + ".swappss.bytes", () -> smaps.get(ProcfsSmaps.KEY.SWAPPSS))
.register(registry);
}
}

828
hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetrics.java
View File

@ -1,414 +1,414 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.metrics;
import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.HISTOGRAMS;
import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABELS;
import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_50;
import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_95;
import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_99;
import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.PREFIX;
import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.TICKERS;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.node.util.HgAssert;
import org.rocksdb.HistogramData;
import org.rocksdb.HistogramType;
import org.rocksdb.MemoryUsageType;
import org.rocksdb.Statistics;
import org.rocksdb.TickerType;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.Meter;
import io.micrometer.core.instrument.MeterRegistry;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com on 2021/12/30
* @version 1.2.0 on 2022/03/22 added auto meter removing when graph has been closed.
*/
@Slf4j
public class RocksDBMetrics {
private static MeterRegistry registry;
private final static RocksDBFactory rocksDBFactory = RocksDBFactory.getInstance();
private final static AtomicInteger rocks = new AtomicInteger(0);
private final static Set<String> graphSet = new HashSet<>();
private final static HgStoreEngine storeEngine = HgStoreEngine.getInstance();
private final static MemoryUseWrapper memoryUseWrapper = new MemoryUseWrapper();
private final static Map<String, StatisticsWrapper> statisticsHolder = new HashMap<>();
private final static Map<HistogramDataWrapper, HistogramType> histogramHolder = new HashMap<>();
private final static Map<String, Set<Meter>> graphMeterMap = new ConcurrentHashMap<>();
private RocksDBMetrics() {
}
public static void init(final MeterRegistry meterRegistry) {
HgAssert.isArgumentNotNull(meterRegistry, "meterRegistry");
if (registry != null) {
return;
}
registry = meterRegistry;
Gauge.builder("rocks.num", RocksDBMetrics::updateRocks)
.description("Number of instance of RocksDB running in this node")
.register(registry);
registerMemoryUse();
}
private static int updateRocks() {
int buf = getRocks();
if (buf != rocks.get()) {
rocks.set(buf);
registerMeter();
}
return buf;
}
private static int getRocks() {
return rocksDBFactory.getSessionSize();
}
private static Set<String> getGraphs() {
return rocksDBFactory.getGraphNames();
}
private static RocksDBSession getRocksDBSession(String graph) {
return rocksDBFactory.queryGraphDB(graph);
}
private static synchronized void registerMeter() {
Set<String> graphs = getGraphs();
if (graphs == null) {
log.error(
"Failed to fetch the collection of names of graph, when invoking to register " +
"RocksDB gauge.");
return;
}
graphs.forEach(g -> {
if (!graphSet.add(g)) {
return;
}
StatisticsWrapper stats = new StatisticsWrapper(g);
statisticsHolder.put(g, stats);
for (final TickerType ticker : TICKERS) {
String gaugeName = PREFIX + "." + ticker.name().toLowerCase();
saveGraphMeter(g,
Gauge.builder(gaugeName, () -> stats.getTickerCount(ticker))
.description("RocksDB reported statistics for " + ticker.name())
.tag("graph", g)
.register(registry)
);
}
for (final HistogramType histogram : HISTOGRAMS) {
registerHistogram(g, registry, histogram, stats);
}
registrySessionRefNum(g);
});
graphSet.removeAll(graphSet.stream().filter(g -> !graphs.contains(g))
.peek(g -> removeMeters(g))
.collect(Collectors.toList())
);
}
private static void saveGraphMeter(String g, Meter meter) {
graphMeterMap.computeIfAbsent(g, k -> new HashSet<>()).add(meter);
}
private static void removeMeters(String g) {
graphMeterMap.getOrDefault(g, Collections.emptySet()).forEach(e -> registry.remove(e));
}
private static void registerHistogram(String graph, MeterRegistry registry, HistogramType
histogramType, StatisticsWrapper stats) {
HistogramDataWrapper histogram = new HistogramDataWrapper(histogramType,
() -> stats.getHistogramData(
histogramType));
histogramHolder.put(histogram, histogramType);
String baseName = PREFIX + "." + histogramType.name().toLowerCase();
saveGraphMeter(graph,
Gauge.builder(baseName + ".max", histogram, HistogramDataWrapper::getMax)
.tag("graph", graph).register(registry));
saveGraphMeter(graph, Gauge.builder(baseName + ".mean", histogram,
HistogramDataWrapper::getAverage).tag("graph", graph)
.register(registry));
saveGraphMeter(graph,
Gauge.builder(baseName + ".min", histogram, HistogramDataWrapper::getMin)
.tag("graph", graph).register(registry));
baseName = baseName + ".summary";
saveGraphMeter(graph, Gauge.builder(baseName, histogram, HistogramDataWrapper::getMedian)
.tags("graph", graph, LABELS, LABEL_50).register(registry));
saveGraphMeter(graph,
Gauge.builder(baseName, histogram, HistogramDataWrapper::getPercentile95)
.tags("graph", graph, LABELS, LABEL_95).register(registry));
saveGraphMeter(graph,
Gauge.builder(baseName, histogram, HistogramDataWrapper::getPercentile99)
.tags("graph", graph, LABELS, LABEL_99).register(registry));
saveGraphMeter(graph,
Gauge.builder(baseName + ".sum", histogram, HistogramDataWrapper::getSum)
.tags("graph", graph).register(registry));
saveGraphMeter(graph,
Gauge.builder(baseName + ".count", histogram, HistogramDataWrapper::getCount)
.tags("graph", graph).register(registry));
}
private static void registerMemoryUse() {
Gauge.builder(PREFIX + ".table.reader.total", memoryUseWrapper,
(e) -> e.getTableReaderTotal())
.description("The current number of threads in the pool.")
.register(registry);
Gauge.builder(PREFIX + ".mem.table.total", memoryUseWrapper, (e) -> e.getMemTableTotal())
.description("The current number of threads in the pool.")
.register(registry);
Gauge.builder(PREFIX + ".mem.table.unFlushed", memoryUseWrapper,
(e) -> e.getMemTableUnFlushed())
.description("The current number of threads in the pool.")
.register(registry);
Gauge.builder(PREFIX + ".cache.total", memoryUseWrapper, (e) -> e.getCacheTotal())
.description("The current number of threads in the pool.")
.register(registry);
Gauge.builder(PREFIX + ".block.cache.pinned-usage", memoryUseWrapper,
(e) -> e.getProperty("rocksdb.block-cache-pinned-usage"))
.description("The current number of threads in the pool.")
.register(registry);
}
private static void registrySessionRefNum(String graph) {
SessionWrapper sessionWrapper = new SessionWrapper(graph);
saveGraphMeter(graph,
Gauge.builder(PREFIX + ".session.ref.count", sessionWrapper,
(e) -> e.getRefCount() - 1)
.description("The current amount of reference of session")
.tag("ref", "self").tag("graph", graph)
.strongReference(true)
.register(registry)
);
}
private static class SessionWrapper {
private final String graphName;
SessionWrapper(String graph) {
this.graphName = graph;
}
public int getRefCount() {
try (RocksDBSession session = getRocksDBSession(graphName)) {
if (session != null) {
return getValue(session, e -> e.getRefCount(), -1);
}
return 0;
}
}
}
private static class MemoryUseWrapper {
Map<MemoryUsageType, Long> mems = null;
long lastTime = 0;
private void loadData() {
if (mems == null || System.currentTimeMillis() - lastTime > 30000) {
mems = storeEngine.getBusinessHandler().getApproximateMemoryUsageByType(null);
lastTime = System.currentTimeMillis();
}
}
public Long getTableReaderTotal() {
loadData();
return mems.get(MemoryUsageType.kTableReadersTotal);
}
public Long getMemTableTotal() {
loadData();
return mems.get(MemoryUsageType.kMemTableTotal);
}
public Long getCacheTotal() {
loadData();
return mems.get(MemoryUsageType.kCacheTotal);
}
public Long getMemTableUnFlushed() {
loadData();
return mems.get(MemoryUsageType.kMemTableUnFlushed);
}
public Long getProperty(String property) {
Set<String> graphs = rocksDBFactory.getGraphNames();
if (graphs.size() > 0) {
try (RocksDBSession session = getRocksDBSession((String) graphs.toArray()[0])) {
if (session != null) {
return Long.parseLong(session.getProperty(property));
}
}
}
return null;
}
}
private static class StatisticsWrapper {
private final String graphName;
long lastTime = 0;
private final Map<TickerType, Long> tickerCounteMap = new ConcurrentHashMap<>();
private final Map<HistogramType, HistogramData> histogramDataMap =
new ConcurrentHashMap<>();
StatisticsWrapper(String graph) {
this.graphName = graph;
loadData();
}
private void loadData() {
if (System.currentTimeMillis() - lastTime < 30000) return;
lastTime = System.currentTimeMillis();
try (RocksDBSession session = getRocksDBSession(graphName)) {
if (session == null) {
// log.error("Failed to fetch the RocksDBSession with graph's name: [ " +
// graph + " ]");
return;
}
Statistics statistics = session.getRocksDbStats();
for (final TickerType ticker : TICKERS) {
tickerCounteMap.put(ticker, statistics.getTickerCount(ticker));
}
for (final HistogramType histogram : HISTOGRAMS) {
histogramDataMap.put(histogram, statistics.getHistogramData(histogram));
}
}
}
public long getTickerCount(TickerType tickerType) {
this.loadData();
return tickerCounteMap.containsKey(tickerType) ? tickerCounteMap.get(tickerType) : 0;
}
public HistogramData getHistogramData(HistogramType histogramType) {
this.loadData();
return histogramDataMap.get(histogramType);
}
}
private static <S, T> T getValue(S stat, Function<S, T> fun, T defaultValue) {
if (stat == null) return defaultValue;
return fun.apply(stat);
}
private static class HistogramDataWrapper {
private HistogramData data = new HistogramData(0d, 0d, 0d, 0d, 0d);
private long ts = System.currentTimeMillis() - 30_000;
private final Supplier<HistogramData> supplier;
private final HistogramType histogramType;
HistogramDataWrapper(HistogramType histogramType, Supplier<HistogramData> supplier) {
this.supplier = supplier;
this.histogramType = histogramType;
}
private HistogramData getData() {
if (System.currentTimeMillis() - this.ts > 30_000) {
HistogramData buf = this.supplier.get();
if (buf != null) {
this.data = buf;
this.ts = System.currentTimeMillis();
}
}
return this.data;
}
public double getMedian() {
return getData().getMedian();
}
public double getPercentile95() {
return getData().getPercentile95();
}
public double getPercentile99() {
return getData().getPercentile99();
}
public double getAverage() {
return getData().getAverage();
}
public double getStandardDeviation() {
return getData().getStandardDeviation();
}
public double getMax() {
return getData().getMax();
}
public long getCount() {
return getData().getCount();
}
public long getSum() {
return getData().getSum();
}
public double getMin() {
return getData().getMin();
}
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.metrics;
import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.HISTOGRAMS;
import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABELS;
import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_50;
import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_95;
import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_99;
import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.PREFIX;
import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.TICKERS;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.hugegraph.rocksdb.access.RocksDBFactory;
import org.apache.hugegraph.rocksdb.access.RocksDBSession;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.node.util.HgAssert;
import org.rocksdb.HistogramData;
import org.rocksdb.HistogramType;
import org.rocksdb.MemoryUsageType;
import org.rocksdb.Statistics;
import org.rocksdb.TickerType;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.Meter;
import io.micrometer.core.instrument.MeterRegistry;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com on 2021/12/30
* @version 1.2.0 on 2022/03/22 added auto meter removing when graph has been closed.
*/
@Slf4j
public class RocksDBMetrics {
private static MeterRegistry registry;
private final static RocksDBFactory rocksDBFactory = RocksDBFactory.getInstance();
private final static AtomicInteger rocks = new AtomicInteger(0);
private final static Set<String> graphSet = new HashSet<>();
private final static HgStoreEngine storeEngine = HgStoreEngine.getInstance();
private final static MemoryUseWrapper memoryUseWrapper = new MemoryUseWrapper();
private final static Map<String, StatisticsWrapper> statisticsHolder = new HashMap<>();
private final static Map<HistogramDataWrapper, HistogramType> histogramHolder = new HashMap<>();
private final static Map<String, Set<Meter>> graphMeterMap = new ConcurrentHashMap<>();
private RocksDBMetrics() {
}
public static void init(final MeterRegistry meterRegistry) {
HgAssert.isArgumentNotNull(meterRegistry, "meterRegistry");
if (registry != null) {
return;
}
registry = meterRegistry;
Gauge.builder("rocks.num", RocksDBMetrics::updateRocks)
.description("Number of instance of RocksDB running in this node")
.register(registry);
registerMemoryUse();
}
private static int updateRocks() {
int buf = getRocks();
if (buf != rocks.get()) {
rocks.set(buf);
registerMeter();
}
return buf;
}
private static int getRocks() {
return rocksDBFactory.getSessionSize();
}
private static Set<String> getGraphs() {
return rocksDBFactory.getGraphNames();
}
private static RocksDBSession getRocksDBSession(String graph) {
return rocksDBFactory.queryGraphDB(graph);
}
private static synchronized void registerMeter() {
Set<String> graphs = getGraphs();
if (graphs == null) {
log.error(
"Failed to fetch the collection of names of graph, when invoking to register " +
"RocksDB gauge.");
return;
}
graphs.forEach(g -> {
if (!graphSet.add(g)) {
return;
}
StatisticsWrapper stats = new StatisticsWrapper(g);
statisticsHolder.put(g, stats);
for (final TickerType ticker : TICKERS) {
String gaugeName = PREFIX + "." + ticker.name().toLowerCase();
saveGraphMeter(g,
Gauge.builder(gaugeName, () -> stats.getTickerCount(ticker))
.description("RocksDB reported statistics for " + ticker.name())
.tag("graph", g)
.register(registry)
);
}
for (final HistogramType histogram : HISTOGRAMS) {
registerHistogram(g, registry, histogram, stats);
}
registrySessionRefNum(g);
});
graphSet.removeAll(graphSet.stream().filter(g -> !graphs.contains(g))
.peek(g -> removeMeters(g))
.collect(Collectors.toList())
);
}
private static void saveGraphMeter(String g, Meter meter) {
graphMeterMap.computeIfAbsent(g, k -> new HashSet<>()).add(meter);
}
private static void removeMeters(String g) {
graphMeterMap.getOrDefault(g, Collections.emptySet()).forEach(e -> registry.remove(e));
}
private static void registerHistogram(String graph, MeterRegistry registry, HistogramType
histogramType, StatisticsWrapper stats) {
HistogramDataWrapper histogram = new HistogramDataWrapper(histogramType,
() -> stats.getHistogramData(
histogramType));
histogramHolder.put(histogram, histogramType);
String baseName = PREFIX + "." + histogramType.name().toLowerCase();
saveGraphMeter(graph,
Gauge.builder(baseName + ".max", histogram, HistogramDataWrapper::getMax)
.tag("graph", graph).register(registry));
saveGraphMeter(graph, Gauge.builder(baseName + ".mean", histogram,
HistogramDataWrapper::getAverage).tag("graph", graph)
.register(registry));
saveGraphMeter(graph,
Gauge.builder(baseName + ".min", histogram, HistogramDataWrapper::getMin)
.tag("graph", graph).register(registry));
baseName = baseName + ".summary";
saveGraphMeter(graph, Gauge.builder(baseName, histogram, HistogramDataWrapper::getMedian)
.tags("graph", graph, LABELS, LABEL_50).register(registry));
saveGraphMeter(graph,
Gauge.builder(baseName, histogram, HistogramDataWrapper::getPercentile95)
.tags("graph", graph, LABELS, LABEL_95).register(registry));
saveGraphMeter(graph,
Gauge.builder(baseName, histogram, HistogramDataWrapper::getPercentile99)
.tags("graph", graph, LABELS, LABEL_99).register(registry));
saveGraphMeter(graph,
Gauge.builder(baseName + ".sum", histogram, HistogramDataWrapper::getSum)
.tags("graph", graph).register(registry));
saveGraphMeter(graph,
Gauge.builder(baseName + ".count", histogram, HistogramDataWrapper::getCount)
.tags("graph", graph).register(registry));
}
private static void registerMemoryUse() {
Gauge.builder(PREFIX + ".table.reader.total", memoryUseWrapper,
(e) -> e.getTableReaderTotal())
.description("The current number of threads in the pool.")
.register(registry);
Gauge.builder(PREFIX + ".mem.table.total", memoryUseWrapper, (e) -> e.getMemTableTotal())
.description("The current number of threads in the pool.")
.register(registry);
Gauge.builder(PREFIX + ".mem.table.unFlushed", memoryUseWrapper,
(e) -> e.getMemTableUnFlushed())
.description("The current number of threads in the pool.")
.register(registry);
Gauge.builder(PREFIX + ".cache.total", memoryUseWrapper, (e) -> e.getCacheTotal())
.description("The current number of threads in the pool.")
.register(registry);
Gauge.builder(PREFIX + ".block.cache.pinned-usage", memoryUseWrapper,
(e) -> e.getProperty("rocksdb.block-cache-pinned-usage"))
.description("The current number of threads in the pool.")
.register(registry);
}
private static void registrySessionRefNum(String graph) {
SessionWrapper sessionWrapper = new SessionWrapper(graph);
saveGraphMeter(graph,
Gauge.builder(PREFIX + ".session.ref.count", sessionWrapper,
(e) -> e.getRefCount() - 1)
.description("The current amount of reference of session")
.tag("ref", "self").tag("graph", graph)
.strongReference(true)
.register(registry)
);
}
private static class SessionWrapper {
private final String graphName;
SessionWrapper(String graph) {
this.graphName = graph;
}
public int getRefCount() {
try (RocksDBSession session = getRocksDBSession(graphName)) {
if (session != null) {
return getValue(session, e -> e.getRefCount(), -1);
}
return 0;
}
}
}
private static class MemoryUseWrapper {
Map<MemoryUsageType, Long> mems = null;
long lastTime = 0;
private void loadData() {
if (mems == null || System.currentTimeMillis() - lastTime > 30000) {
mems = storeEngine.getBusinessHandler().getApproximateMemoryUsageByType(null);
lastTime = System.currentTimeMillis();
}
}
public Long getTableReaderTotal() {
loadData();
return mems.get(MemoryUsageType.kTableReadersTotal);
}
public Long getMemTableTotal() {
loadData();
return mems.get(MemoryUsageType.kMemTableTotal);
}
public Long getCacheTotal() {
loadData();
return mems.get(MemoryUsageType.kCacheTotal);
}
public Long getMemTableUnFlushed() {
loadData();
return mems.get(MemoryUsageType.kMemTableUnFlushed);
}
public Long getProperty(String property) {
Set<String> graphs = rocksDBFactory.getGraphNames();
if (graphs.size() > 0) {
try (RocksDBSession session = getRocksDBSession((String) graphs.toArray()[0])) {
if (session != null) {
return Long.parseLong(session.getProperty(property));
}
}
}
return null;
}
}
private static class StatisticsWrapper {
private final String graphName;
long lastTime = 0;
private final Map<TickerType, Long> tickerCounteMap = new ConcurrentHashMap<>();
private final Map<HistogramType, HistogramData> histogramDataMap =
new ConcurrentHashMap<>();
StatisticsWrapper(String graph) {
this.graphName = graph;
loadData();
}
private void loadData() {
if (System.currentTimeMillis() - lastTime < 30000) return;
lastTime = System.currentTimeMillis();
try (RocksDBSession session = getRocksDBSession(graphName)) {
if (session == null) {
// log.error("Failed to fetch the RocksDBSession with graph's name: [ " +
// graph + " ]");
return;
}
Statistics statistics = session.getRocksDbStats();
for (final TickerType ticker : TICKERS) {
tickerCounteMap.put(ticker, statistics.getTickerCount(ticker));
}
for (final HistogramType histogram : HISTOGRAMS) {
histogramDataMap.put(histogram, statistics.getHistogramData(histogram));
}
}
}
public long getTickerCount(TickerType tickerType) {
this.loadData();
return tickerCounteMap.containsKey(tickerType) ? tickerCounteMap.get(tickerType) : 0;
}
public HistogramData getHistogramData(HistogramType histogramType) {
this.loadData();
return histogramDataMap.get(histogramType);
}
}
private static <S, T> T getValue(S stat, Function<S, T> fun, T defaultValue) {
if (stat == null) return defaultValue;
return fun.apply(stat);
}
private static class HistogramDataWrapper {
private HistogramData data = new HistogramData(0d, 0d, 0d, 0d, 0d);
private long ts = System.currentTimeMillis() - 30_000;
private final Supplier<HistogramData> supplier;
private final HistogramType histogramType;
HistogramDataWrapper(HistogramType histogramType, Supplier<HistogramData> supplier) {
this.supplier = supplier;
this.histogramType = histogramType;
}
private HistogramData getData() {
if (System.currentTimeMillis() - this.ts > 30_000) {
HistogramData buf = this.supplier.get();
if (buf != null) {
this.data = buf;
this.ts = System.currentTimeMillis();
}
}
return this.data;
}
public double getMedian() {
return getData().getMedian();
}
public double getPercentile95() {
return getData().getPercentile95();
}
public double getPercentile99() {
return getData().getPercentile99();
}
public double getAverage() {
return getData().getAverage();
}
public double getStandardDeviation() {
return getData().getStandardDeviation();
}
public double getMax() {
return getData().getMax();
}
public long getCount() {
return getData().getCount();
}
public long getSum() {
return getData().getSum();
}
public double getMin() {
return getData().getMin();
}
}
}

330
hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java
View File

@ -1,165 +1,165 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.metrics;
import org.rocksdb.HistogramType;
import org.rocksdb.TickerType;
/**
* @author lynn.bond@hotmail.com on 2021/12/30
*/
public final class RocksDBMetricsConst {
public static final String PREFIX = "rocks.stats";
public static final String LABELS = "quantile";
public static final String LABEL_50 = "0.5";
public static final String LABEL_95 = "0.95";
public static final String LABEL_99 = "0.99";
// Tickers - RocksDB equivalent of counters
public static final TickerType[] TICKERS = {
TickerType.BLOCK_CACHE_ADD,
TickerType.BLOCK_CACHE_HIT,
TickerType.BLOCK_CACHE_ADD_FAILURES,
TickerType.BLOCK_CACHE_INDEX_MISS,
TickerType.BLOCK_CACHE_INDEX_HIT,
TickerType.BLOCK_CACHE_INDEX_ADD,
TickerType.BLOCK_CACHE_INDEX_BYTES_INSERT,
TickerType.BLOCK_CACHE_INDEX_BYTES_EVICT,
TickerType.BLOCK_CACHE_FILTER_MISS,
TickerType.BLOCK_CACHE_FILTER_HIT,
TickerType.BLOCK_CACHE_FILTER_ADD,
TickerType.BLOCK_CACHE_FILTER_BYTES_INSERT,
TickerType.BLOCK_CACHE_FILTER_BYTES_EVICT,
TickerType.BLOCK_CACHE_DATA_MISS,
TickerType.BLOCK_CACHE_DATA_HIT,
TickerType.BLOCK_CACHE_DATA_ADD,
TickerType.BLOCK_CACHE_DATA_BYTES_INSERT,
TickerType.BLOCK_CACHE_BYTES_READ,
TickerType.BLOCK_CACHE_BYTES_WRITE,
TickerType.BLOOM_FILTER_USEFUL,
TickerType.PERSISTENT_CACHE_HIT,
TickerType.PERSISTENT_CACHE_MISS,
TickerType.SIM_BLOCK_CACHE_HIT,
TickerType.SIM_BLOCK_CACHE_MISS,
TickerType.MEMTABLE_HIT,
TickerType.MEMTABLE_MISS,
TickerType.GET_HIT_L0,
TickerType.GET_HIT_L1,
TickerType.GET_HIT_L2_AND_UP,
TickerType.COMPACTION_KEY_DROP_NEWER_ENTRY,
TickerType.COMPACTION_KEY_DROP_OBSOLETE,
TickerType.COMPACTION_KEY_DROP_RANGE_DEL,
TickerType.COMPACTION_KEY_DROP_USER,
TickerType.COMPACTION_RANGE_DEL_DROP_OBSOLETE,
TickerType.NUMBER_KEYS_WRITTEN,
TickerType.NUMBER_KEYS_READ,
TickerType.NUMBER_KEYS_UPDATED,
TickerType.BYTES_WRITTEN,
TickerType.BYTES_READ,
TickerType.NUMBER_DB_SEEK,
TickerType.NUMBER_DB_NEXT,
TickerType.NUMBER_DB_PREV,
TickerType.NUMBER_DB_SEEK_FOUND,
TickerType.NUMBER_DB_NEXT_FOUND,
TickerType.NUMBER_DB_PREV_FOUND,
TickerType.ITER_BYTES_READ,
TickerType.NO_FILE_CLOSES,
TickerType.NO_FILE_OPENS,
TickerType.NO_FILE_ERRORS,
// TickerType.STALL_L0_SLOWDOWN_MICROS,
// TickerType.STALL_MEMTABLE_COMPACTION_MICROS,
// TickerType.STALL_L0_NUM_FILES_MICROS,
TickerType.STALL_MICROS,
TickerType.DB_MUTEX_WAIT_MICROS,
TickerType.RATE_LIMIT_DELAY_MILLIS,
TickerType.NO_ITERATORS,
TickerType.NUMBER_MULTIGET_BYTES_READ,
TickerType.NUMBER_MULTIGET_KEYS_READ,
TickerType.NUMBER_MULTIGET_CALLS,
TickerType.NUMBER_FILTERED_DELETES,
TickerType.NUMBER_MERGE_FAILURES,
TickerType.BLOOM_FILTER_PREFIX_CHECKED,
TickerType.BLOOM_FILTER_PREFIX_USEFUL,
TickerType.NUMBER_OF_RESEEKS_IN_ITERATION,
TickerType.GET_UPDATES_SINCE_CALLS,
TickerType.BLOCK_CACHE_COMPRESSED_MISS,
TickerType.BLOCK_CACHE_COMPRESSED_HIT,
TickerType.BLOCK_CACHE_COMPRESSED_ADD,
TickerType.BLOCK_CACHE_COMPRESSED_ADD_FAILURES,
TickerType.WAL_FILE_SYNCED,
TickerType.WAL_FILE_BYTES,
TickerType.WRITE_DONE_BY_SELF,
TickerType.WRITE_DONE_BY_OTHER,
TickerType.WRITE_TIMEDOUT,
TickerType.WRITE_WITH_WAL,
TickerType.COMPACT_READ_BYTES,
TickerType.COMPACT_WRITE_BYTES,
TickerType.FLUSH_WRITE_BYTES,
TickerType.NUMBER_DIRECT_LOAD_TABLE_PROPERTIES,
TickerType.NUMBER_SUPERVERSION_ACQUIRES,
TickerType.NUMBER_SUPERVERSION_RELEASES,
TickerType.NUMBER_SUPERVERSION_CLEANUPS,
TickerType.NUMBER_BLOCK_COMPRESSED,
TickerType.NUMBER_BLOCK_DECOMPRESSED,
TickerType.NUMBER_BLOCK_NOT_COMPRESSED,
TickerType.MERGE_OPERATION_TOTAL_TIME,
TickerType.FILTER_OPERATION_TOTAL_TIME,
TickerType.ROW_CACHE_HIT,
TickerType.ROW_CACHE_MISS,
TickerType.READ_AMP_ESTIMATE_USEFUL_BYTES,
TickerType.READ_AMP_TOTAL_READ_BYTES,
TickerType.NUMBER_RATE_LIMITER_DRAINS,
TickerType.NUMBER_ITER_SKIP,
TickerType.NUMBER_MULTIGET_KEYS_FOUND,
};
// Histograms - treated as prometheus summaries
public static final HistogramType[] HISTOGRAMS = {
HistogramType.DB_GET,
HistogramType.DB_WRITE,
HistogramType.COMPACTION_TIME,
HistogramType.SUBCOMPACTION_SETUP_TIME,
HistogramType.TABLE_SYNC_MICROS,
HistogramType.COMPACTION_OUTFILE_SYNC_MICROS,
HistogramType.WAL_FILE_SYNC_MICROS,
HistogramType.MANIFEST_FILE_SYNC_MICROS,
HistogramType.TABLE_OPEN_IO_MICROS,
HistogramType.DB_MULTIGET,
HistogramType.READ_BLOCK_COMPACTION_MICROS,
HistogramType.READ_BLOCK_GET_MICROS,
HistogramType.WRITE_RAW_BLOCK_MICROS,
HistogramType.STALL_L0_SLOWDOWN_COUNT,
HistogramType.STALL_MEMTABLE_COMPACTION_COUNT,
HistogramType.STALL_L0_NUM_FILES_COUNT,
HistogramType.HARD_RATE_LIMIT_DELAY_COUNT,
HistogramType.SOFT_RATE_LIMIT_DELAY_COUNT,
HistogramType.NUM_FILES_IN_SINGLE_COMPACTION,
HistogramType.DB_SEEK,
HistogramType.WRITE_STALL,
HistogramType.SST_READ_MICROS,
HistogramType.NUM_SUBCOMPACTIONS_SCHEDULED,
HistogramType.BYTES_PER_READ,
HistogramType.BYTES_PER_WRITE,
HistogramType.BYTES_PER_MULTIGET,
HistogramType.BYTES_COMPRESSED,
HistogramType.BYTES_DECOMPRESSED,
HistogramType.COMPRESSION_TIMES_NANOS,
HistogramType.DECOMPRESSION_TIMES_NANOS,
HistogramType.READ_NUM_MERGE_OPERANDS,
};
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.metrics;
import org.rocksdb.HistogramType;
import org.rocksdb.TickerType;
/**
* @author lynn.bond@hotmail.com on 2021/12/30
*/
public final class RocksDBMetricsConst {
public static final String PREFIX = "rocks.stats";
public static final String LABELS = "quantile";
public static final String LABEL_50 = "0.5";
public static final String LABEL_95 = "0.95";
public static final String LABEL_99 = "0.99";
// Tickers - RocksDB equivalent of counters
public static final TickerType[] TICKERS = {
TickerType.BLOCK_CACHE_ADD,
TickerType.BLOCK_CACHE_HIT,
TickerType.BLOCK_CACHE_ADD_FAILURES,
TickerType.BLOCK_CACHE_INDEX_MISS,
TickerType.BLOCK_CACHE_INDEX_HIT,
TickerType.BLOCK_CACHE_INDEX_ADD,
TickerType.BLOCK_CACHE_INDEX_BYTES_INSERT,
TickerType.BLOCK_CACHE_INDEX_BYTES_EVICT,
TickerType.BLOCK_CACHE_FILTER_MISS,
TickerType.BLOCK_CACHE_FILTER_HIT,
TickerType.BLOCK_CACHE_FILTER_ADD,
TickerType.BLOCK_CACHE_FILTER_BYTES_INSERT,
TickerType.BLOCK_CACHE_FILTER_BYTES_EVICT,
TickerType.BLOCK_CACHE_DATA_MISS,
TickerType.BLOCK_CACHE_DATA_HIT,
TickerType.BLOCK_CACHE_DATA_ADD,
TickerType.BLOCK_CACHE_DATA_BYTES_INSERT,
TickerType.BLOCK_CACHE_BYTES_READ,
TickerType.BLOCK_CACHE_BYTES_WRITE,
TickerType.BLOOM_FILTER_USEFUL,
TickerType.PERSISTENT_CACHE_HIT,
TickerType.PERSISTENT_CACHE_MISS,
TickerType.SIM_BLOCK_CACHE_HIT,
TickerType.SIM_BLOCK_CACHE_MISS,
TickerType.MEMTABLE_HIT,
TickerType.MEMTABLE_MISS,
TickerType.GET_HIT_L0,
TickerType.GET_HIT_L1,
TickerType.GET_HIT_L2_AND_UP,
TickerType.COMPACTION_KEY_DROP_NEWER_ENTRY,
TickerType.COMPACTION_KEY_DROP_OBSOLETE,
TickerType.COMPACTION_KEY_DROP_RANGE_DEL,
TickerType.COMPACTION_KEY_DROP_USER,
TickerType.COMPACTION_RANGE_DEL_DROP_OBSOLETE,
TickerType.NUMBER_KEYS_WRITTEN,
TickerType.NUMBER_KEYS_READ,
TickerType.NUMBER_KEYS_UPDATED,
TickerType.BYTES_WRITTEN,
TickerType.BYTES_READ,
TickerType.NUMBER_DB_SEEK,
TickerType.NUMBER_DB_NEXT,
TickerType.NUMBER_DB_PREV,
TickerType.NUMBER_DB_SEEK_FOUND,
TickerType.NUMBER_DB_NEXT_FOUND,
TickerType.NUMBER_DB_PREV_FOUND,
TickerType.ITER_BYTES_READ,
TickerType.NO_FILE_CLOSES,
TickerType.NO_FILE_OPENS,
TickerType.NO_FILE_ERRORS,
// TickerType.STALL_L0_SLOWDOWN_MICROS,
// TickerType.STALL_MEMTABLE_COMPACTION_MICROS,
// TickerType.STALL_L0_NUM_FILES_MICROS,
TickerType.STALL_MICROS,
TickerType.DB_MUTEX_WAIT_MICROS,
TickerType.RATE_LIMIT_DELAY_MILLIS,
TickerType.NO_ITERATORS,
TickerType.NUMBER_MULTIGET_BYTES_READ,
TickerType.NUMBER_MULTIGET_KEYS_READ,
TickerType.NUMBER_MULTIGET_CALLS,
TickerType.NUMBER_FILTERED_DELETES,
TickerType.NUMBER_MERGE_FAILURES,
TickerType.BLOOM_FILTER_PREFIX_CHECKED,
TickerType.BLOOM_FILTER_PREFIX_USEFUL,
TickerType.NUMBER_OF_RESEEKS_IN_ITERATION,
TickerType.GET_UPDATES_SINCE_CALLS,
TickerType.BLOCK_CACHE_COMPRESSED_MISS,
TickerType.BLOCK_CACHE_COMPRESSED_HIT,
TickerType.BLOCK_CACHE_COMPRESSED_ADD,
TickerType.BLOCK_CACHE_COMPRESSED_ADD_FAILURES,
TickerType.WAL_FILE_SYNCED,
TickerType.WAL_FILE_BYTES,
TickerType.WRITE_DONE_BY_SELF,
TickerType.WRITE_DONE_BY_OTHER,
TickerType.WRITE_TIMEDOUT,
TickerType.WRITE_WITH_WAL,
TickerType.COMPACT_READ_BYTES,
TickerType.COMPACT_WRITE_BYTES,
TickerType.FLUSH_WRITE_BYTES,
TickerType.NUMBER_DIRECT_LOAD_TABLE_PROPERTIES,
TickerType.NUMBER_SUPERVERSION_ACQUIRES,
TickerType.NUMBER_SUPERVERSION_RELEASES,
TickerType.NUMBER_SUPERVERSION_CLEANUPS,
TickerType.NUMBER_BLOCK_COMPRESSED,
TickerType.NUMBER_BLOCK_DECOMPRESSED,
TickerType.NUMBER_BLOCK_NOT_COMPRESSED,
TickerType.MERGE_OPERATION_TOTAL_TIME,
TickerType.FILTER_OPERATION_TOTAL_TIME,
TickerType.ROW_CACHE_HIT,
TickerType.ROW_CACHE_MISS,
TickerType.READ_AMP_ESTIMATE_USEFUL_BYTES,
TickerType.READ_AMP_TOTAL_READ_BYTES,
TickerType.NUMBER_RATE_LIMITER_DRAINS,
TickerType.NUMBER_ITER_SKIP,
TickerType.NUMBER_MULTIGET_KEYS_FOUND,
};
// Histograms - treated as prometheus summaries
public static final HistogramType[] HISTOGRAMS = {
HistogramType.DB_GET,
HistogramType.DB_WRITE,
HistogramType.COMPACTION_TIME,
HistogramType.SUBCOMPACTION_SETUP_TIME,
HistogramType.TABLE_SYNC_MICROS,
HistogramType.COMPACTION_OUTFILE_SYNC_MICROS,
HistogramType.WAL_FILE_SYNC_MICROS,
HistogramType.MANIFEST_FILE_SYNC_MICROS,
HistogramType.TABLE_OPEN_IO_MICROS,
HistogramType.DB_MULTIGET,
HistogramType.READ_BLOCK_COMPACTION_MICROS,
HistogramType.READ_BLOCK_GET_MICROS,
HistogramType.WRITE_RAW_BLOCK_MICROS,
HistogramType.STALL_L0_SLOWDOWN_COUNT,
HistogramType.STALL_MEMTABLE_COMPACTION_COUNT,
HistogramType.STALL_L0_NUM_FILES_COUNT,
HistogramType.HARD_RATE_LIMIT_DELAY_COUNT,
HistogramType.SOFT_RATE_LIMIT_DELAY_COUNT,
HistogramType.NUM_FILES_IN_SINGLE_COMPACTION,
HistogramType.DB_SEEK,
HistogramType.WRITE_STALL,
HistogramType.SST_READ_MICROS,
HistogramType.NUM_SUBCOMPACTIONS_SCHEDULED,
HistogramType.BYTES_PER_READ,
HistogramType.BYTES_PER_WRITE,
HistogramType.BYTES_PER_MULTIGET,
HistogramType.BYTES_COMPRESSED,
HistogramType.BYTES_DECOMPRESSED,
HistogramType.COMPRESSION_TIMES_NANOS,
HistogramType.DECOMPRESSION_TIMES_NANOS,
HistogramType.READ_NUM_MERGE_OPERANDS,
};
}

224
hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/StoreMetrics.java
View File

@ -1,112 +1,112 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.metrics;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.meta.Partition;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
/**
* @author lynn.bond@hotmail.com on 2021/12/28
*/
public final class StoreMetrics {
public final static String PREFIX = "hg";
private final static HgStoreEngine storeEngine = HgStoreEngine.getInstance();
private final static AtomicInteger graphs = new AtomicInteger(0);
private static MeterRegistry registry;
private StoreMetrics() {
}
public synchronized static void init(MeterRegistry meterRegistry) {
if (registry == null) {
registry = meterRegistry;
registerMeters();
}
}
private static void registerMeters() {
Gauge.builder(PREFIX + ".up", () -> 1).register(registry);
Gauge.builder(PREFIX + ".graphs", StoreMetrics::updateGraphs)
.description("Number of graphs stored in this node")
.register(registry);
}
private static int getGraphs() {
return getGraphPartitions().size();
}
private static int updateGraphs() {
int buf = getGraphs();
if (buf != graphs.get()) {
graphs.set(buf);
registerPartitionGauge();
}
return buf;
}
private static void registerPartitionGauge() {
Map<String, Map<Integer, Partition>> map = getGraphPartitions();
map.forEach((k, v) -> Gauge.builder(PREFIX + ".partitions", new PartitionsGetter(k))
.description("Number of partitions stored in the node")
.tag("graph", k)
.register(registry));
}
private static int getPartitions(String graph) {
Map<Integer, Partition> map = getGraphPartitions().get(graph);
if (map == null) {
return 0;
} else {
return map.size();
}
}
private static Map<String, Map<Integer, Partition>> getGraphPartitions() {
Map<String, Map<Integer, Partition>> map =
storeEngine.getPartitionManager().getPartitions();
if (map == null) {
return Collections.emptyMap();
}
return map;
}
private static class PartitionsGetter implements Supplier<Number> {
private final String graph;
PartitionsGetter(String graph) {
this.graph = graph;
}
@Override
public Number get() {
return getPartitions(this.graph);
}
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.metrics;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import org.apache.hugegraph.store.HgStoreEngine;
import org.apache.hugegraph.store.meta.Partition;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
/**
* @author lynn.bond@hotmail.com on 2021/12/28
*/
public final class StoreMetrics {
public final static String PREFIX = "hg";
private final static HgStoreEngine storeEngine = HgStoreEngine.getInstance();
private final static AtomicInteger graphs = new AtomicInteger(0);
private static MeterRegistry registry;
private StoreMetrics() {
}
public synchronized static void init(MeterRegistry meterRegistry) {
if (registry == null) {
registry = meterRegistry;
registerMeters();
}
}
private static void registerMeters() {
Gauge.builder(PREFIX + ".up", () -> 1).register(registry);
Gauge.builder(PREFIX + ".graphs", StoreMetrics::updateGraphs)
.description("Number of graphs stored in this node")
.register(registry);
}
private static int getGraphs() {
return getGraphPartitions().size();
}
private static int updateGraphs() {
int buf = getGraphs();
if (buf != graphs.get()) {
graphs.set(buf);
registerPartitionGauge();
}
return buf;
}
private static void registerPartitionGauge() {
Map<String, Map<Integer, Partition>> map = getGraphPartitions();
map.forEach((k, v) -> Gauge.builder(PREFIX + ".partitions", new PartitionsGetter(k))
.description("Number of partitions stored in the node")
.tag("graph", k)
.register(registry));
}
private static int getPartitions(String graph) {
Map<Integer, Partition> map = getGraphPartitions().get(graph);
if (map == null) {
return 0;
} else {
return map.size();
}
}
private static Map<String, Map<Integer, Partition>> getGraphPartitions() {
Map<String, Map<Integer, Partition>> map =
storeEngine.getPartitionManager().getPartitions();
if (map == null) {
return Collections.emptyMap();
}
return map;
}
private static class PartitionsGetter implements Supplier<Number> {
private final String graph;
PartitionsGetter(String graph) {
this.graph = graph;
}
@Override
public Number get() {
return getPartitions(this.graph);
}
}
}

354
hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Base58.java
View File

@ -1,177 +1,177 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.util;
import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
/**
* Created with IntelliJ IDEA.
* User: noah
* Date: 8/2/13
* Time: 10:36 AM
* To change this template use File | Settings | File Templates.
*/
public class Base58 {
public static final char[] ALPHABET =
"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray();
private static final int[] INDEXES = new int[128];
static {
for (int i = 0; i < INDEXES.length; i++) {
INDEXES[i] = -1;
}
for (int i = 0; i < ALPHABET.length; i++) {
INDEXES[ALPHABET[i]] = i;
}
}
/**
* Encodes the given bytes in base58. No checksum is appended.
*/
public static String encode(byte[] input) {
if (input.length == 0) {
return "";
}
input = copyOfRange(input, 0, input.length);
// Count leading zeroes.
int zeroCount = 0;
while (zeroCount < input.length && input[zeroCount] == 0) {
++zeroCount;
}
// The actual encoding.
byte[] temp = new byte[input.length * 2];
int j = temp.length;
int startAt = zeroCount;
while (startAt < input.length) {
byte mod = divmod58(input, startAt);
if (input[startAt] == 0) {
++startAt;
}
temp[--j] = (byte) ALPHABET[mod];
}
// Strip extra '1' if there are some after decoding.
while (j < temp.length && temp[j] == ALPHABET[0]) {
++j;
}
// Add as many leading '1' as there were leading zeros.
while (--zeroCount >= 0) {
temp[--j] = (byte) ALPHABET[0];
}
byte[] output = copyOfRange(temp, j, temp.length);
return new String(output, StandardCharsets.US_ASCII);
}
public static byte[] decode(String input) throws IllegalArgumentException {
if (input.length() == 0) {
return new byte[0];
}
byte[] input58 = new byte[input.length()];
// Transform the String to a base58 byte sequence
for (int i = 0; i < input.length(); ++i) {
char c = input.charAt(i);
int digit58 = -1;
if (c >= 0 && c < 128) {
digit58 = INDEXES[c];
}
if (digit58 < 0) {
throw new IllegalArgumentException("Illegal character " + c + " at " + i);
}
input58[i] = (byte) digit58;
}
// Count leading zeroes
int zeroCount = 0;
while (zeroCount < input58.length && input58[zeroCount] == 0) {
++zeroCount;
}
// The encoding
byte[] temp = new byte[input.length()];
int j = temp.length;
int startAt = zeroCount;
while (startAt < input58.length) {
byte mod = divmod256(input58, startAt);
if (input58[startAt] == 0) {
++startAt;
}
temp[--j] = mod;
}
// Do no add extra leading zeroes, move j to first non null byte.
while (j < temp.length && temp[j] == 0) {
++j;
}
return copyOfRange(temp, j - zeroCount, temp.length);
}
public static BigInteger decodeToBigInteger(String input) throws IllegalArgumentException {
return new BigInteger(1, decode(input));
}
//
// number -> number / 58, returns number % 58
//
private static byte divmod58(byte[] number, int startAt) {
int remainder = 0;
for (int i = startAt; i < number.length; i++) {
int digit256 = (int) number[i] & 0xFF;
int temp = remainder * 256 + digit256;
number[i] = (byte) (temp / 58);
remainder = temp % 58;
}
return (byte) remainder;
}
//
// number -> number / 256, returns number % 256
//
private static byte divmod256(byte[] number58, int startAt) {
int remainder = 0;
for (int i = startAt; i < number58.length; i++) {
int digit58 = (int) number58[i] & 0xFF;
int temp = remainder * 58 + digit58;
number58[i] = (byte) (temp / 256);
remainder = temp % 256;
}
return (byte) remainder;
}
private static byte[] copyOfRange(byte[] source, int from, int to) {
byte[] range = new byte[to - from];
System.arraycopy(source, from, range, 0, range.length);
return range;
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.util;
import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
/**
* Created with IntelliJ IDEA.
* User: noah
* Date: 8/2/13
* Time: 10:36 AM
* To change this template use File | Settings | File Templates.
*/
public class Base58 {
public static final char[] ALPHABET =
"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray();
private static final int[] INDEXES = new int[128];
static {
for (int i = 0; i < INDEXES.length; i++) {
INDEXES[i] = -1;
}
for (int i = 0; i < ALPHABET.length; i++) {
INDEXES[ALPHABET[i]] = i;
}
}
/**
* Encodes the given bytes in base58. No checksum is appended.
*/
public static String encode(byte[] input) {
if (input.length == 0) {
return "";
}
input = copyOfRange(input, 0, input.length);
// Count leading zeroes.
int zeroCount = 0;
while (zeroCount < input.length && input[zeroCount] == 0) {
++zeroCount;
}
// The actual encoding.
byte[] temp = new byte[input.length * 2];
int j = temp.length;
int startAt = zeroCount;
while (startAt < input.length) {
byte mod = divmod58(input, startAt);
if (input[startAt] == 0) {
++startAt;
}
temp[--j] = (byte) ALPHABET[mod];
}
// Strip extra '1' if there are some after decoding.
while (j < temp.length && temp[j] == ALPHABET[0]) {
++j;
}
// Add as many leading '1' as there were leading zeros.
while (--zeroCount >= 0) {
temp[--j] = (byte) ALPHABET[0];
}
byte[] output = copyOfRange(temp, j, temp.length);
return new String(output, StandardCharsets.US_ASCII);
}
public static byte[] decode(String input) throws IllegalArgumentException {
if (input.length() == 0) {
return new byte[0];
}
byte[] input58 = new byte[input.length()];
// Transform the String to a base58 byte sequence
for (int i = 0; i < input.length(); ++i) {
char c = input.charAt(i);
int digit58 = -1;
if (c >= 0 && c < 128) {
digit58 = INDEXES[c];
}
if (digit58 < 0) {
throw new IllegalArgumentException("Illegal character " + c + " at " + i);
}
input58[i] = (byte) digit58;
}
// Count leading zeroes
int zeroCount = 0;
while (zeroCount < input58.length && input58[zeroCount] == 0) {
++zeroCount;
}
// The encoding
byte[] temp = new byte[input.length()];
int j = temp.length;
int startAt = zeroCount;
while (startAt < input58.length) {
byte mod = divmod256(input58, startAt);
if (input58[startAt] == 0) {
++startAt;
}
temp[--j] = mod;
}
// Do no add extra leading zeroes, move j to first non null byte.
while (j < temp.length && temp[j] == 0) {
++j;
}
return copyOfRange(temp, j - zeroCount, temp.length);
}
public static BigInteger decodeToBigInteger(String input) throws IllegalArgumentException {
return new BigInteger(1, decode(input));
}
//
// number -> number / 58, returns number % 58
//
private static byte divmod58(byte[] number, int startAt) {
int remainder = 0;
for (int i = startAt; i < number.length; i++) {
int digit256 = (int) number[i] & 0xFF;
int temp = remainder * 256 + digit256;
number[i] = (byte) (temp / 58);
remainder = temp % 58;
}
return (byte) remainder;
}
//
// number -> number / 256, returns number % 256
//
private static byte divmod256(byte[] number58, int startAt) {
int remainder = 0;
for (int i = startAt; i < number58.length; i++) {
int digit58 = (int) number58[i] & 0xFF;
int temp = remainder * 58 + digit58;
number58[i] = (byte) (temp / 256);
remainder = temp % 256;
}
return (byte) remainder;
}
private static byte[] copyOfRange(byte[] source, int from, int to) {
byte[] range = new byte[to - from];
System.arraycopy(source, from, range, 0, range.length);
return range;
}
}

348
hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgBufferProxy.java
View File

@ -1,174 +1,174 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.util;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Executor;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
import java.util.function.Supplier;
import javax.annotation.CheckForNull;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com on 2022/3/15
* @version 0.2.0
*/
@Slf4j
public final class HgBufferProxy<T> {
private final BlockingQueue<Supplier<T>> queue;
private final AtomicBoolean closed = new AtomicBoolean(false);
private final ReentrantLock lock = new ReentrantLock();
private final Runnable applier;
private final Executor executor;
public static HgBufferProxy of(Executor executor, Runnable applier) {
HgAssert.isArgumentNotNull(applier, "applier");
HgAssert.isArgumentNotNull(executor, "executor");
return new HgBufferProxy(executor, applier);
}
private HgBufferProxy(Executor executor, Runnable applier) {
this.executor = executor;
this.applier = applier;
this.queue = new LinkedBlockingQueue<>();
}
public void send(T t) {
if (t == null) {
throw new IllegalArgumentException("the argument t is null");
}
if (this.closed.get()) {
log.warn("the proxy has been closed");
return;
}
this.lock.lock();
try {
this.queue.offer(() -> t);
} finally {
lock.unlock();
}
}
/**
* return an item from the chan
*
* @throws RuntimeException when fail to receive an item.
*/
@CheckForNull
public T receive(int timeoutSeconds) {
return receive(timeoutSeconds, (timeout) -> {
throw new RuntimeException("Timeout, max time: " + timeout + " seconds;");
});
}
private void apply() {
this.lock.lock();
try {
if (!this.closed.get()) {
this.executor.execute(this.applier);
Thread.yield();
}
} finally {
this.lock.unlock();
}
}
/**
* return an item from the chan
*
* @return null when the chan has been closed
* @throws RuntimeException
*/
@CheckForNull
public T receive(int timeoutSeconds, Consumer<Integer> timeoutCallBack) {
Supplier<T> s = null;
if (this.closed.get()) {
s = this.queue.poll();
return s != null ? s.get() : null;
}
if (this.queue.size() <= 1) {
this.apply();
}
lock.lock();
try {
if (this.isClosed()) {
s = this.queue.poll();
return s != null ? s.get() : null;
}
} finally {
lock.unlock();
}
try {
s = this.queue.poll(timeoutSeconds, TimeUnit.SECONDS);
} catch (Throwable t) {
log.error("Failed to receive a item from chan. cause by: ", t);
throw new RuntimeException(t);
}
if (s == null) {
if (this.closed.get()) {
s = this.queue.poll();
} else {
if (timeoutCallBack == null) {
throw new RuntimeException(
"Timeout, max time: " + timeoutSeconds + " seconds;");
} else {
timeoutCallBack.accept(timeoutSeconds);
}
}
}
return s != null ? s.get() : null;
}
public boolean isClosed() {
return this.closed.get();
}
/**
* @throws RuntimeException when fail to close the chan
*/
public void close() {
if (this.closed.get()) {
return;
}
lock.lock();
this.closed.set(true);
try {
this.queue.offer(() -> null);
} finally {
lock.unlock();
}
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.util;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Executor;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
import java.util.function.Supplier;
import javax.annotation.CheckForNull;
import lombok.extern.slf4j.Slf4j;
/**
* @author lynn.bond@hotmail.com on 2022/3/15
* @version 0.2.0
*/
@Slf4j
public final class HgBufferProxy<T> {
private final BlockingQueue<Supplier<T>> queue;
private final AtomicBoolean closed = new AtomicBoolean(false);
private final ReentrantLock lock = new ReentrantLock();
private final Runnable applier;
private final Executor executor;
public static HgBufferProxy of(Executor executor, Runnable applier) {
HgAssert.isArgumentNotNull(applier, "applier");
HgAssert.isArgumentNotNull(executor, "executor");
return new HgBufferProxy(executor, applier);
}
private HgBufferProxy(Executor executor, Runnable applier) {
this.executor = executor;
this.applier = applier;
this.queue = new LinkedBlockingQueue<>();
}
public void send(T t) {
if (t == null) {
throw new IllegalArgumentException("the argument t is null");
}
if (this.closed.get()) {
log.warn("the proxy has been closed");
return;
}
this.lock.lock();
try {
this.queue.offer(() -> t);
} finally {
lock.unlock();
}
}
/**
* return an item from the chan
*
* @throws RuntimeException when fail to receive an item.
*/
@CheckForNull
public T receive(int timeoutSeconds) {
return receive(timeoutSeconds, (timeout) -> {
throw new RuntimeException("Timeout, max time: " + timeout + " seconds;");
});
}
private void apply() {
this.lock.lock();
try {
if (!this.closed.get()) {
this.executor.execute(this.applier);
Thread.yield();
}
} finally {
this.lock.unlock();
}
}
/**
* return an item from the chan
*
* @return null when the chan has been closed
* @throws RuntimeException
*/
@CheckForNull
public T receive(int timeoutSeconds, Consumer<Integer> timeoutCallBack) {
Supplier<T> s = null;
if (this.closed.get()) {
s = this.queue.poll();
return s != null ? s.get() : null;
}
if (this.queue.size() <= 1) {
this.apply();
}
lock.lock();
try {
if (this.isClosed()) {
s = this.queue.poll();
return s != null ? s.get() : null;
}
} finally {
lock.unlock();
}
try {
s = this.queue.poll(timeoutSeconds, TimeUnit.SECONDS);
} catch (Throwable t) {
log.error("Failed to receive a item from chan. cause by: ", t);
throw new RuntimeException(t);
}
if (s == null) {
if (this.closed.get()) {
s = this.queue.poll();
} else {
if (timeoutCallBack == null) {
throw new RuntimeException(
"Timeout, max time: " + timeoutSeconds + " seconds;");
} else {
timeoutCallBack.accept(timeoutSeconds);
}
}
}
return s != null ? s.get() : null;
}
public boolean isClosed() {
return this.closed.get();
}
/**
* @throws RuntimeException when fail to close the chan
*/
public void close() {
if (this.closed.get()) {
return;
}
lock.lock();
this.closed.set(true);
try {
this.queue.offer(() -> null);
} finally {
lock.unlock();
}
}
}

172
hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgRegexUtil.java
View File

@ -1,86 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.util;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author lynn.bond@hotmail.com created on 2022/03/07
*/
public class HgRegexUtil {
public static String getGroupValue(String regex, String source, int groupId) {
if (regex == null || "".equals(regex) || source == null || "".equals(source)) return null;
String value = "";
Pattern p = Pattern.compile(regex);
Matcher m = p.matcher(source);
while (m.find()) {
for (int i = 0; i <= m.groupCount(); i++) {
if (i == groupId) {
value = m.group(i);
}
}
}
return value;
}
public static List<String> toGroupValues(String regex, String source) {
if (regex == null || "".equals(regex) || source == null || "".equals(source)) return null;
Pattern p = Pattern.compile(regex);
Matcher m = p.matcher(source);
List<String> list = null;
while (m.find()) {
list = new ArrayList<>(m.groupCount());
for (int i = 0; i <= m.groupCount(); i++) {
list.add(m.group(i));
}
}
return list;
}
public static List<String> getMatchList(String regex, String source) {
if (regex == null || "".equals(regex) || source == null || "".equals(source)) return null;
Pattern p = Pattern.compile(regex, Pattern.MULTILINE);
Matcher m = p.matcher(source);
List<String> list = new ArrayList<String>();
while (m.find()) {
list.add(m.group(0));
}
return list.isEmpty() ? null : list;
}
public static void main(String[] args) {
List<String> res = toGroupValues("(replicator)(.+?:\\d+)(.*)",
"replicator_10.14.139.10:8081_append_entries_times");
if (res != null) {
res.stream().forEach(e -> System.out.println(e));
}
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.util;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author lynn.bond@hotmail.com created on 2022/03/07
*/
public class HgRegexUtil {
public static String getGroupValue(String regex, String source, int groupId) {
if (regex == null || "".equals(regex) || source == null || "".equals(source)) return null;
String value = "";
Pattern p = Pattern.compile(regex);
Matcher m = p.matcher(source);
while (m.find()) {
for (int i = 0; i <= m.groupCount(); i++) {
if (i == groupId) {
value = m.group(i);
}
}
}
return value;
}
public static List<String> toGroupValues(String regex, String source) {
if (regex == null || "".equals(regex) || source == null || "".equals(source)) return null;
Pattern p = Pattern.compile(regex);
Matcher m = p.matcher(source);
List<String> list = null;
while (m.find()) {
list = new ArrayList<>(m.groupCount());
for (int i = 0; i <= m.groupCount(); i++) {
list.add(m.group(i));
}
}
return list;
}
public static List<String> getMatchList(String regex, String source) {
if (regex == null || "".equals(regex) || source == null || "".equals(source)) return null;
Pattern p = Pattern.compile(regex, Pattern.MULTILINE);
Matcher m = p.matcher(source);
List<String> list = new ArrayList<String>();
while (m.find()) {
list.add(m.group(0));
}
return list.isEmpty() ? null : list;
}
public static void main(String[] args) {
List<String> res = toGroupValues("(replicator)(.+?:\\d+)(.*)",
"replicator_10.14.139.10:8081_append_entries_times");
if (res != null) {
res.stream().forEach(e -> System.out.println(e));
}
}
}

118
hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/JraftMetricsTest.java
View File

@ -1,59 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.metrics;
import java.util.Arrays;
import java.util.List;
import org.apache.hugegraph.store.node.util.HgRegexUtil;
import org.junit.Test;
/**
* @author lynn.bond@hotmail.com on 2022/3/7
*/
public class JraftMetricsTest {
@Test
public void testRefineLabel() {
String regex = "(replicator)(.+?:\\d+)(.*)";
String[] sources = {
"replicator_10.14.139.10:8081_append_entries_times",
"replicator10.14.139.10:8081appendentriestimes",
"replicator.10.14.139.10:8081.append.entries.times",
"replicator-10-14-139-10:8081-append-entries-times",
"replicator_hg_0_10_14_139_10:8081::100_replicate_inflights_count_min",
"replicasdf14-13dasfies-times",
};
Arrays.stream(sources).forEach(e -> {
System.out.println("--- " + e + " ---");
List<String> list = HgRegexUtil.toGroupValues(regex, e);
if (list != null) {
list.forEach(System.out::println);
} else {
System.out.println("NONE");
}
System.out.println();
});
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package org.apache.hugegraph.store.node.metrics;
import java.util.Arrays;
import java.util.List;
import org.apache.hugegraph.store.node.util.HgRegexUtil;
import org.junit.Test;
/**
* @author lynn.bond@hotmail.com on 2022/3/7
*/
public class JraftMetricsTest {
@Test
public void testRefineLabel() {
String regex = "(replicator)(.+?:\\d+)(.*)";
String[] sources = {
"replicator_10.14.139.10:8081_append_entries_times",
"replicator10.14.139.10:8081appendentriestimes",
"replicator.10.14.139.10:8081.append.entries.times",
"replicator-10-14-139-10:8081-append-entries-times",
"replicator_hg_0_10_14_139_10:8081::100_replicate_inflights_count_min",
"replicasdf14-13dasfies-times",
};
Arrays.stream(sources).forEach(e -> {
System.out.println("--- " + e + " ---");
List<String> list = HgRegexUtil.toGroupValues(regex, e);
if (list != null) {
list.forEach(System.out::println);
} else {
System.out.println("NONE");
}
System.out.println();
});
}
}

176
hg-store-rocksdb/pom.xml
View File

@ -1,89 +1,89 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hugegraph-store-root</artifactId>
<version>${revision}</version>
</parent>
<artifactId>hg-store-rocksdb</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hugegraph-common</artifactId>
<version>1.8.12</version>
<exclusions>
<exclusion>
<groupId>org.glassfish.jersey.inject</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.connectors</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-common</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
<version>7.7.3</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.22</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>2.15.0</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.83</version>
</dependency>
</dependencies>
<distributionManagement>
<repository>
<id>Baidu_Local</id>
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local</url>
</repository>
<snapshotRepository>
<id>Baidu_Local_Snapshots</id>
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots</url>
</snapshotRepository>
</distributionManagement>
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hugegraph-store-root</artifactId>
<version>${revision}</version>
</parent>
<artifactId>hg-store-rocksdb</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hugegraph-common</artifactId>
<version>1.8.12</version>
<exclusions>
<exclusion>
<groupId>org.glassfish.jersey.inject</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.connectors</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-common</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
<version>7.7.3</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.22</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>2.15.0</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.83</version>
</dependency>
</dependencies>
<distributionManagement>
<repository>
<id>Baidu_Local</id>
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local</url>
</repository>
<snapshotRepository>
<id>Baidu_Local_Snapshots</id>
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots</url>
</snapshotRepository>
</distributionManagement>
</project>

512
pom.xml
View File

@ -1,256 +1,256 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hugegraph-store-root</artifactId>
<version>${revision}</version>
<packaging>pom</packaging>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.5.14</version>
</parent>
<modules>
<module>hg-store-common</module>
<module>hg-store-grpc</module>
<module>hg-store-client</module>
<module>hg-store-rocksdb</module>
<module>hg-store-core</module>
<module>hg-store-node</module>
<module>hg-store-dist</module>
<module>hg-store-cli</module>
<module>hg-store-test</module>
</modules>
<properties>
<revision>1.5.0</revision>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<hugegraph.server.version>3.6.4-SNAPSHOT</hugegraph.server.version>
<log4j2.version>2.15.0</log4j2.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-grpc</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-rocksdb</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-transfer</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>2.15.0</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<distributionManagement>
<repository>
<id>Baidu_Local</id>
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local</url>
</repository>
<snapshotRepository>
<id>Baidu_Local_Snapshots</id>
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.4</version>
<executions>
<execution>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>client-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>client-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>core-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>core-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>common-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>common-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>rocksdb-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>rocksdb-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<!--profile>
<id>server-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>server-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile-->
</profiles>
</project>
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hugegraph-store-root</artifactId>
<version>${revision}</version>
<packaging>pom</packaging>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.5.14</version>
</parent>
<modules>
<module>hg-store-common</module>
<module>hg-store-grpc</module>
<module>hg-store-client</module>
<module>hg-store-rocksdb</module>
<module>hg-store-core</module>
<module>hg-store-node</module>
<module>hg-store-dist</module>
<module>hg-store-cli</module>
<module>hg-store-test</module>
</modules>
<properties>
<revision>1.5.0</revision>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<hugegraph.server.version>3.6.4-SNAPSHOT</hugegraph.server.version>
<log4j2.version>2.15.0</log4j2.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-grpc</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-rocksdb</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hg-store-transfer</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>2.15.0</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<distributionManagement>
<repository>
<id>Baidu_Local</id>
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local</url>
</repository>
<snapshotRepository>
<id>Baidu_Local_Snapshots</id>
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.4</version>
<executions>
<execution>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>client-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>client-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>core-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>core-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>common-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>common-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>rocksdb-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>rocksdb-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<!--profile>
<id>server-test</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20</version>
<executions>
<execution>
<id>server-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>test</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile-->
</profiles>
</project>

262
settings.xml
View File

@ -1,132 +1,132 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<!-- 修改这里更改本地缓存路径
<localRepository>
/path/to/cache/
</localRepository>
-->
<!-- <servers>-->
<!-- <server>-->
<!-- <id>star-local</id>-->
<!-- <username>superstar</username>-->
<!-- <password>Superstar12345</password>-->
<!-- </server>-->
<!-- <server>-->
<!-- <id>star-snapshot</id>-->
<!-- <username>superstar</username>-->
<!-- <password>Superstar12345</password>-->
<!-- </server>-->
<!-- </servers>-->
<profiles>
<profile>
<id>baidu</id>
<repositories>
<repository>
<id>baidu-nexus</id>
<url>http://maven.baidu-int.com/nexus/content/groups/public</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>baidu-nexus-snapshot</id>
<url>http://maven.baidu-int.com/nexus/content/groups/public-snapshots</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<!-- 项目组自建仓库-->
<!-- <repository>-->
<!-- <id>star</id>-->
<!-- <url>http://10.14.139.8:8082/artifactory/star</url>-->
<!-- <releases>-->
<!-- <enabled>true</enabled>-->
<!-- <updatePolicy>always</updatePolicy>-->
<!-- </releases>-->
<!-- <snapshots>-->
<!-- <enabled>true</enabled>-->
<!-- <updatePolicy>always</updatePolicy>-->
<!-- </snapshots>-->
<!-- </repository>-->
<!-- end -->
<repository>
<id>Baidu_Local</id>
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>Baidu_Local_Snapshots</id>
<url>
http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots
</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
<updatePolicy>always</updatePolicy>
</snapshots>
</repository>
</repositories>
<pluginRepositories> <!-- plugin也需要从repository中获取 -->
<pluginRepository>
<id>baidu-nexus</id>
<url>http://maven.baidu-int.com/nexus/content/groups/public</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</pluginRepository>
<pluginRepository>
<id>baidu-nexus-snapshot</id>
<url>http://maven.baidu-int.com/nexus/content/groups/public-snapshots</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</pluginRepository>
</pluginRepositories>
</profile>
</profiles>
<activeProfiles>
<activeProfile>baidu</activeProfile> <!--别忘了激活配置 -->
</activeProfiles>
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<!-- 修改这里更改本地缓存路径
<localRepository>
/path/to/cache/
</localRepository>
-->
<!-- <servers>-->
<!-- <server>-->
<!-- <id>star-local</id>-->
<!-- <username>superstar</username>-->
<!-- <password>Superstar12345</password>-->
<!-- </server>-->
<!-- <server>-->
<!-- <id>star-snapshot</id>-->
<!-- <username>superstar</username>-->
<!-- <password>Superstar12345</password>-->
<!-- </server>-->
<!-- </servers>-->
<profiles>
<profile>
<id>baidu</id>
<repositories>
<repository>
<id>baidu-nexus</id>
<url>http://maven.baidu-int.com/nexus/content/groups/public</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>baidu-nexus-snapshot</id>
<url>http://maven.baidu-int.com/nexus/content/groups/public-snapshots</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<!-- 项目组自建仓库-->
<!-- <repository>-->
<!-- <id>star</id>-->
<!-- <url>http://10.14.139.8:8082/artifactory/star</url>-->
<!-- <releases>-->
<!-- <enabled>true</enabled>-->
<!-- <updatePolicy>always</updatePolicy>-->
<!-- </releases>-->
<!-- <snapshots>-->
<!-- <enabled>true</enabled>-->
<!-- <updatePolicy>always</updatePolicy>-->
<!-- </snapshots>-->
<!-- </repository>-->
<!-- end -->
<repository>
<id>Baidu_Local</id>
<url>http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>Baidu_Local_Snapshots</id>
<url>
http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots
</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
<updatePolicy>always</updatePolicy>
</snapshots>
</repository>
</repositories>
<pluginRepositories> <!-- plugin也需要从repository中获取 -->
<pluginRepository>
<id>baidu-nexus</id>
<url>http://maven.baidu-int.com/nexus/content/groups/public</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</pluginRepository>
<pluginRepository>
<id>baidu-nexus-snapshot</id>
<url>http://maven.baidu-int.com/nexus/content/groups/public-snapshots</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</pluginRepository>
</pluginRepositories>
</profile>
</profiles>
<activeProfiles>
<activeProfile>baidu</activeProfile> <!--别忘了激活配置 -->
</activeProfiles>
</settings>