From 6bce8990af50ef21da3017ebe277b50a2e49966e Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 26 Apr 2023 11:12:02 +0800 Subject: [PATCH] refact: fix all CRLF files to LF together (Breaking Change) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GraphPlatform-1906 梳理入库与查询流程 Change-Id: I5e7f57893c904e50be320e263a01283b147545a8 --- .../apache/hugegraph/store/cli/AppConfig.java | 116 +- .../store/cli/StoreConsoleApplication.java | 236 +- .../store/cli/scan/HgStoreCommitter.java | 174 +- .../store/cli/scan/HgStoreScanner.java | 476 ++-- .../hugegraph/store/cli/util/HgCliUtil.java | 482 ++-- .../hugegraph/store/cli/util/HgMetricX.java | 122 +- .../apache/hugegraph/store/HgKvIterator.java | 68 +- .../apache/hugegraph/store/HgScanQuery.java | 664 ++--- .../store/client/ShiftWorkIteratorProxy.java | 314 +-- .../store/client/TopWorkIteratorProxy.java | 274 +- .../store/client/grpc/GrpcKvIteratorImpl.java | 322 +-- .../client/grpc/GrpcStoreStreamClient.java | 436 ++-- .../hugegraph/store/client/grpc/GrpcUtil.java | 298 +-- .../client/grpc/KvBatchOneShotScanner.java | 244 +- .../store/client/grpc/KvBatchScanner5.java | 892 +++---- .../store/client/grpc/KvBatchUtil.java | 302 +-- .../client/grpc/KvCloseableIterator.java | 56 +- .../store/client/grpc/KvListIterator.java | 96 +- .../store/client/grpc/KvOneShotScanner.java | 424 ++-- .../store/client/grpc/SeekAbleIterator.java | 136 +- .../hugegraph/store/client/util/Base58.java | 354 +-- .../store/client/util/HgBufferProxy.java | 320 +-- .../hugegraph/store/client/util/HgUuid.java | 88 +- .../hugegraph/store/client/util/MetricX.java | 206 +- hg-store-client/src/main/resources/log4j2.xml | 204 +- .../hugegraph/store/HgSessionManagerTest.java | 2248 ++++++++--------- hg-store-client/src/test/resources/log4j2.xml | 204 +- .../store/node/grpc/BatchGrpcClosure.java | 422 ++-- .../store/node/grpc/BatchScanIterator.java | 320 +-- .../store/node/grpc/GRpcServerConfig.java | 92 +- .../store/node/grpc/GrpcClosure.java | 142 +- .../store/node/grpc/ScanBatchResponse3.java | 816 +++--- .../hugegraph/store/node/grpc/ScanQuery.java | 208 +- .../store/node/metrics/GRpcExMetrics.java | 188 +- .../store/node/metrics/JRaftMetrics.java | 620 ++--- .../store/node/metrics/ProcfsMetrics.java | 134 +- .../store/node/metrics/RocksDBMetrics.java | 828 +++--- .../node/metrics/RocksDBMetricsConst.java | 330 +-- .../store/node/metrics/StoreMetrics.java | 224 +- .../hugegraph/store/node/util/Base58.java | 354 +-- .../store/node/util/HgBufferProxy.java | 348 +-- .../store/node/util/HgRegexUtil.java | 172 +- .../store/node/metrics/JraftMetricsTest.java | 118 +- hg-store-rocksdb/pom.xml | 176 +- pom.xml | 512 ++-- settings.xml | 262 +- 46 files changed, 8011 insertions(+), 8011 deletions(-) diff --git a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/AppConfig.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/AppConfig.java index 4d728fe95..4b0e5fac1 100644 --- a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/AppConfig.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/AppConfig.java @@ -1,58 +1,58 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.cli; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -import lombok.Data; - -/** - * @author lynn.bond@hotmail.com on 2022/2/15 - */ -@Data -@Component -public class AppConfig { - - @Value("${pd.address}") - private String pdAddress; - - @Value("${net.kv.scanner.page.size}") - private int scannerPageSize; - - @Value("${scanner.graph}") - private String scannerGraph; - - @Value("${scanner.table}") - private String scannerTable; - - @Value("${scanner.max}") - private int scannerMax; - - @Value("${scanner.mod}") - private int scannerModNumber; - - @Value("${committer.graph}") - private String committerGraph; - - @Value("${committer.table}") - private String committerTable; - - @Value("${committer.amount}") - private int committerAmount; -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cli; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import lombok.Data; + +/** + * @author lynn.bond@hotmail.com on 2022/2/15 + */ +@Data +@Component +public class AppConfig { + + @Value("${pd.address}") + private String pdAddress; + + @Value("${net.kv.scanner.page.size}") + private int scannerPageSize; + + @Value("${scanner.graph}") + private String scannerGraph; + + @Value("${scanner.table}") + private String scannerTable; + + @Value("${scanner.max}") + private int scannerMax; + + @Value("${scanner.mod}") + private int scannerModNumber; + + @Value("${committer.graph}") + private String committerGraph; + + @Value("${committer.table}") + private String committerTable; + + @Value("${committer.amount}") + private int committerAmount; +} diff --git a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/StoreConsoleApplication.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/StoreConsoleApplication.java index 64b41ec35..08f8c027b 100644 --- a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/StoreConsoleApplication.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/StoreConsoleApplication.java @@ -1,118 +1,118 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.cli; - -import java.io.IOException; - -import org.apache.hugegraph.pd.client.PDConfig; -import org.apache.hugegraph.pd.common.PDException; -import org.apache.hugegraph.store.HgStoreClient; -import org.apache.hugegraph.store.cli.loader.HgThread2DB; -import org.apache.hugegraph.store.cli.scan.GrpcShardScanner; -import org.apache.hugegraph.store.cli.scan.HgStoreCommitter; -import org.apache.hugegraph.store.cli.scan.HgStoreScanner; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.CommandLineRunner; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; - -import lombok.extern.slf4j.Slf4j; - - -/** - * @author lynn.bond@hotmail.com on 2022/2/14 - */ -@SpringBootApplication -@Slf4j -public class StoreConsoleApplication implements CommandLineRunner { - - @Autowired - private AppConfig appConfig; - - public static void main(String[] args) { - log.info("Starting StoreConsoleApplication"); - SpringApplication.run(StoreConsoleApplication.class, args); - log.info("StoreConsoleApplication finished."); - } - - @Override - public void run(String... args) throws IOException, InterruptedException, PDException { - if (args.length <= 0) { - // HgThread2DB hB = new HgThread2DB("localhost:8686"); - // hB.startMultiprocessQuery("12", "10"); - System.out.println("参数类型 cmd[-load, -query, -scan]"); - } else { - switch (args[0]) { - case "-load": - HgThread2DB hgThread2DB = new HgThread2DB(args[1]); - if (!args[3].isEmpty()) { - hgThread2DB.setGraphName(args[3]); - } - try { - if (args[2].equals("order")) { - hgThread2DB.testOrder(args[4]); - } else { - hgThread2DB.startMultiprocessInsert(args[2]); - } - } catch (IOException e) { - e.printStackTrace(); - } - break; - case "-query": - HgThread2DB hgDB = new HgThread2DB(args[1]); - try { - hgDB.startMultiprocessQuery("12", args[2]); - } catch (IOException e) { - e.printStackTrace(); - } - break; - case "-scan": - if (args.length < 4) { - System.out.println("参数类型 -scan pd graphName tableName"); - } else { - doScan(args[1], args[2], args[3]); - } - break; - case "-shard": - GrpcShardScanner scanner = new GrpcShardScanner(); - scanner.getData(); - break; - case "-shard-single": - scanner = new GrpcShardScanner(); - scanner.getDataSingle(); - break; - default: - System.out.println("参数类型错误，未执行任何程序"); - } - } - } - - private void doCommit() { - HgStoreCommitter committer = HgStoreCommitter.of(appConfig.getCommitterGraph()); - committer.put(appConfig.getScannerTable(), appConfig.getCommitterAmount()); - } - - private void doScan(String pd, String graphName, String tableName) throws PDException { - HgStoreClient storeClient = HgStoreClient.create(PDConfig.of(pd) - .setEnableCache(true)); - - HgStoreScanner storeScanner = HgStoreScanner.of(storeClient, graphName); - storeScanner.scanTable2(tableName); - // storeScanner.scanHash(); - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cli; + +import java.io.IOException; + +import org.apache.hugegraph.pd.client.PDConfig; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.store.HgStoreClient; +import org.apache.hugegraph.store.cli.loader.HgThread2DB; +import org.apache.hugegraph.store.cli.scan.GrpcShardScanner; +import org.apache.hugegraph.store.cli.scan.HgStoreCommitter; +import org.apache.hugegraph.store.cli.scan.HgStoreScanner; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +import lombok.extern.slf4j.Slf4j; + + +/** + * @author lynn.bond@hotmail.com on 2022/2/14 + */ +@SpringBootApplication +@Slf4j +public class StoreConsoleApplication implements CommandLineRunner { + + @Autowired + private AppConfig appConfig; + + public static void main(String[] args) { + log.info("Starting StoreConsoleApplication"); + SpringApplication.run(StoreConsoleApplication.class, args); + log.info("StoreConsoleApplication finished."); + } + + @Override + public void run(String... args) throws IOException, InterruptedException, PDException { + if (args.length <= 0) { + // HgThread2DB hB = new HgThread2DB("localhost:8686"); + // hB.startMultiprocessQuery("12", "10"); + System.out.println("参数类型 cmd[-load, -query, -scan]"); + } else { + switch (args[0]) { + case "-load": + HgThread2DB hgThread2DB = new HgThread2DB(args[1]); + if (!args[3].isEmpty()) { + hgThread2DB.setGraphName(args[3]); + } + try { + if (args[2].equals("order")) { + hgThread2DB.testOrder(args[4]); + } else { + hgThread2DB.startMultiprocessInsert(args[2]); + } + } catch (IOException e) { + e.printStackTrace(); + } + break; + case "-query": + HgThread2DB hgDB = new HgThread2DB(args[1]); + try { + hgDB.startMultiprocessQuery("12", args[2]); + } catch (IOException e) { + e.printStackTrace(); + } + break; + case "-scan": + if (args.length < 4) { + System.out.println("参数类型 -scan pd graphName tableName"); + } else { + doScan(args[1], args[2], args[3]); + } + break; + case "-shard": + GrpcShardScanner scanner = new GrpcShardScanner(); + scanner.getData(); + break; + case "-shard-single": + scanner = new GrpcShardScanner(); + scanner.getDataSingle(); + break; + default: + System.out.println("参数类型错误，未执行任何程序"); + } + } + } + + private void doCommit() { + HgStoreCommitter committer = HgStoreCommitter.of(appConfig.getCommitterGraph()); + committer.put(appConfig.getScannerTable(), appConfig.getCommitterAmount()); + } + + private void doScan(String pd, String graphName, String tableName) throws PDException { + HgStoreClient storeClient = HgStoreClient.create(PDConfig.of(pd) + .setEnableCache(true)); + + HgStoreScanner storeScanner = HgStoreScanner.of(storeClient, graphName); + storeScanner.scanTable2(tableName); + // storeScanner.scanHash(); + } +} diff --git a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreCommitter.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreCommitter.java index 48ace5444..c665f015e 100644 --- a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreCommitter.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreCommitter.java @@ -1,87 +1,87 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.cli.scan; - -import org.apache.hugegraph.store.HgOwnerKey; -import org.apache.hugegraph.store.HgSessionManager; -import org.apache.hugegraph.store.HgStoreSession; -import org.apache.hugegraph.store.cli.util.HgCliUtil; -import org.apache.hugegraph.store.client.HgStoreNodeManager; - -/** - * @author lynn.bond@hotmail.com on 2022/2/28 - */ -public class HgStoreCommitter { - - protected final static HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); - - private final String graph; - - - public static HgStoreCommitter of(String graph) { - return new HgStoreCommitter(graph); - } - - private HgStoreCommitter(String graph) { - this.graph = graph; - } - - protected HgStoreSession getStoreSession() { - return HgSessionManager.getInstance().openSession(this.graph); - } - - protected HgStoreSession getStoreSession(String graphName) { - return HgSessionManager.getInstance().openSession(graphName); - } - - public void put(String tableName, int amount) { - //*************** Put Benchmark **************//* - String keyPrefix = "PUT-BENCHMARK"; - HgStoreSession session = getStoreSession(); - - int length = String.valueOf(amount).length(); - - session.beginTx(); - - long start = System.currentTimeMillis(); - for (int i = 0; i < amount; i++) { - HgOwnerKey key = HgCliUtil.toOwnerKey( - keyPrefix + "-" + HgCliUtil.padLeftZeros(String.valueOf(i), length)); - byte[] value = HgCliUtil.toBytes(keyPrefix + "-V-" + i); - - session.put(tableName, key, value); - - if ((i + 1) % 100_000 == 0) { - HgCliUtil.println("---------- " + (i + 1) + " --------"); - HgCliUtil.println( - "Preparing took: " + (System.currentTimeMillis() - start) + " ms."); - session.commit(); - HgCliUtil.println( - "Committing took: " + (System.currentTimeMillis() - start) + " ms."); - start = System.currentTimeMillis(); - session.beginTx(); - } - } - - if (session.isTx()) { - session.commit(); - } - - - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cli.scan; + +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgSessionManager; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.cli.util.HgCliUtil; +import org.apache.hugegraph.store.client.HgStoreNodeManager; + +/** + * @author lynn.bond@hotmail.com on 2022/2/28 + */ +public class HgStoreCommitter { + + protected final static HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); + + private final String graph; + + + public static HgStoreCommitter of(String graph) { + return new HgStoreCommitter(graph); + } + + private HgStoreCommitter(String graph) { + this.graph = graph; + } + + protected HgStoreSession getStoreSession() { + return HgSessionManager.getInstance().openSession(this.graph); + } + + protected HgStoreSession getStoreSession(String graphName) { + return HgSessionManager.getInstance().openSession(graphName); + } + + public void put(String tableName, int amount) { + //*************** Put Benchmark **************//* + String keyPrefix = "PUT-BENCHMARK"; + HgStoreSession session = getStoreSession(); + + int length = String.valueOf(amount).length(); + + session.beginTx(); + + long start = System.currentTimeMillis(); + for (int i = 0; i < amount; i++) { + HgOwnerKey key = HgCliUtil.toOwnerKey( + keyPrefix + "-" + HgCliUtil.padLeftZeros(String.valueOf(i), length)); + byte[] value = HgCliUtil.toBytes(keyPrefix + "-V-" + i); + + session.put(tableName, key, value); + + if ((i + 1) % 100_000 == 0) { + HgCliUtil.println("---------- " + (i + 1) + " --------"); + HgCliUtil.println( + "Preparing took: " + (System.currentTimeMillis() - start) + " ms."); + session.commit(); + HgCliUtil.println( + "Committing took: " + (System.currentTimeMillis() - start) + " ms."); + start = System.currentTimeMillis(); + session.beginTx(); + } + } + + if (session.isTx()) { + session.commit(); + } + + + } +} diff --git a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreScanner.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreScanner.java index 58f732be2..62f494db1 100644 --- a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreScanner.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/scan/HgStoreScanner.java @@ -1,238 +1,238 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.cli.scan; - -import java.util.Arrays; -import java.util.List; - -import org.apache.hugegraph.pd.client.PDClient; -import org.apache.hugegraph.pd.common.PDException; -import org.apache.hugegraph.pd.grpc.Metapb; -import org.apache.hugegraph.store.HgKvEntry; -import org.apache.hugegraph.store.HgKvIterator; -import org.apache.hugegraph.store.HgKvStore; -import org.apache.hugegraph.store.HgScanQuery; -import org.apache.hugegraph.store.HgSessionManager; -import org.apache.hugegraph.store.HgStoreClient; -import org.apache.hugegraph.store.HgStoreSession; -import org.apache.hugegraph.store.cli.util.HgCliUtil; -import org.apache.hugegraph.store.cli.util.HgMetricX; -import org.apache.hugegraph.store.client.grpc.KvCloseableIterator; -import org.apache.hugegraph.store.client.util.HgStoreClientConfig; -import org.apache.hugegraph.store.client.util.MetricX; - -import lombok.extern.slf4j.Slf4j; - -/** - * @author lynn.bond@hotmail.com on 2022/2/14 - */ -@Slf4j -public class HgStoreScanner { - - private final HgStoreClient storeClient; - private final String graphName; - private long modNumber = 1_000_000; - private int max = 10_000_000; - - public long getModNumber() { - return modNumber; - } - - public void setModNumber(int modNumber) { - if (modNumber <= 0) { - return; - } - this.modNumber = modNumber; - } - - public int getMax() { - return max; - } - - public void setMax(int max) { - if (modNumber <= 0) { - return; - } - this.max = max; - } - - public static HgStoreScanner of(HgStoreClient storeClient, String graph) { - return new HgStoreScanner(storeClient, graph); - } - - private HgStoreScanner(HgStoreClient storeClient, String graph) { - this.storeClient = storeClient; - this.graphName = graph; - } - - protected HgStoreSession getStoreSession() { - return HgSessionManager.getInstance().openSession(this.graphName); - } - - protected HgStoreSession getStoreSession(String graphName) { - return HgSessionManager.getInstance().openSession(graphName); - } - - public void scanTable(String tableName) { - log.info("Starting scan table [{}] of graph [{}] ...", tableName, graphName); - HgMetricX hgMetricX = HgMetricX.ofStart(); - HgStoreSession session = getStoreSession(); - int count = 0; - KvCloseableIterator> iterator = - session.scanBatch2(HgScanQuery.tableOf(tableName)); - - long start = System.currentTimeMillis(); - while (iterator.hasNext()) { - HgKvIterator iterator2 = iterator.next(); - while (iterator2.hasNext()) { - - count++; - iterator2.next(); - if (count % (modNumber) == 0) { - log.info("Scanning keys: " + count + " time is " + modNumber * 1000 - / - (System.currentTimeMillis() - - start)); - start = System.currentTimeMillis(); - } - if (count == max) { - break; - } - - } - } - iterator.close(); - - hgMetricX.end(); - log.info("*************************************************"); - log.info("************* Scanning Completed **************"); - log.info("Graph: {}", graphName); - log.info("Table: {}", tableName); - log.info("Keys: {}", count); - log.info("Max: {}", max); - log.info("Waiting: {} seconds.", MetricX.getIteratorWait() / 1000); - log.info("Total: {} seconds.", hgMetricX.past() / 1000); - log.info("Iterator: [{}]", iterator.getClass().getSimpleName()); - log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize()); - log.info("*************************************************"); - } - - public void scanHash() { - - String tableName = "g+i"; - HgMetricX hgMetricX = HgMetricX.ofStart(); - String graphName = "/DEFAULT/graphs/hugegraph1/"; - HgStoreSession session = getStoreSession(graphName); - int count = 0; - String query = - "{\"conditions\":[{\"cls\":\"S\",\"el\":{\"key\":\"ID\",\"relation\":\"SCAN\"," + - "\"value\"" + - ":{\"start\":\"61180\",\"end\":\"63365\",\"length\":0}}}]," + - "\"optimizedType\":\"NONE\",\"ids\":[]," + - "\"mustSortByInput\":true,\"resultType\":\"EDGE\",\"offset\":0," + - "\"actualOffset\":0,\"actualStoreOffset\":" + - "0,\"limit\":9223372036854775807,\"capacity\":-1,\"showHidden\":false," + - "\"showDeleting\":false," + - "\"showExpired\":false,\"olap\":false,\"withProperties\":false,\"olapPks\":[]}"; - //HgKvIterator iterator = session.scanIterator(tableName,0,715827883, - // HgKvStore.SCAN_ANY,null); - - //HgKvIterator iterator = session.scanIterator(tableName,61180,63365, 348, null); - //HgKvIterator iterator = session.scanIterator(tableName,0,65535, 348, null); - HgKvIterator iterator = session.scanIterator(tableName); - while (iterator.hasNext()) { - - count++; - //iterator.next(); - // if (count % (modNumber) == 0) { - // log.info("Scanning keys: " + count); - HgCliUtil.println(Arrays.toString(iterator.next().key())); - // } - if (count == max) { - break; - } - - } - - hgMetricX.end(); - log.info("*************************************************"); - log.info("************* Scanning Completed **************"); - log.info("Graph: {}", this.graphName); - log.info("Table: {}", tableName); - log.info("Keys: {}", count); - log.info("Max: {}", max); - log.info("Waiting: {} seconds.", MetricX.getIteratorWait() / 1000); - log.info("Total: {} seconds.", hgMetricX.past() / 1000); - log.info("Iterator: [{}]", iterator.getClass().getSimpleName()); - log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize()); - log.info("*************************************************"); - } - - public static final byte[] EMPTY_BYTES = new byte[0]; - - public void scanTable2(String tableName) throws PDException { - // java -jar hg-store-cli-3.6.0-SNAPSHOT.jar -scan 10.45.30.212:8989 "DEFAULT/case_112/g" - // g+ie - PDClient pdClient = storeClient.getPdClient(); - List partitions = pdClient.getPartitions(0, graphName); - HgStoreSession session = storeClient.openSession(graphName); - int count = 0; - byte[] position = null; - HgMetricX hgMetricX = HgMetricX.ofStart(); - for (Metapb.Partition partition : partitions) { - while (true) { - try (HgKvIterator iterator = session.scanIterator(tableName, - (int) (partition.getStartKey()), - (int) (partition.getEndKey()), - HgKvStore.SCAN_HASHCODE, - EMPTY_BYTES)) { - if (position != null) { - iterator.seek(position); - } - while (iterator.hasNext()) { - iterator.next(); - count++; - if (count % 3000 == 0) { - if (iterator.hasNext()) { - iterator.next(); - position = iterator.position(); - System.out.println("count is " + count); - } else { - position = null; - } - break; - } - } - if (!iterator.hasNext()) { - position = null; - break; - } - } - } - } - hgMetricX.end(); - log.info("*************************************************"); - log.info("************* Scanning Completed **************"); - log.info("Graph: {}", graphName); - log.info("Table: {}", tableName); - log.info("Keys: {}", count); - log.info("Total: {} seconds.", hgMetricX.past() / 1000); - log.info("*************************************************"); - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cli.scan; + +import java.util.Arrays; +import java.util.List; + +import org.apache.hugegraph.pd.client.PDClient; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvStore; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.HgSessionManager; +import org.apache.hugegraph.store.HgStoreClient; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.cli.util.HgCliUtil; +import org.apache.hugegraph.store.cli.util.HgMetricX; +import org.apache.hugegraph.store.client.grpc.KvCloseableIterator; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.client.util.MetricX; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author lynn.bond@hotmail.com on 2022/2/14 + */ +@Slf4j +public class HgStoreScanner { + + private final HgStoreClient storeClient; + private final String graphName; + private long modNumber = 1_000_000; + private int max = 10_000_000; + + public long getModNumber() { + return modNumber; + } + + public void setModNumber(int modNumber) { + if (modNumber <= 0) { + return; + } + this.modNumber = modNumber; + } + + public int getMax() { + return max; + } + + public void setMax(int max) { + if (modNumber <= 0) { + return; + } + this.max = max; + } + + public static HgStoreScanner of(HgStoreClient storeClient, String graph) { + return new HgStoreScanner(storeClient, graph); + } + + private HgStoreScanner(HgStoreClient storeClient, String graph) { + this.storeClient = storeClient; + this.graphName = graph; + } + + protected HgStoreSession getStoreSession() { + return HgSessionManager.getInstance().openSession(this.graphName); + } + + protected HgStoreSession getStoreSession(String graphName) { + return HgSessionManager.getInstance().openSession(graphName); + } + + public void scanTable(String tableName) { + log.info("Starting scan table [{}] of graph [{}] ...", tableName, graphName); + HgMetricX hgMetricX = HgMetricX.ofStart(); + HgStoreSession session = getStoreSession(); + int count = 0; + KvCloseableIterator> iterator = + session.scanBatch2(HgScanQuery.tableOf(tableName)); + + long start = System.currentTimeMillis(); + while (iterator.hasNext()) { + HgKvIterator iterator2 = iterator.next(); + while (iterator2.hasNext()) { + + count++; + iterator2.next(); + if (count % (modNumber) == 0) { + log.info("Scanning keys: " + count + " time is " + modNumber * 1000 + / + (System.currentTimeMillis() - + start)); + start = System.currentTimeMillis(); + } + if (count == max) { + break; + } + + } + } + iterator.close(); + + hgMetricX.end(); + log.info("*************************************************"); + log.info("************* Scanning Completed **************"); + log.info("Graph: {}", graphName); + log.info("Table: {}", tableName); + log.info("Keys: {}", count); + log.info("Max: {}", max); + log.info("Waiting: {} seconds.", MetricX.getIteratorWait() / 1000); + log.info("Total: {} seconds.", hgMetricX.past() / 1000); + log.info("Iterator: [{}]", iterator.getClass().getSimpleName()); + log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize()); + log.info("*************************************************"); + } + + public void scanHash() { + + String tableName = "g+i"; + HgMetricX hgMetricX = HgMetricX.ofStart(); + String graphName = "/DEFAULT/graphs/hugegraph1/"; + HgStoreSession session = getStoreSession(graphName); + int count = 0; + String query = + "{\"conditions\":[{\"cls\":\"S\",\"el\":{\"key\":\"ID\",\"relation\":\"SCAN\"," + + "\"value\"" + + ":{\"start\":\"61180\",\"end\":\"63365\",\"length\":0}}}]," + + "\"optimizedType\":\"NONE\",\"ids\":[]," + + "\"mustSortByInput\":true,\"resultType\":\"EDGE\",\"offset\":0," + + "\"actualOffset\":0,\"actualStoreOffset\":" + + "0,\"limit\":9223372036854775807,\"capacity\":-1,\"showHidden\":false," + + "\"showDeleting\":false," + + "\"showExpired\":false,\"olap\":false,\"withProperties\":false,\"olapPks\":[]}"; + //HgKvIterator iterator = session.scanIterator(tableName,0,715827883, + // HgKvStore.SCAN_ANY,null); + + //HgKvIterator iterator = session.scanIterator(tableName,61180,63365, 348, null); + //HgKvIterator iterator = session.scanIterator(tableName,0,65535, 348, null); + HgKvIterator iterator = session.scanIterator(tableName); + while (iterator.hasNext()) { + + count++; + //iterator.next(); + // if (count % (modNumber) == 0) { + // log.info("Scanning keys: " + count); + HgCliUtil.println(Arrays.toString(iterator.next().key())); + // } + if (count == max) { + break; + } + + } + + hgMetricX.end(); + log.info("*************************************************"); + log.info("************* Scanning Completed **************"); + log.info("Graph: {}", this.graphName); + log.info("Table: {}", tableName); + log.info("Keys: {}", count); + log.info("Max: {}", max); + log.info("Waiting: {} seconds.", MetricX.getIteratorWait() / 1000); + log.info("Total: {} seconds.", hgMetricX.past() / 1000); + log.info("Iterator: [{}]", iterator.getClass().getSimpleName()); + log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize()); + log.info("*************************************************"); + } + + public static final byte[] EMPTY_BYTES = new byte[0]; + + public void scanTable2(String tableName) throws PDException { + // java -jar hg-store-cli-3.6.0-SNAPSHOT.jar -scan 10.45.30.212:8989 "DEFAULT/case_112/g" + // g+ie + PDClient pdClient = storeClient.getPdClient(); + List partitions = pdClient.getPartitions(0, graphName); + HgStoreSession session = storeClient.openSession(graphName); + int count = 0; + byte[] position = null; + HgMetricX hgMetricX = HgMetricX.ofStart(); + for (Metapb.Partition partition : partitions) { + while (true) { + try (HgKvIterator iterator = session.scanIterator(tableName, + (int) (partition.getStartKey()), + (int) (partition.getEndKey()), + HgKvStore.SCAN_HASHCODE, + EMPTY_BYTES)) { + if (position != null) { + iterator.seek(position); + } + while (iterator.hasNext()) { + iterator.next(); + count++; + if (count % 3000 == 0) { + if (iterator.hasNext()) { + iterator.next(); + position = iterator.position(); + System.out.println("count is " + count); + } else { + position = null; + } + break; + } + } + if (!iterator.hasNext()) { + position = null; + break; + } + } + } + } + hgMetricX.end(); + log.info("*************************************************"); + log.info("************* Scanning Completed **************"); + log.info("Graph: {}", graphName); + log.info("Table: {}", tableName); + log.info("Keys: {}", count); + log.info("Total: {} seconds.", hgMetricX.past() / 1000); + log.info("*************************************************"); + } + +} diff --git a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgCliUtil.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgCliUtil.java index 188c524bc..2b848695e 100644 --- a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgCliUtil.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgCliUtil.java @@ -1,241 +1,241 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.cli.util; - -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -import org.apache.hugegraph.store.HgKvEntry; -import org.apache.hugegraph.store.HgOwnerKey; -import org.apache.hugegraph.store.HgStoreSession; -import org.apache.hugegraph.store.client.util.HgStoreClientConst; - - -/** - * @author lynn.bond@hotmail.com on 2022/2/14 - */ -public class HgCliUtil { - - public final static String TABLE_NAME = "cli-table"; - - - public static Map batchPut(HgStoreSession session, String keyPrefix) { - return batchPut(session, keyPrefix, 100); - } - - public static Map batchPut(HgStoreSession session, String keyPrefix, - int loop) { - return batchPut(session, TABLE_NAME, keyPrefix, loop); - } - - public static Map batchPut(HgStoreSession session, String tableName - , String keyPrefix, int loop) { - return batchPut(session, tableName, keyPrefix, loop, 1, key -> toOwnerKey(key)); - } - - public static Map batchPut(HgStoreSession session, String tableName - , String keyPrefix, int loop, int start) { - return batchPut(session, tableName, keyPrefix, loop, start, key -> toOwnerKey(key)); - } - - public static Map batchPut(HgStoreSession session, String tableName - , String keyPrefix, int loop, Function f) { - return batchPut(session, tableName, keyPrefix, loop, 1, f); - } - - public static Map batchPut(HgStoreSession session, String tableName - , String keyPrefix, int loop, int start, Function f) { - - Map res = new LinkedHashMap<>(); - - int length = String.valueOf(loop).length(); - - session.beginTx(); - for (int i = start; i <= loop; i++) { - - HgOwnerKey key = f.apply(keyPrefix + "-" + padLeftZeros(String.valueOf(i), length)); - - byte[] value = toBytes(keyPrefix + "-V-" + i); - res.put(key, value); - session.put(tableName, key, value); - - if ((i + 1) % 10000 == 0) { - println("commit: " + (i + 1)); - session.commit(); - session.beginTx(); - } - } - if (session.isTx()) { - session.commit(); - } - - return res; - } - - public static void printNum(List list, String title) { - if (list == null) return; - - println(title + " size: " + list.size()); - } - - public static void println(Iterator iterator) { - if (iterator == null) return; - while (iterator.hasNext()) { - println(iterator.next()); - } - - } - - public static void printOwner(List list) { - if (list == null) return; - - for (HgOwnerKey entry : list) { - println(entry); - } - } - - public static void println(List list) { - if (list == null) return; - - for (HgKvEntry entry : list) { - println(entry); - } - } - - public static void println(List list, int mod) { - if (list == null) return; - - for (int i = 0; i < list.size(); i++) { - if (i % mod == 0) { - println(list.get(i)); - } - } - } - - public static void println(HgKvEntry kv) { - if (kv == null) { - System.out.println("null"); - return; - } - println("[ " + toStr(kv.key()) + " : " + toStr(kv.value()) + " ]"); - } - - public static void println(HgOwnerKey key) { - if (key == null) { - System.out.println("null"); - return; - } - println("[ " + toInt(key.getOwner()) + " : " + toStr(key.getKey()) + " ]"); - } - - public static void println(String str) { - System.out.println(str); - } - - public static HgOwnerKey toOwnerKey(String key) { - return new HgOwnerKey(getOwner(key), toBytes(key)); - } - - public static HgOwnerKey toOwnerKey(byte[] key) { - return new HgOwnerKey(getOwner(key), key); - } - - private static byte[] getOwner(String key) { - return getOwner(toBytes(key)); - } - - private static byte[] getOwner(byte[] key) { - return toBytes(Arrays.hashCode(key)); - } - - public static HgOwnerKey toAllPartitionKey(String key) { - return HgOwnerKey.of(HgStoreClientConst.ALL_PARTITION_OWNER, toBytes(key)); - } - - public static HgOwnerKey toOwnerKey(String owner, String key) { - return HgOwnerKey.of(toBytes(owner), toBytes(key)); - } - - public static String toStr(byte[] b) { - if (b == null) return ""; - if (b.length == 0) return ""; - return new String(b, StandardCharsets.UTF_8); - } - - public static byte[] toBytes(String str) { - if (str == null) return null; - return str.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] toBytes(long l) { - ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); - buffer.putLong(l); - return buffer.array(); - } - - private static byte[] toBytes(final int i) { - ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); - buffer.putInt(i); - return buffer.array(); - } - - public static long toLong(byte[] bytes) { - ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); - buffer.put(bytes); - buffer.flip();//need flip - return buffer.getLong(); - } - - public static long toInt(byte[] bytes) { - ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); - buffer.put(bytes); - buffer.flip();//need flip - return buffer.getInt(); - } - - public static String padLeftZeros(String str, int n) { - return String.format("%1$" + n + "s", str).replace(' ', '0'); - } - - public static String toSuffix(int num, int length) { - return "-" + padLeftZeros(String.valueOf(num), length); - } - - public static int amountOf(List list) { - if (list == null) { - return 0; - } - return list.size(); - } - - public static int amountOf(Iterator iterator) { - if (iterator == null) return 0; - int count = 0; - while (iterator.hasNext()) { - iterator.next(); - count++; - } - return count; - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cli.util; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; + + +/** + * @author lynn.bond@hotmail.com on 2022/2/14 + */ +public class HgCliUtil { + + public final static String TABLE_NAME = "cli-table"; + + + public static Map batchPut(HgStoreSession session, String keyPrefix) { + return batchPut(session, keyPrefix, 100); + } + + public static Map batchPut(HgStoreSession session, String keyPrefix, + int loop) { + return batchPut(session, TABLE_NAME, keyPrefix, loop); + } + + public static Map batchPut(HgStoreSession session, String tableName + , String keyPrefix, int loop) { + return batchPut(session, tableName, keyPrefix, loop, 1, key -> toOwnerKey(key)); + } + + public static Map batchPut(HgStoreSession session, String tableName + , String keyPrefix, int loop, int start) { + return batchPut(session, tableName, keyPrefix, loop, start, key -> toOwnerKey(key)); + } + + public static Map batchPut(HgStoreSession session, String tableName + , String keyPrefix, int loop, Function f) { + return batchPut(session, tableName, keyPrefix, loop, 1, f); + } + + public static Map batchPut(HgStoreSession session, String tableName + , String keyPrefix, int loop, int start, Function f) { + + Map res = new LinkedHashMap<>(); + + int length = String.valueOf(loop).length(); + + session.beginTx(); + for (int i = start; i <= loop; i++) { + + HgOwnerKey key = f.apply(keyPrefix + "-" + padLeftZeros(String.valueOf(i), length)); + + byte[] value = toBytes(keyPrefix + "-V-" + i); + res.put(key, value); + session.put(tableName, key, value); + + if ((i + 1) % 10000 == 0) { + println("commit: " + (i + 1)); + session.commit(); + session.beginTx(); + } + } + if (session.isTx()) { + session.commit(); + } + + return res; + } + + public static void printNum(List list, String title) { + if (list == null) return; + + println(title + " size: " + list.size()); + } + + public static void println(Iterator iterator) { + if (iterator == null) return; + while (iterator.hasNext()) { + println(iterator.next()); + } + + } + + public static void printOwner(List list) { + if (list == null) return; + + for (HgOwnerKey entry : list) { + println(entry); + } + } + + public static void println(List list) { + if (list == null) return; + + for (HgKvEntry entry : list) { + println(entry); + } + } + + public static void println(List list, int mod) { + if (list == null) return; + + for (int i = 0; i < list.size(); i++) { + if (i % mod == 0) { + println(list.get(i)); + } + } + } + + public static void println(HgKvEntry kv) { + if (kv == null) { + System.out.println("null"); + return; + } + println("[ " + toStr(kv.key()) + " : " + toStr(kv.value()) + " ]"); + } + + public static void println(HgOwnerKey key) { + if (key == null) { + System.out.println("null"); + return; + } + println("[ " + toInt(key.getOwner()) + " : " + toStr(key.getKey()) + " ]"); + } + + public static void println(String str) { + System.out.println(str); + } + + public static HgOwnerKey toOwnerKey(String key) { + return new HgOwnerKey(getOwner(key), toBytes(key)); + } + + public static HgOwnerKey toOwnerKey(byte[] key) { + return new HgOwnerKey(getOwner(key), key); + } + + private static byte[] getOwner(String key) { + return getOwner(toBytes(key)); + } + + private static byte[] getOwner(byte[] key) { + return toBytes(Arrays.hashCode(key)); + } + + public static HgOwnerKey toAllPartitionKey(String key) { + return HgOwnerKey.of(HgStoreClientConst.ALL_PARTITION_OWNER, toBytes(key)); + } + + public static HgOwnerKey toOwnerKey(String owner, String key) { + return HgOwnerKey.of(toBytes(owner), toBytes(key)); + } + + public static String toStr(byte[] b) { + if (b == null) return ""; + if (b.length == 0) return ""; + return new String(b, StandardCharsets.UTF_8); + } + + public static byte[] toBytes(String str) { + if (str == null) return null; + return str.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] toBytes(long l) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(l); + return buffer.array(); + } + + private static byte[] toBytes(final int i) { + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); + buffer.putInt(i); + return buffer.array(); + } + + public static long toLong(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.put(bytes); + buffer.flip();//need flip + return buffer.getLong(); + } + + public static long toInt(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); + buffer.put(bytes); + buffer.flip();//need flip + return buffer.getInt(); + } + + public static String padLeftZeros(String str, int n) { + return String.format("%1$" + n + "s", str).replace(' ', '0'); + } + + public static String toSuffix(int num, int length) { + return "-" + padLeftZeros(String.valueOf(num), length); + } + + public static int amountOf(List list) { + if (list == null) { + return 0; + } + return list.size(); + } + + public static int amountOf(Iterator iterator) { + if (iterator == null) return 0; + int count = 0; + while (iterator.hasNext()) { + iterator.next(); + count++; + } + return count; + } +} diff --git a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgMetricX.java b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgMetricX.java index c4b989a55..fb510c093 100644 --- a/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgMetricX.java +++ b/hg-store-cli/src/main/java/org/apache/hugegraph/store/cli/util/HgMetricX.java @@ -1,61 +1,61 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.cli.util; - -/** - * @author lynn.bond@hotmail.com on 2022/1/29 - */ -public class HgMetricX { - private long start; - private long end; - - private long waitStart = System.currentTimeMillis(); - private long waitTotal; - - public static HgMetricX ofStart() { - return new HgMetricX(System.currentTimeMillis()); - } - - private HgMetricX(long start) { - this.start = start; - } - - public long start() { - return this.start = System.currentTimeMillis(); - } - - public long end() { - return this.end = System.currentTimeMillis(); - } - - public long past() { - return this.end - this.start; - } - - public long getWaitTotal() { - return this.waitTotal; - } - - public void startWait() { - this.waitStart = System.currentTimeMillis(); - } - - public void appendWait() { - this.waitTotal += System.currentTimeMillis() - waitStart; - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.cli.util; + +/** + * @author lynn.bond@hotmail.com on 2022/1/29 + */ +public class HgMetricX { + private long start; + private long end; + + private long waitStart = System.currentTimeMillis(); + private long waitTotal; + + public static HgMetricX ofStart() { + return new HgMetricX(System.currentTimeMillis()); + } + + private HgMetricX(long start) { + this.start = start; + } + + public long start() { + return this.start = System.currentTimeMillis(); + } + + public long end() { + return this.end = System.currentTimeMillis(); + } + + public long past() { + return this.end - this.start; + } + + public long getWaitTotal() { + return this.waitTotal; + } + + public void startWait() { + this.waitStart = System.currentTimeMillis(); + } + + public void appendWait() { + this.waitTotal += System.currentTimeMillis() - waitStart; + } +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/HgKvIterator.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgKvIterator.java index 12524ff25..995aef79b 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/HgKvIterator.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgKvIterator.java @@ -1,34 +1,34 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store; - -import java.io.Closeable; -import java.util.Iterator; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/21 - */ -public interface HgKvIterator extends Iterator, HgSeekAble, Closeable { - - byte[] key(); - - byte[] value(); - - void close(); - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import java.io.Closeable; +import java.util.Iterator; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/21 + */ +public interface HgKvIterator extends Iterator, HgSeekAble, Closeable { + + byte[] key(); + + byte[] value(); + + void close(); + +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/HgScanQuery.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgScanQuery.java index f497739b7..fc631b877 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/HgScanQuery.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/HgScanQuery.java @@ -1,332 +1,332 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store; - -import java.util.Collections; -import java.util.Iterator; -import java.util.List; - -import org.apache.hugegraph.store.client.util.HgAssert; -import org.apache.hugegraph.store.grpc.common.ScanOrderType; - -/** - * @author lynn.bond@hotmail.com on 2022/3/4 - * @version 0.5.0 - */ -public interface HgScanQuery { - String getTable(); - - HgScanQuery.ScanMethod getScanMethod(); - - List getPrefixList(); - - Iterator getPrefixItr(); - - List getStartList(); - - List getEndList(); - - long getLimit(); - - long getPerKeyLimit(); - - long getPerKeyMax(); - - long getSkipDegree(); - - int getScanType(); - - ScanOrderType getOrderType(); - - boolean isOnlyKey(); - - byte[] getQuery(); - - enum ScanMethod { - ALL, - PREFIX, - RANGE - } - - enum SortType { - UNSORTED, - SORT_BY_EDGE, - SORT_BY_VERTEX - } - - - ScanBuilder builder(); - - static HgScanQuery tableOf(String table) { - return ScanBuilder.tableOf(table).build(); - } - - static HgScanQuery rangeOf(String table, List startList, List endList) { - return ScanBuilder.rangeOf(table, startList, endList).build(); - } - - static HgScanQuery prefixOf(String table, List prefixList) { - return ScanBuilder.prefixOf(table, prefixList).build(); - } - - static HgScanQuery prefixOf(String table, List prefixList, - ScanOrderType orderType) { - return ScanBuilder.prefixOf(table, prefixList).setOrderType(orderType).build(); - } - - static HgScanQuery prefixIteratorOf(String table, Iterator prefixItr) { - return ScanBuilder.prefixIteratorOf(table, prefixItr).build(); - } - - static HgScanQuery prefixIteratorOf(String table, Iterator prefixItr, - ScanOrderType orderType) { - return ScanBuilder.prefixIteratorOf(table, prefixItr).setOrderType(orderType).build(); - } - - class ScanBuilder { - private final String table; - private final HgScanQuery.ScanMethod sanMethod; - private long limit = Integer.MAX_VALUE; - private long perKeyLimit = Integer.MAX_VALUE; - private long perKeyMax = Integer.MAX_VALUE; - private int scanType; - private ScanOrderType orderType; - - private long skipDegree; - - private boolean onlyKey; - private byte[] query; - private List prefixList; - private List startList; - private List endList; - private Iterator prefixItr; - - public static ScanBuilder rangeOf(String table, List startList, - List endList) { - HgAssert.isArgumentValid(table, "table"); - HgAssert.isArgumentValid(startList, "startList"); - HgAssert.isArgumentValid(endList, "endList"); - HgAssert.isTrue(startList.size() == endList.size() - , "The size of startList not equals endList's."); - - ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.RANGE, table); - res.startList = startList; - res.endList = endList; - res.scanType = HgKvStore.SCAN_GTE_BEGIN | HgKvStore.SCAN_LTE_END; - return res; - } - - public static ScanBuilder prefixOf(String table, List prefixList) { - HgAssert.isArgumentValid(table, "table"); - HgAssert.isArgumentValid(prefixList, "prefixList"); - - ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.PREFIX, table); - res.prefixList = prefixList; - return res; - - } - - public static ScanBuilder prefixIteratorOf(String table, Iterator prefixItr) { - HgAssert.isArgumentValid(table, "table"); - - ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.PREFIX, table); - res.prefixItr = prefixItr; - return res; - - } - - public static ScanBuilder tableOf(String table) { - HgAssert.isArgumentValid(table, "table"); - - return new ScanBuilder(HgScanQuery.ScanMethod.ALL, table); - } - - ScanBuilder(HgScanQuery.ScanMethod sanMethod, String table) { - this.table = table; - this.sanMethod = sanMethod; - this.orderType = ScanOrderType.ORDER_NONE; - } - - public ScanBuilder setLimit(long limit) { - this.limit = limit; - return this; - } - - public ScanBuilder setPerKeyLimit(long limit) { - this.perKeyLimit = limit; - return this; - } - - public ScanBuilder setPerKeyMax(long max) { - this.perKeyMax = max; - return this; - } - - public ScanBuilder setScanType(int scanType) { - this.scanType = scanType; - return this; - } - - public ScanBuilder setOrderType(ScanOrderType orderType) { - this.orderType = orderType; - return this; - } - - public ScanBuilder setQuery(byte[] query) { - this.query = query; - return this; - } - - public ScanBuilder setSkipDegree(long skipDegree) { - this.skipDegree = skipDegree; - return this; - } - - public ScanBuilder setOnlyKey(boolean onlyKey) { - this.onlyKey = onlyKey; - return this; - } - - - public HgScanQuery build() { - return this.new BatchScanQuery(); - } - - private class BatchScanQuery implements HgScanQuery { - - @Override - public String getTable() { - return table; - } - - @Override - public HgScanQuery.ScanMethod getScanMethod() { - return sanMethod; - } - - @Override - public List getPrefixList() { - if (prefixList == null) { - return Collections.EMPTY_LIST; - } else { - return Collections.unmodifiableList(prefixList); - } - } - - @Override - public Iterator getPrefixItr() { - return prefixItr; - } - - @Override - public List getStartList() { - if (startList == null) { - return Collections.EMPTY_LIST; - } else { - return Collections.unmodifiableList(startList); - } - } - - @Override - public List getEndList() { - if (endList == null) { - return Collections.EMPTY_LIST; - } else { - return Collections.unmodifiableList(endList); - } - } - - @Override - public long getLimit() { - return limit; - } - - @Override - public long getPerKeyLimit() { - return perKeyLimit; - } - - @Override - public long getPerKeyMax() { - return perKeyMax; - } - - @Override - public long getSkipDegree() { - return skipDegree; - } - - @Override - public int getScanType() { - return scanType; - } - - @Override - public ScanOrderType getOrderType() { - return orderType; - } - - @Override - public boolean isOnlyKey() { - return onlyKey; - } - - @Override - public byte[] getQuery() { - return query; - } - - @Override - public ScanBuilder builder() { - return ScanBuilder.this; - } - - @Override - public String toString() { - final StringBuffer sb = new StringBuffer("HgScanQuery{"); - sb.append("table='").append(getTable()).append('\''); - sb.append(", scanMethod=").append(getScanMethod()); - sb.append(", prefixList=").append(getPrefixList()); - sb.append(", startList=").append(getStartList()); - sb.append(", endList=").append(getEndList()); - sb.append(", limit=").append(getLimit()); - sb.append(", perKeyLimit=").append(getPerKeyLimit()); - sb.append(", perKeyMax=").append(getPerKeyMax()); - sb.append(", skipDegree=").append(getSkipDegree()); - sb.append(", scanType=").append(getScanType()); - sb.append(", orderType=").append(getOrderType()); - sb.append(", onlyKey=").append(isOnlyKey()); - sb.append(", query="); - if (query == null) { - sb.append("null"); - } else { - sb.append('['); - for (int i = 0; i < query.length; ++i) { - sb.append(i == 0 ? "" : ", ").append(query[i]); - } - sb.append(']'); - } - sb.append('}'); - return sb.toString(); - } - } - - - } -} - +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import org.apache.hugegraph.store.client.util.HgAssert; +import org.apache.hugegraph.store.grpc.common.ScanOrderType; + +/** + * @author lynn.bond@hotmail.com on 2022/3/4 + * @version 0.5.0 + */ +public interface HgScanQuery { + String getTable(); + + HgScanQuery.ScanMethod getScanMethod(); + + List getPrefixList(); + + Iterator getPrefixItr(); + + List getStartList(); + + List getEndList(); + + long getLimit(); + + long getPerKeyLimit(); + + long getPerKeyMax(); + + long getSkipDegree(); + + int getScanType(); + + ScanOrderType getOrderType(); + + boolean isOnlyKey(); + + byte[] getQuery(); + + enum ScanMethod { + ALL, + PREFIX, + RANGE + } + + enum SortType { + UNSORTED, + SORT_BY_EDGE, + SORT_BY_VERTEX + } + + + ScanBuilder builder(); + + static HgScanQuery tableOf(String table) { + return ScanBuilder.tableOf(table).build(); + } + + static HgScanQuery rangeOf(String table, List startList, List endList) { + return ScanBuilder.rangeOf(table, startList, endList).build(); + } + + static HgScanQuery prefixOf(String table, List prefixList) { + return ScanBuilder.prefixOf(table, prefixList).build(); + } + + static HgScanQuery prefixOf(String table, List prefixList, + ScanOrderType orderType) { + return ScanBuilder.prefixOf(table, prefixList).setOrderType(orderType).build(); + } + + static HgScanQuery prefixIteratorOf(String table, Iterator prefixItr) { + return ScanBuilder.prefixIteratorOf(table, prefixItr).build(); + } + + static HgScanQuery prefixIteratorOf(String table, Iterator prefixItr, + ScanOrderType orderType) { + return ScanBuilder.prefixIteratorOf(table, prefixItr).setOrderType(orderType).build(); + } + + class ScanBuilder { + private final String table; + private final HgScanQuery.ScanMethod sanMethod; + private long limit = Integer.MAX_VALUE; + private long perKeyLimit = Integer.MAX_VALUE; + private long perKeyMax = Integer.MAX_VALUE; + private int scanType; + private ScanOrderType orderType; + + private long skipDegree; + + private boolean onlyKey; + private byte[] query; + private List prefixList; + private List startList; + private List endList; + private Iterator prefixItr; + + public static ScanBuilder rangeOf(String table, List startList, + List endList) { + HgAssert.isArgumentValid(table, "table"); + HgAssert.isArgumentValid(startList, "startList"); + HgAssert.isArgumentValid(endList, "endList"); + HgAssert.isTrue(startList.size() == endList.size() + , "The size of startList not equals endList's."); + + ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.RANGE, table); + res.startList = startList; + res.endList = endList; + res.scanType = HgKvStore.SCAN_GTE_BEGIN | HgKvStore.SCAN_LTE_END; + return res; + } + + public static ScanBuilder prefixOf(String table, List prefixList) { + HgAssert.isArgumentValid(table, "table"); + HgAssert.isArgumentValid(prefixList, "prefixList"); + + ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.PREFIX, table); + res.prefixList = prefixList; + return res; + + } + + public static ScanBuilder prefixIteratorOf(String table, Iterator prefixItr) { + HgAssert.isArgumentValid(table, "table"); + + ScanBuilder res = new ScanBuilder(HgScanQuery.ScanMethod.PREFIX, table); + res.prefixItr = prefixItr; + return res; + + } + + public static ScanBuilder tableOf(String table) { + HgAssert.isArgumentValid(table, "table"); + + return new ScanBuilder(HgScanQuery.ScanMethod.ALL, table); + } + + ScanBuilder(HgScanQuery.ScanMethod sanMethod, String table) { + this.table = table; + this.sanMethod = sanMethod; + this.orderType = ScanOrderType.ORDER_NONE; + } + + public ScanBuilder setLimit(long limit) { + this.limit = limit; + return this; + } + + public ScanBuilder setPerKeyLimit(long limit) { + this.perKeyLimit = limit; + return this; + } + + public ScanBuilder setPerKeyMax(long max) { + this.perKeyMax = max; + return this; + } + + public ScanBuilder setScanType(int scanType) { + this.scanType = scanType; + return this; + } + + public ScanBuilder setOrderType(ScanOrderType orderType) { + this.orderType = orderType; + return this; + } + + public ScanBuilder setQuery(byte[] query) { + this.query = query; + return this; + } + + public ScanBuilder setSkipDegree(long skipDegree) { + this.skipDegree = skipDegree; + return this; + } + + public ScanBuilder setOnlyKey(boolean onlyKey) { + this.onlyKey = onlyKey; + return this; + } + + + public HgScanQuery build() { + return this.new BatchScanQuery(); + } + + private class BatchScanQuery implements HgScanQuery { + + @Override + public String getTable() { + return table; + } + + @Override + public HgScanQuery.ScanMethod getScanMethod() { + return sanMethod; + } + + @Override + public List getPrefixList() { + if (prefixList == null) { + return Collections.EMPTY_LIST; + } else { + return Collections.unmodifiableList(prefixList); + } + } + + @Override + public Iterator getPrefixItr() { + return prefixItr; + } + + @Override + public List getStartList() { + if (startList == null) { + return Collections.EMPTY_LIST; + } else { + return Collections.unmodifiableList(startList); + } + } + + @Override + public List getEndList() { + if (endList == null) { + return Collections.EMPTY_LIST; + } else { + return Collections.unmodifiableList(endList); + } + } + + @Override + public long getLimit() { + return limit; + } + + @Override + public long getPerKeyLimit() { + return perKeyLimit; + } + + @Override + public long getPerKeyMax() { + return perKeyMax; + } + + @Override + public long getSkipDegree() { + return skipDegree; + } + + @Override + public int getScanType() { + return scanType; + } + + @Override + public ScanOrderType getOrderType() { + return orderType; + } + + @Override + public boolean isOnlyKey() { + return onlyKey; + } + + @Override + public byte[] getQuery() { + return query; + } + + @Override + public ScanBuilder builder() { + return ScanBuilder.this; + } + + @Override + public String toString() { + final StringBuffer sb = new StringBuffer("HgScanQuery{"); + sb.append("table='").append(getTable()).append('\''); + sb.append(", scanMethod=").append(getScanMethod()); + sb.append(", prefixList=").append(getPrefixList()); + sb.append(", startList=").append(getStartList()); + sb.append(", endList=").append(getEndList()); + sb.append(", limit=").append(getLimit()); + sb.append(", perKeyLimit=").append(getPerKeyLimit()); + sb.append(", perKeyMax=").append(getPerKeyMax()); + sb.append(", skipDegree=").append(getSkipDegree()); + sb.append(", scanType=").append(getScanType()); + sb.append(", orderType=").append(getOrderType()); + sb.append(", onlyKey=").append(isOnlyKey()); + sb.append(", query="); + if (query == null) { + sb.append("null"); + } else { + sb.append('['); + for (int i = 0; i < query.length; ++i) { + sb.append(i == 0 ? "" : ", ").append(query[i]); + } + sb.append(']'); + } + sb.append('}'); + return sb.toString(); + } + } + + + } +} + diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/ShiftWorkIteratorProxy.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/ShiftWorkIteratorProxy.java index 47c8abbea..d2a2392b5 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/ShiftWorkIteratorProxy.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/ShiftWorkIteratorProxy.java @@ -1,157 +1,157 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client; - -import java.util.LinkedList; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.Queue; - -import org.apache.hugegraph.store.HgKvEntry; -import org.apache.hugegraph.store.HgKvIterator; -import org.apache.hugegraph.store.HgKvPagingIterator; - -import lombok.extern.slf4j.Slf4j; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/24 - * @version 0.1.1 - */ -@Slf4j -public class ShiftWorkIteratorProxy implements HgKvIterator { - private static final byte[] EMPTY_BYTES = new byte[0]; - private HgKvPagingIterator iterator; - private Queue queue = new LinkedList<>(); - private HgKvEntry entry; - private final int limit; - private int count; - private int shiftCount; - - ShiftWorkIteratorProxy(List iterators, int limit) { - this.queue = new LinkedList<>(iterators); - this.limit = limit <= 0 ? Integer.MAX_VALUE : limit; - } - - private HgKvPagingIterator getIterator() { - if (this.queue.isEmpty()) return null; - - HgKvPagingIterator buf = null; - - while ((buf = this.queue.poll()) != null) { - if (buf.hasNext()) { - break; - } - } - - if (buf == null) { - return null; - } - - this.queue.add(buf); - - return buf; - } - - private void closeIterators() { - if (this.queue.isEmpty()) return; - - HgKvPagingIterator buf; - - while ((buf = this.queue.poll()) != null) { - buf.close(); - } - - } - - private void setIterator() { - - // if (++this.shiftCount >= this.iterator.getPageSize() / 2) { - if (++this.shiftCount >= this.iterator.getPageSize()) { - this.iterator = null; - this.shiftCount = 0; - } - - } - - private void doNext() { - - } - - @Override - public byte[] key() { - if (this.entry != null) { - return this.entry.key(); - } - return null; - } - - @Override - public byte[] value() { - if (this.entry != null) { - return this.entry.value(); - } - return null; - } - - @Override - public byte[] position() { - return this.iterator != null ? this.iterator.position() : EMPTY_BYTES; - } - - @Override - public void seek(byte[] position) { - if (this.iterator != null) { - this.iterator.seek(position); - } - } - - @Override - public boolean hasNext() { - if (this.count >= this.limit) { - return false; - } - if (this.iterator == null - || !this.iterator.hasNext()) { - this.iterator = this.getIterator(); - } - return this.iterator != null; - } - - @Override - public Object next() { - if (this.iterator == null) { - hasNext(); - } - if (this.iterator == null) { - throw new NoSuchElementException(); - } - this.entry = this.iterator.next(); - this.setIterator(); - this.count++; - //log.info("next - > {}",this.entry); - return this.entry; - } - - @Override - public void close() { - if (this.iterator != null) { - this.iterator.close(); - } - this.closeIterators(); - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +import java.util.LinkedList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Queue; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvPagingIterator; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/24 + * @version 0.1.1 + */ +@Slf4j +public class ShiftWorkIteratorProxy implements HgKvIterator { + private static final byte[] EMPTY_BYTES = new byte[0]; + private HgKvPagingIterator iterator; + private Queue queue = new LinkedList<>(); + private HgKvEntry entry; + private final int limit; + private int count; + private int shiftCount; + + ShiftWorkIteratorProxy(List iterators, int limit) { + this.queue = new LinkedList<>(iterators); + this.limit = limit <= 0 ? Integer.MAX_VALUE : limit; + } + + private HgKvPagingIterator getIterator() { + if (this.queue.isEmpty()) return null; + + HgKvPagingIterator buf = null; + + while ((buf = this.queue.poll()) != null) { + if (buf.hasNext()) { + break; + } + } + + if (buf == null) { + return null; + } + + this.queue.add(buf); + + return buf; + } + + private void closeIterators() { + if (this.queue.isEmpty()) return; + + HgKvPagingIterator buf; + + while ((buf = this.queue.poll()) != null) { + buf.close(); + } + + } + + private void setIterator() { + + // if (++this.shiftCount >= this.iterator.getPageSize() / 2) { + if (++this.shiftCount >= this.iterator.getPageSize()) { + this.iterator = null; + this.shiftCount = 0; + } + + } + + private void doNext() { + + } + + @Override + public byte[] key() { + if (this.entry != null) { + return this.entry.key(); + } + return null; + } + + @Override + public byte[] value() { + if (this.entry != null) { + return this.entry.value(); + } + return null; + } + + @Override + public byte[] position() { + return this.iterator != null ? this.iterator.position() : EMPTY_BYTES; + } + + @Override + public void seek(byte[] position) { + if (this.iterator != null) { + this.iterator.seek(position); + } + } + + @Override + public boolean hasNext() { + if (this.count >= this.limit) { + return false; + } + if (this.iterator == null + || !this.iterator.hasNext()) { + this.iterator = this.getIterator(); + } + return this.iterator != null; + } + + @Override + public Object next() { + if (this.iterator == null) { + hasNext(); + } + if (this.iterator == null) { + throw new NoSuchElementException(); + } + this.entry = this.iterator.next(); + this.setIterator(); + this.count++; + //log.info("next - > {}",this.entry); + return this.entry; + } + + @Override + public void close() { + if (this.iterator != null) { + this.iterator.close(); + } + this.closeIterators(); + } +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/TopWorkIteratorProxy.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/TopWorkIteratorProxy.java index a101f5b6e..7e46eaa75 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/TopWorkIteratorProxy.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/TopWorkIteratorProxy.java @@ -1,137 +1,137 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client; - -import java.util.LinkedList; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.Queue; - -import org.apache.hugegraph.store.HgKvEntry; -import org.apache.hugegraph.store.HgKvIterator; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/21 - * @version 0.1.0 - */ -class TopWorkIteratorProxy implements HgKvIterator { - private static final byte[] EMPTY_BYTES = new byte[0]; - private HgKvIterator iterator; - private final Queue queue; - private HgKvEntry entry; - private final long limit; - private int count; - - TopWorkIteratorProxy(List iterators, long limit) { - this.queue = new LinkedList<>(iterators); - this.limit = limit <= 0 ? Integer.MAX_VALUE : limit; - } - - private HgKvIterator getIterator() { - if (this.queue.isEmpty()) return null; - - HgKvIterator buf = null; - - while ((buf = this.queue.poll()) != null) { - if (buf.hasNext()) { - break; - } - } - - if (buf == null) { - return null; - } - - this.queue.add(buf); - - return buf; - } - - private void closeIterators() { - if (this.queue.isEmpty()) return; - - HgKvIterator buf; - - while ((buf = this.queue.poll()) != null) { - buf.close(); - } - - } - - private void setIterator() { - this.iterator = null; - } - - @Override - public byte[] key() { - if (this.entry != null) { - return this.entry.key(); - } - return null; - } - - @Override - public byte[] value() { - if (this.entry != null) { - return this.entry.value(); - } - return null; - } - - @Override - public byte[] position() { - return this.iterator != null ? this.iterator.position() : EMPTY_BYTES; - } - - @Override - public void seek(byte[] position) { - if (this.iterator != null) this.iterator.seek(position); - } - - @Override - public boolean hasNext() { - if (this.count >= this.limit) { - return false; - } - if (this.iterator == null) { - this.iterator = this.getIterator(); - } - return this.iterator != null; - - } - - @Override - public Object next() { - if (this.iterator == null) { - hasNext(); - } - if (this.iterator == null) { - throw new NoSuchElementException(); - } - this.entry = this.iterator.next(); - this.setIterator(); - this.count++; - return this.entry; - } - - @Override - public void close() { - if (this.iterator != null) this.iterator.close(); - this.closeIterators(); - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client; + +import java.util.LinkedList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Queue; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/21 + * @version 0.1.0 + */ +class TopWorkIteratorProxy implements HgKvIterator { + private static final byte[] EMPTY_BYTES = new byte[0]; + private HgKvIterator iterator; + private final Queue queue; + private HgKvEntry entry; + private final long limit; + private int count; + + TopWorkIteratorProxy(List iterators, long limit) { + this.queue = new LinkedList<>(iterators); + this.limit = limit <= 0 ? Integer.MAX_VALUE : limit; + } + + private HgKvIterator getIterator() { + if (this.queue.isEmpty()) return null; + + HgKvIterator buf = null; + + while ((buf = this.queue.poll()) != null) { + if (buf.hasNext()) { + break; + } + } + + if (buf == null) { + return null; + } + + this.queue.add(buf); + + return buf; + } + + private void closeIterators() { + if (this.queue.isEmpty()) return; + + HgKvIterator buf; + + while ((buf = this.queue.poll()) != null) { + buf.close(); + } + + } + + private void setIterator() { + this.iterator = null; + } + + @Override + public byte[] key() { + if (this.entry != null) { + return this.entry.key(); + } + return null; + } + + @Override + public byte[] value() { + if (this.entry != null) { + return this.entry.value(); + } + return null; + } + + @Override + public byte[] position() { + return this.iterator != null ? this.iterator.position() : EMPTY_BYTES; + } + + @Override + public void seek(byte[] position) { + if (this.iterator != null) this.iterator.seek(position); + } + + @Override + public boolean hasNext() { + if (this.count >= this.limit) { + return false; + } + if (this.iterator == null) { + this.iterator = this.getIterator(); + } + return this.iterator != null; + + } + + @Override + public Object next() { + if (this.iterator == null) { + hasNext(); + } + if (this.iterator == null) { + throw new NoSuchElementException(); + } + this.entry = this.iterator.next(); + this.setIterator(); + this.count++; + return this.entry; + } + + @Override + public void close() { + if (this.iterator != null) this.iterator.close(); + this.closeIterators(); + } +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcKvIteratorImpl.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcKvIteratorImpl.java index 777a9ffae..f1b85ebf3 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcKvIteratorImpl.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcKvIteratorImpl.java @@ -1,161 +1,161 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client.grpc; - -import java.util.List; - -import org.apache.hugegraph.store.HgKvEntry; -import org.apache.hugegraph.store.HgKvIterator; -import org.apache.hugegraph.store.HgKvOrderedIterator; -import org.apache.hugegraph.store.HgKvPagingIterator; -import org.apache.hugegraph.store.HgPageSize; -import org.apache.hugegraph.store.HgSeekAble; -import org.apache.hugegraph.store.client.HgStoreNodeSession; -import org.apache.hugegraph.store.client.util.HgStoreClientConst; -import org.apache.hugegraph.store.client.util.HgStoreClientUtil; -import org.apache.hugegraph.store.grpc.common.Kv; - -import lombok.extern.slf4j.Slf4j; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/20 - * @version 0.2.1 - */ -@Slf4j -class GrpcKvIteratorImpl implements HgKvPagingIterator, HgKvOrderedIterator { - - private final byte[] emptyBytes = HgStoreClientConst.EMPTY_BYTES; - private final KvCloseableIterator iterator; - private HgKvEntry element; - private final HgPageSize pageLimiter; - private final HgStoreNodeSession session; - - public static HgKvIterator of(HgStoreNodeSession nodeSession, - KvCloseableIterator iterator) { - if (iterator instanceof HgPageSize) { - return of(nodeSession, iterator, (HgPageSize) iterator); - } - return new GrpcKvIteratorImpl(nodeSession, iterator, () -> 1); - } - - public static HgKvIterator of(HgStoreNodeSession nodeSession, - KvCloseableIterator iterator, - HgPageSize pageLimiter) { - return new GrpcKvIteratorImpl(nodeSession, iterator, pageLimiter); - } - - public static HgKvIterator of(HgStoreNodeSession nodeSession, List kvList) { - int pageSize = kvList.size(); - return new GrpcKvIteratorImpl(nodeSession, new KvListIterator(kvList), () -> pageSize); - } - - private GrpcKvIteratorImpl(HgStoreNodeSession session, KvCloseableIterator iterator, - HgPageSize pageLimiter) { - this.iterator = iterator; - this.pageLimiter = pageLimiter; - this.session = session; - } - - @Override - public boolean hasNext() { - // if (log.isDebugEnabled()) { - // if (!this.iterator.hasNext() && !nodeSession.getGraphName().endsWith("/s")) { - // log.debug("[ANALYSIS GrpcKv hasNext-> FALSE] "); - // } - // } - return this.iterator.hasNext(); - } - - @Override - public HgKvEntry next() { - Kv kv = this.iterator.next(); - this.element = new GrpcKvEntryImpl(kv.getKey().toByteArray(), kv.getValue().toByteArray(), - kv.getCode()); - return this.element; - } - - @Override - public byte[] key() { - if (this.element == null) { - return null; - } - return this.element.key(); - } - - @Override - public byte[] value() { - if (this.element == null) { - return null; - } - return this.element.value(); - } - - @Override - public byte[] position() { - if (this.element == null) { - return emptyBytes; - } - byte[] key = this.element.key(); - if (key == null) { - return emptyBytes; - } - if (!(this.iterator instanceof HgSeekAble)) { - return emptyBytes; - } - byte[] upstream = ((HgSeekAble) this.iterator).position(); - byte[] code = HgStoreClientUtil.toIntBytes(this.element.code()); - byte[] result = new byte[upstream.length + Integer.BYTES + key.length]; - System.arraycopy(upstream, 0, result, 0, upstream.length); - System.arraycopy(code, 0, result, upstream.length, Integer.BYTES); - System.arraycopy(key, 0, result, upstream.length + Integer.BYTES, key.length); - return result; - } - - @Override - public void seek(byte[] position) { - if (this.iterator instanceof HgSeekAble) { - ((HgSeekAble) this.iterator).seek(position); - } - } - - @Override - public long getPageSize() { - return pageLimiter.getPageSize(); - } - - @Override - public boolean isPageEmpty() { - return !iterator.hasNext(); - } - - - @Override - public int compareTo(HgKvOrderedIterator o) { - return Long.compare(this.getSequence(), o.getSequence()); - } - - @Override - public long getSequence() { - return this.session.getStoreNode().getNodeId().longValue(); - } - - @Override - public void close() { - this.iterator.close(); - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.util.List; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvOrderedIterator; +import org.apache.hugegraph.store.HgKvPagingIterator; +import org.apache.hugegraph.store.HgPageSize; +import org.apache.hugegraph.store.HgSeekAble; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.HgStoreClientUtil; +import org.apache.hugegraph.store.grpc.common.Kv; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/20 + * @version 0.2.1 + */ +@Slf4j +class GrpcKvIteratorImpl implements HgKvPagingIterator, HgKvOrderedIterator { + + private final byte[] emptyBytes = HgStoreClientConst.EMPTY_BYTES; + private final KvCloseableIterator iterator; + private HgKvEntry element; + private final HgPageSize pageLimiter; + private final HgStoreNodeSession session; + + public static HgKvIterator of(HgStoreNodeSession nodeSession, + KvCloseableIterator iterator) { + if (iterator instanceof HgPageSize) { + return of(nodeSession, iterator, (HgPageSize) iterator); + } + return new GrpcKvIteratorImpl(nodeSession, iterator, () -> 1); + } + + public static HgKvIterator of(HgStoreNodeSession nodeSession, + KvCloseableIterator iterator, + HgPageSize pageLimiter) { + return new GrpcKvIteratorImpl(nodeSession, iterator, pageLimiter); + } + + public static HgKvIterator of(HgStoreNodeSession nodeSession, List kvList) { + int pageSize = kvList.size(); + return new GrpcKvIteratorImpl(nodeSession, new KvListIterator(kvList), () -> pageSize); + } + + private GrpcKvIteratorImpl(HgStoreNodeSession session, KvCloseableIterator iterator, + HgPageSize pageLimiter) { + this.iterator = iterator; + this.pageLimiter = pageLimiter; + this.session = session; + } + + @Override + public boolean hasNext() { + // if (log.isDebugEnabled()) { + // if (!this.iterator.hasNext() && !nodeSession.getGraphName().endsWith("/s")) { + // log.debug("[ANALYSIS GrpcKv hasNext-> FALSE] "); + // } + // } + return this.iterator.hasNext(); + } + + @Override + public HgKvEntry next() { + Kv kv = this.iterator.next(); + this.element = new GrpcKvEntryImpl(kv.getKey().toByteArray(), kv.getValue().toByteArray(), + kv.getCode()); + return this.element; + } + + @Override + public byte[] key() { + if (this.element == null) { + return null; + } + return this.element.key(); + } + + @Override + public byte[] value() { + if (this.element == null) { + return null; + } + return this.element.value(); + } + + @Override + public byte[] position() { + if (this.element == null) { + return emptyBytes; + } + byte[] key = this.element.key(); + if (key == null) { + return emptyBytes; + } + if (!(this.iterator instanceof HgSeekAble)) { + return emptyBytes; + } + byte[] upstream = ((HgSeekAble) this.iterator).position(); + byte[] code = HgStoreClientUtil.toIntBytes(this.element.code()); + byte[] result = new byte[upstream.length + Integer.BYTES + key.length]; + System.arraycopy(upstream, 0, result, 0, upstream.length); + System.arraycopy(code, 0, result, upstream.length, Integer.BYTES); + System.arraycopy(key, 0, result, upstream.length + Integer.BYTES, key.length); + return result; + } + + @Override + public void seek(byte[] position) { + if (this.iterator instanceof HgSeekAble) { + ((HgSeekAble) this.iterator).seek(position); + } + } + + @Override + public long getPageSize() { + return pageLimiter.getPageSize(); + } + + @Override + public boolean isPageEmpty() { + return !iterator.hasNext(); + } + + + @Override + public int compareTo(HgKvOrderedIterator o) { + return Long.compare(this.getSequence(), o.getSequence()); + } + + @Override + public long getSequence() { + return this.session.getStoreNode().getNodeId().longValue(); + } + + @Override + public void close() { + this.iterator.close(); + } +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreStreamClient.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreStreamClient.java index 727526a4e..2a5bed53f 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreStreamClient.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreStreamClient.java @@ -1,218 +1,218 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client.grpc; - -import javax.annotation.concurrent.ThreadSafe; - -import org.apache.hugegraph.store.HgKvEntry; -import org.apache.hugegraph.store.HgKvIterator; -import org.apache.hugegraph.store.HgOwnerKey; -import org.apache.hugegraph.store.HgScanQuery; -import org.apache.hugegraph.store.client.HgStoreNodeSession; -import org.apache.hugegraph.store.grpc.common.Kv; -import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc; -import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamBlockingStub; -import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamStub; - -import io.grpc.ManagedChannel; -import io.grpc.stub.AbstractAsyncStub; -import io.grpc.stub.AbstractBlockingStub; -import lombok.extern.slf4j.Slf4j; - -/** - * @author lynn.bond@hotmail.com created on 2021/10/19 - * @version 1.1.1 added synchronized in getChannel. - */ -@Slf4j -@ThreadSafe -public class GrpcStoreStreamClient extends AbstractGrpcClient { - - public HgStoreStreamStub getStub(HgStoreNodeSession nodeSession) { - return (HgStoreStreamStub) getAsyncStub(nodeSession.getStoreNode().getAddress()); - } - - @Override - public AbstractAsyncStub getAsyncStub(ManagedChannel channel) { - return HgStoreStreamGrpc.newStub(channel); - } - - private HgStoreStreamBlockingStub getBlockingStub(HgStoreNodeSession nodeSession) { - return (HgStoreStreamBlockingStub) getBlockingStub(nodeSession.getStoreNode().getAddress()); - } - - @Override - public AbstractBlockingStub getBlockingStub(ManagedChannel channel) { - return HgStoreStreamGrpc.newBlockingStub(channel); - } - - KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, long limit, - byte[] query) { - return KvOneShotScanner.scanAll(nodeSession - , this.getBlockingStub(nodeSession) - , table - , limit - , query - ); - } - - KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, - long limit) { - return KvOneShotScanner.scanAll(nodeSession - , this.getBlockingStub(nodeSession) - , table - , limit - , null - ); - } - - KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, - HgOwnerKey prefix, long limit) { - return KvOneShotScanner.scanPrefix(nodeSession - , this.getBlockingStub(nodeSession) - , table - , prefix - , limit - , null - ); - } - - KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, - HgOwnerKey prefix, long limit, - byte[] query) { - return KvOneShotScanner.scanPrefix(nodeSession - , this.getBlockingStub(nodeSession) - , table - , prefix - , limit - , query - ); - } - - KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, - HgOwnerKey startKey, - HgOwnerKey endKey - , long limit - , int scanType - , byte[] query) { - - return KvOneShotScanner.scanRange(nodeSession - , this.getBlockingStub(nodeSession) - , table - , startKey - , endKey - , limit - , scanType - , query - ); - } - - - KvCloseableIterator doScan(HgStoreNodeSession nodeSession - , String table - , long limit - , byte[] query) { - - return KvPageScanner.scanAll(nodeSession - , this.getStub(nodeSession) - , table - , limit - , query - ); - } - - KvCloseableIterator doScan(HgStoreNodeSession nodeSession - , String table - , long limit) { - - return KvPageScanner.scanAll(nodeSession - , this.getStub(nodeSession) - , table - , limit - , null - ); - } - - KvCloseableIterator doScan(HgStoreNodeSession nodeSession - , String table - , HgOwnerKey prefix - , long limit) { - - return KvPageScanner.scanPrefix(nodeSession - , this.getStub(nodeSession) - , table - , prefix - , limit - , null - ); - } - - KvCloseableIterator doScan(HgStoreNodeSession nodeSession - , String table - , HgOwnerKey prefix - , long limit - , byte[] query) { - - return KvPageScanner.scanPrefix(nodeSession - , this.getStub(nodeSession) - , table - , prefix - , limit - , query - ); - } - - KvCloseableIterator doScan(HgStoreNodeSession nodeSession - , String table - , HgOwnerKey startKey - , HgOwnerKey endKey - , long limit - , int scanType - , byte[] query) { - - return KvPageScanner.scanRange(nodeSession - , this.getStub(nodeSession) - , table - , startKey - , endKey - , limit - , scanType - , query - ); - } - - KvCloseableIterator doBatchScan(HgStoreNodeSession nodeSession, HgScanQuery scanQuery) { - return KvBatchScanner5.scan(nodeSession, this.getStub(nodeSession), scanQuery); - } - - - // 返回多个小的迭代器，允许上层并行处理 - KvCloseableIterator> doBatchScan3(HgStoreNodeSession nodeSession, - HgScanQuery scanQuery, - KvCloseableIterator iterator) { - KvBatchScanner.scan(this.getStub(nodeSession), nodeSession.getGraphName(), scanQuery, - iterator); - return iterator; - } - - KvCloseableIterator doBatchScanOneShot(HgStoreNodeSession nodeSession, - HgScanQuery scanQuery) { - return KvBatchOneShotScanner.scan(nodeSession, this.getBlockingStub(nodeSession), - scanQuery); - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import javax.annotation.concurrent.ThreadSafe; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.grpc.common.Kv; +import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc; +import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamBlockingStub; +import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamStub; + +import io.grpc.ManagedChannel; +import io.grpc.stub.AbstractAsyncStub; +import io.grpc.stub.AbstractBlockingStub; +import lombok.extern.slf4j.Slf4j; + +/** + * @author lynn.bond@hotmail.com created on 2021/10/19 + * @version 1.1.1 added synchronized in getChannel. + */ +@Slf4j +@ThreadSafe +public class GrpcStoreStreamClient extends AbstractGrpcClient { + + public HgStoreStreamStub getStub(HgStoreNodeSession nodeSession) { + return (HgStoreStreamStub) getAsyncStub(nodeSession.getStoreNode().getAddress()); + } + + @Override + public AbstractAsyncStub getAsyncStub(ManagedChannel channel) { + return HgStoreStreamGrpc.newStub(channel); + } + + private HgStoreStreamBlockingStub getBlockingStub(HgStoreNodeSession nodeSession) { + return (HgStoreStreamBlockingStub) getBlockingStub(nodeSession.getStoreNode().getAddress()); + } + + @Override + public AbstractBlockingStub getBlockingStub(ManagedChannel channel) { + return HgStoreStreamGrpc.newBlockingStub(channel); + } + + KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, long limit, + byte[] query) { + return KvOneShotScanner.scanAll(nodeSession + , this.getBlockingStub(nodeSession) + , table + , limit + , query + ); + } + + KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, + long limit) { + return KvOneShotScanner.scanAll(nodeSession + , this.getBlockingStub(nodeSession) + , table + , limit + , null + ); + } + + KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, + HgOwnerKey prefix, long limit) { + return KvOneShotScanner.scanPrefix(nodeSession + , this.getBlockingStub(nodeSession) + , table + , prefix + , limit + , null + ); + } + + KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, + HgOwnerKey prefix, long limit, + byte[] query) { + return KvOneShotScanner.scanPrefix(nodeSession + , this.getBlockingStub(nodeSession) + , table + , prefix + , limit + , query + ); + } + + KvCloseableIterator doScanOneShot(HgStoreNodeSession nodeSession, String table, + HgOwnerKey startKey, + HgOwnerKey endKey + , long limit + , int scanType + , byte[] query) { + + return KvOneShotScanner.scanRange(nodeSession + , this.getBlockingStub(nodeSession) + , table + , startKey + , endKey + , limit + , scanType + , query + ); + } + + + KvCloseableIterator doScan(HgStoreNodeSession nodeSession + , String table + , long limit + , byte[] query) { + + return KvPageScanner.scanAll(nodeSession + , this.getStub(nodeSession) + , table + , limit + , query + ); + } + + KvCloseableIterator doScan(HgStoreNodeSession nodeSession + , String table + , long limit) { + + return KvPageScanner.scanAll(nodeSession + , this.getStub(nodeSession) + , table + , limit + , null + ); + } + + KvCloseableIterator doScan(HgStoreNodeSession nodeSession + , String table + , HgOwnerKey prefix + , long limit) { + + return KvPageScanner.scanPrefix(nodeSession + , this.getStub(nodeSession) + , table + , prefix + , limit + , null + ); + } + + KvCloseableIterator doScan(HgStoreNodeSession nodeSession + , String table + , HgOwnerKey prefix + , long limit + , byte[] query) { + + return KvPageScanner.scanPrefix(nodeSession + , this.getStub(nodeSession) + , table + , prefix + , limit + , query + ); + } + + KvCloseableIterator doScan(HgStoreNodeSession nodeSession + , String table + , HgOwnerKey startKey + , HgOwnerKey endKey + , long limit + , int scanType + , byte[] query) { + + return KvPageScanner.scanRange(nodeSession + , this.getStub(nodeSession) + , table + , startKey + , endKey + , limit + , scanType + , query + ); + } + + KvCloseableIterator doBatchScan(HgStoreNodeSession nodeSession, HgScanQuery scanQuery) { + return KvBatchScanner5.scan(nodeSession, this.getStub(nodeSession), scanQuery); + } + + + // 返回多个小的迭代器，允许上层并行处理 + KvCloseableIterator> doBatchScan3(HgStoreNodeSession nodeSession, + HgScanQuery scanQuery, + KvCloseableIterator iterator) { + KvBatchScanner.scan(this.getStub(nodeSession), nodeSession.getGraphName(), scanQuery, + iterator); + return iterator; + } + + KvCloseableIterator doBatchScanOneShot(HgStoreNodeSession nodeSession, + HgScanQuery scanQuery) { + return KvBatchOneShotScanner.scan(nodeSession, this.getBlockingStub(nodeSession), + scanQuery); + } + +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcUtil.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcUtil.java index 17d61f538..25b60c2ab 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcUtil.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcUtil.java @@ -1,149 +1,149 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client.grpc; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.hugegraph.store.HgKvEntry; -import org.apache.hugegraph.store.HgOwnerKey; -import org.apache.hugegraph.store.client.HgStoreNodeSession; -import org.apache.hugegraph.store.client.util.HgStoreClientConst; -import org.apache.hugegraph.store.grpc.common.Header; -import org.apache.hugegraph.store.grpc.common.Key; -import org.apache.hugegraph.store.grpc.common.Kv; -import org.apache.hugegraph.store.grpc.common.Tk; -import org.apache.hugegraph.store.grpc.common.Tkv; -import org.apache.hugegraph.store.grpc.common.Tp; -import org.apache.hugegraph.store.grpc.common.Tse; - -import com.google.protobuf.ByteString; - -import io.grpc.Status; -import io.grpc.StatusRuntimeException; - -/** - * @author lynn.bond@hotmail.com on 2022/1/19 - */ -final class GrpcUtil { - - private static final ThreadLocal keyBuilder = new ThreadLocal(); - - static Header getHeader(HgStoreNodeSession nodeSession) { - return Header.newBuilder() - .setGraph(nodeSession.getGraphName()) - .build(); - } - - static Tk toTk(String table, HgOwnerKey ownerKey) { - return Tk.newBuilder() - .setTable(table) - .setKey(ByteString.copyFrom(ownerKey.getKey())) - .setCode(ownerKey.getKeyCode()) - .build(); - } - - static Key.Builder getOwnerKeyBuilder() { - Key.Builder builder = keyBuilder.get(); - if (builder == null) { - builder = Key.newBuilder(); - // TODO 线程级变量，寻找删除时机 - keyBuilder.set(builder); - } - return builder; - } - - static Key toKey(HgOwnerKey ownerKey, Key.Builder builder) { - if (ownerKey == null) { - return null; - } - return builder - .setKey(ByteString.copyFrom(ownerKey.getKey())) - .setCode(ownerKey.getKeyCode()) - .build(); - } - - - static Key toKey(HgOwnerKey ownerKey) { - if (ownerKey == null) { - return null; - } - Key.Builder builder = keyBuilder.get(); - if (builder == null) { - builder = Key.newBuilder(); - // TODO 线程级变量，寻找删除时机 - keyBuilder.set(builder); - } - return builder - .setKey(ByteString.copyFrom(ownerKey.getKey())) - .setCode(ownerKey.getKeyCode()) - .build(); - } - - static Tkv toTkv(String table, HgOwnerKey ownerKey, byte[] value) { - return Tkv.newBuilder() - .setTable(table) - .setKey(ByteString.copyFrom(ownerKey.getKey())) - .setValue(ByteString.copyFrom(value)) - .setCode(ownerKey.getKeyCode()) - .build(); - } - - static Tp toTp(String table, HgOwnerKey ownerKey) { - return Tp.newBuilder() - .setTable(table) - .setPrefix(ByteString.copyFrom(ownerKey.getKey())) - .setCode(ownerKey.getKeyCode()) - .build(); - } - - static Tse toTse(String table, HgOwnerKey startKey, HgOwnerKey endKey) { - return Tse.newBuilder() - .setTable(table) - .setStart(toKey(startKey)) - .setEnd(toKey(endKey)) - .build(); - - } - - static List toList(List kvList) { - if (kvList == null || kvList.isEmpty()) { - return HgStoreClientConst.EMPTY_LIST; - } - - Iterator iter = kvList.iterator(); - List resList = new ArrayList<>(kvList.size()); - - while (iter.hasNext()) { - Kv entry = iter.next(); - resList.add(new GrpcKvEntryImpl(entry.getKey().toByteArray(), - entry.getValue().toByteArray(), entry.getCode())); - } - - return resList; - } - - static StatusRuntimeException toErr(String msg) { - return new StatusRuntimeException(Status.UNKNOWN.withDescription(msg)); - } - - static ByteString toBs(byte[] bytes) { - return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES); - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.grpc.common.Header; +import org.apache.hugegraph.store.grpc.common.Key; +import org.apache.hugegraph.store.grpc.common.Kv; +import org.apache.hugegraph.store.grpc.common.Tk; +import org.apache.hugegraph.store.grpc.common.Tkv; +import org.apache.hugegraph.store.grpc.common.Tp; +import org.apache.hugegraph.store.grpc.common.Tse; + +import com.google.protobuf.ByteString; + +import io.grpc.Status; +import io.grpc.StatusRuntimeException; + +/** + * @author lynn.bond@hotmail.com on 2022/1/19 + */ +final class GrpcUtil { + + private static final ThreadLocal keyBuilder = new ThreadLocal(); + + static Header getHeader(HgStoreNodeSession nodeSession) { + return Header.newBuilder() + .setGraph(nodeSession.getGraphName()) + .build(); + } + + static Tk toTk(String table, HgOwnerKey ownerKey) { + return Tk.newBuilder() + .setTable(table) + .setKey(ByteString.copyFrom(ownerKey.getKey())) + .setCode(ownerKey.getKeyCode()) + .build(); + } + + static Key.Builder getOwnerKeyBuilder() { + Key.Builder builder = keyBuilder.get(); + if (builder == null) { + builder = Key.newBuilder(); + // TODO 线程级变量，寻找删除时机 + keyBuilder.set(builder); + } + return builder; + } + + static Key toKey(HgOwnerKey ownerKey, Key.Builder builder) { + if (ownerKey == null) { + return null; + } + return builder + .setKey(ByteString.copyFrom(ownerKey.getKey())) + .setCode(ownerKey.getKeyCode()) + .build(); + } + + + static Key toKey(HgOwnerKey ownerKey) { + if (ownerKey == null) { + return null; + } + Key.Builder builder = keyBuilder.get(); + if (builder == null) { + builder = Key.newBuilder(); + // TODO 线程级变量，寻找删除时机 + keyBuilder.set(builder); + } + return builder + .setKey(ByteString.copyFrom(ownerKey.getKey())) + .setCode(ownerKey.getKeyCode()) + .build(); + } + + static Tkv toTkv(String table, HgOwnerKey ownerKey, byte[] value) { + return Tkv.newBuilder() + .setTable(table) + .setKey(ByteString.copyFrom(ownerKey.getKey())) + .setValue(ByteString.copyFrom(value)) + .setCode(ownerKey.getKeyCode()) + .build(); + } + + static Tp toTp(String table, HgOwnerKey ownerKey) { + return Tp.newBuilder() + .setTable(table) + .setPrefix(ByteString.copyFrom(ownerKey.getKey())) + .setCode(ownerKey.getKeyCode()) + .build(); + } + + static Tse toTse(String table, HgOwnerKey startKey, HgOwnerKey endKey) { + return Tse.newBuilder() + .setTable(table) + .setStart(toKey(startKey)) + .setEnd(toKey(endKey)) + .build(); + + } + + static List toList(List kvList) { + if (kvList == null || kvList.isEmpty()) { + return HgStoreClientConst.EMPTY_LIST; + } + + Iterator iter = kvList.iterator(); + List resList = new ArrayList<>(kvList.size()); + + while (iter.hasNext()) { + Kv entry = iter.next(); + resList.add(new GrpcKvEntryImpl(entry.getKey().toByteArray(), + entry.getValue().toByteArray(), entry.getCode())); + } + + return resList; + } + + static StatusRuntimeException toErr(String msg) { + return new StatusRuntimeException(Status.UNKNOWN.withDescription(msg)); + } + + static ByteString toBs(byte[] bytes) { + return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES); + } +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchOneShotScanner.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchOneShotScanner.java index 809c950bf..2d4b5c9d9 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchOneShotScanner.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchOneShotScanner.java @@ -1,123 +1,123 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client.grpc; - -import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.EMPTY_POSITION; -import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.createQueryReq; -import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.getHeader; - -import java.util.Iterator; -import java.util.List; - -import javax.annotation.concurrent.NotThreadSafe; - -import org.apache.hugegraph.store.HgPageSize; -import org.apache.hugegraph.store.HgScanQuery; -import org.apache.hugegraph.store.HgSeekAble; -import org.apache.hugegraph.store.client.HgStoreNodeSession; -import org.apache.hugegraph.store.grpc.common.Kv; -import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc; -import org.apache.hugegraph.store.grpc.stream.ScanStreamBatchReq; - -import lombok.extern.slf4j.Slf4j; - -/** - * @author lynn.bond@hotmail.com created on 2022/04/08 - */ -@Slf4j -@NotThreadSafe -class KvBatchOneShotScanner implements KvCloseableIterator, HgPageSize, HgSeekAble { - private final HgStoreNodeSession nodeSession; - private final HgStoreStreamGrpc.HgStoreStreamBlockingStub stub; - private final HgScanQuery scanQuery; - - private Iterator iterator; - private List list = null; - - public static KvCloseableIterator scan(HgStoreNodeSession nodeSession, - HgStoreStreamGrpc.HgStoreStreamBlockingStub stub, - HgScanQuery scanQuery) { - - return new KvBatchOneShotScanner(nodeSession, stub, scanQuery); - } - - private KvBatchOneShotScanner(HgStoreNodeSession nodeSession, - HgStoreStreamGrpc.HgStoreStreamBlockingStub stub, - HgScanQuery scanQuery) { - - this.nodeSession = nodeSession; - this.stub = stub; - this.scanQuery = scanQuery; - } - - private ScanStreamBatchReq createReq() { - return ScanStreamBatchReq.newBuilder() - .setHeader(getHeader(this.nodeSession)) - .setQueryRequest(createQueryReq(this.scanQuery, Integer.MAX_VALUE)) - .build(); - } - - private Iterator createIterator() { - this.list = this.stub.scanBatchOneShot(this.createReq()).getDataList(); - return this.list.iterator(); - } - - /*** Iterator ***/ - @Override - public boolean hasNext() { - if (this.iterator == null) { - this.iterator = this.createIterator(); - } - return this.iterator.hasNext(); - } - - @Override - public Kv next() { - if (this.iterator == null) { - this.iterator = this.createIterator(); - } - return this.iterator.next(); - } - - @Override - public long getPageSize() { - return Integer.MAX_VALUE; - } - - @Override - public boolean isPageEmpty() { - return !this.iterator.hasNext(); - } - - @Override - public byte[] position() { - //TODO: to implement - return EMPTY_POSITION; - } - - @Override - public void seek(byte[] position) { - //TODO: to implement - } - - @Override - public void close() { - //Nothing to do - } - +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.EMPTY_POSITION; +import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.createQueryReq; +import static org.apache.hugegraph.store.client.grpc.KvBatchUtil.getHeader; + +import java.util.Iterator; +import java.util.List; + +import javax.annotation.concurrent.NotThreadSafe; + +import org.apache.hugegraph.store.HgPageSize; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.HgSeekAble; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.grpc.common.Kv; +import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc; +import org.apache.hugegraph.store.grpc.stream.ScanStreamBatchReq; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author lynn.bond@hotmail.com created on 2022/04/08 + */ +@Slf4j +@NotThreadSafe +class KvBatchOneShotScanner implements KvCloseableIterator, HgPageSize, HgSeekAble { + private final HgStoreNodeSession nodeSession; + private final HgStoreStreamGrpc.HgStoreStreamBlockingStub stub; + private final HgScanQuery scanQuery; + + private Iterator iterator; + private List list = null; + + public static KvCloseableIterator scan(HgStoreNodeSession nodeSession, + HgStoreStreamGrpc.HgStoreStreamBlockingStub stub, + HgScanQuery scanQuery) { + + return new KvBatchOneShotScanner(nodeSession, stub, scanQuery); + } + + private KvBatchOneShotScanner(HgStoreNodeSession nodeSession, + HgStoreStreamGrpc.HgStoreStreamBlockingStub stub, + HgScanQuery scanQuery) { + + this.nodeSession = nodeSession; + this.stub = stub; + this.scanQuery = scanQuery; + } + + private ScanStreamBatchReq createReq() { + return ScanStreamBatchReq.newBuilder() + .setHeader(getHeader(this.nodeSession)) + .setQueryRequest(createQueryReq(this.scanQuery, Integer.MAX_VALUE)) + .build(); + } + + private Iterator createIterator() { + this.list = this.stub.scanBatchOneShot(this.createReq()).getDataList(); + return this.list.iterator(); + } + + /*** Iterator ***/ + @Override + public boolean hasNext() { + if (this.iterator == null) { + this.iterator = this.createIterator(); + } + return this.iterator.hasNext(); + } + + @Override + public Kv next() { + if (this.iterator == null) { + this.iterator = this.createIterator(); + } + return this.iterator.next(); + } + + @Override + public long getPageSize() { + return Integer.MAX_VALUE; + } + + @Override + public boolean isPageEmpty() { + return !this.iterator.hasNext(); + } + + @Override + public byte[] position() { + //TODO: to implement + return EMPTY_POSITION; + } + + @Override + public void seek(byte[] position) { + //TODO: to implement + } + + @Override + public void close() { + //Nothing to do + } + } \ No newline at end of file diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchScanner5.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchScanner5.java index 754e29041..28e660969 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchScanner5.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchScanner5.java @@ -1,447 +1,447 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client.grpc; - -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Supplier; - -import javax.annotation.concurrent.NotThreadSafe; - -import org.apache.hugegraph.store.HgPageSize; -import org.apache.hugegraph.store.HgScanQuery; -import org.apache.hugegraph.store.client.HgStoreNodeSession; -import org.apache.hugegraph.store.client.type.HgStoreClientException; -import org.apache.hugegraph.store.client.util.Base58; -import org.apache.hugegraph.store.client.util.HgStoreClientConfig; -import org.apache.hugegraph.store.grpc.common.Kv; -import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc; - -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; - -/** - * @author lynn.bond@hotmail.com created on 2022/03/23 - * @version 5.0.0 - */ -@Slf4j -@NotThreadSafe -class KvBatchScanner5 { - private final static HgStoreClientConfig storeClientConfig = HgStoreClientConfig.of(); - //private final static int HAVE_NEXT_TIMEOUT_SECONDS = storeClientConfig - // .getNetKvScannerHaveNextTimeout(); - private final static int HAVE_NEXT_TIMEOUT_SECONDS = 60; - private final static long PAGE_SIZE = storeClientConfig.getNetKvScannerPageSize(); - - private enum OrderState { - NEW(0), - WORKING(1), - COMPLETED(10); - - int value; - - OrderState(int value) { - this.value = value; - } - } - - public static KvCloseableIterator scan(HgStoreNodeSession nodeSession, - HgStoreStreamGrpc.HgStoreStreamStub stub, - HgScanQuery scanQuery) { - return new OrderConsumer(new OrderBroker(stub, scanQuery, nodeSession)); - } - - /*** Broker ***/ - private static class OrderBroker { - private final HgScanQuery scanQuery; - private final StreamObserver requestObserver; - private final ScanStreamBatchReq.Builder reqBuilder; - private final ReentrantLock senderLock = new ReentrantLock(); - private final AtomicBoolean serverFinished = new AtomicBoolean(); - private final AtomicBoolean clientFinished = new AtomicBoolean(); - - private final ScanReceiptRequest.Builder receiptReqBuilder = - ScanReceiptRequest.newBuilder(); - private final ScanCancelRequest cancelReq = ScanCancelRequest.newBuilder().build(); - private final HgStoreNodeSession nodeSession; - private final OrderAgent agent; - - public final OrderKeeper keeper = new OrderKeeper(); - - private OrderState state = OrderState.NEW; - - private final AtomicLong receivedCount = new AtomicLong(); - private final AtomicInteger receivedLastTimes = new AtomicInteger(); - private final BlockingQueue timesQueue = new LinkedBlockingQueue(); - String brokerId = ""; - - OrderBroker(HgStoreStreamGrpc.HgStoreStreamStub stub, - HgScanQuery scanQuery, - HgStoreNodeSession nodeSession) { - - if (log.isDebugEnabled()) { - if (scanQuery.getPrefixList() != null && scanQuery.getPrefixList().size() > 0) { - brokerId = Base58.encode(scanQuery.getPrefixList().get(0).getKey()); - - log.debug( - "[ANALYSIS START] [{}] firstKey: {}, keyLength: {}, table: {}, node: {}" - , brokerId - , scanQuery.getPrefixList().get(0) - , scanQuery.getPrefixList().size() - , scanQuery.getTable() - , nodeSession.getStoreNode().getAddress()); - } - } - - this.scanQuery = scanQuery; - this.reqBuilder = KvBatchUtil.getRequestBuilder(nodeSession); - this.nodeSession = nodeSession; - this.agent = new OrderAgent(brokerId); - this.requestObserver = stub.scanBatch(agent); - - } - - List oneMore() { - - if (this.state == OrderState.NEW) { - synchronized (this.state) { - if (this.state == OrderState.NEW) { - this.makeADeal(); - this.state = OrderState.WORKING; - } - } - } else { - this.sendReceipt(); - } - - return this.keeper.pickUp(); - } - - void receipt(int times) { - this.timesQueue.offer(times); - receivedLastTimes.set(times); - } - - void sendReceipt() { - Integer buf = this.timesQueue.poll(); - - if (buf == null) { - buf = this.receivedLastTimes.get(); - } - - AtomicInteger timesBuf = new AtomicInteger(buf); - - if (!this.clientFinished.get()) { - this.send(() -> - getReqBuilder().setReceiptRequest( - this.receiptReqBuilder.setTimes(timesBuf.get()).build()) - .build() - ); - } - } - - private void makeADeal() { - this.send(() -> getReqBuilder() - .setQueryRequest(KvBatchUtil.createQueryReq(scanQuery, PAGE_SIZE)).build() - ); - } - - private void finish(long tookAmt) { - this.clientFinished.set(true); - if (log.isDebugEnabled()) { - log.debug("[ANALYSIS END] [{}] times: {}, received: {}, took: {}" - , this.brokerId - , this.receivedLastTimes.get() - , this.receivedCount.get() - , tookAmt - ); - } - if (this.receivedCount.get() != tookAmt) { - if (log.isDebugEnabled()) { - log.debug("[ANALYSIS END] [{}] times: {}, received: {}, took: {}" - , this.brokerId - , this.receivedLastTimes.get() - , this.receivedCount.get() - , tookAmt - ); - } - } - synchronized (this.state) { - if (this.state.value < OrderState.COMPLETED.value) { - this.send(() -> getReqBuilder().setCancelRequest(this.cancelReq).build()); - this.state = OrderState.COMPLETED; - } - } - } - - private ScanStreamBatchReq.Builder getReqBuilder() { - return this.reqBuilder.clearQueryRequest(); - } - - private void send(Supplier supplier) { - this.senderLock.lock(); - try { - if (!this.serverFinished.get()) { - this.requestObserver.onNext(supplier.get()); - } - Thread.yield(); - } finally { - this.senderLock.unlock(); - } - } - - private class OrderAgent implements StreamObserver { - private final AtomicInteger count = new AtomicInteger(0); - private final AtomicBoolean over = new AtomicBoolean(false); - private final String agentId; - - OrderAgent(String agentId) { - this.agentId = agentId; - } - - @Override - public void onNext(KvPageRes value) { - if (log.isDebugEnabled()) { - log.debug("Scan [ {} ] [ {} ] times, received: [ {} ]" - , nodeSession.getStoreNode().getAddress(), value.getTimes(), - value.getDataList().size()); - } - - serverFinished.set(value.getOver()); - - List buffer = value.getDataList(); - count.addAndGet(buffer.size()); - if (log.isDebugEnabled()) { - if (value.getOver()) { - log.debug("[ANALYSIS OVER] [{}] count: {}", agentId, count); - } - } - keeper.receive(buffer, value.getTimes()); - this.over.set(value.getOver()); - this.checkOver(value.getTimes()); - } - - private void checkOver(int times) { - if (this.over.get()) { - requestObserver.onCompleted(); - keeper.done(times); - } - } - - @Override - public void onError(Throwable t) { - log.error("received server onError event, Throwable:", t); - keeper.shout(t); - } - - @Override - public void onCompleted() { - if (log.isDebugEnabled()) log.debug("received sever completed event."); - serverFinished.set(true); - - } - - } - - /*** Inventory Keeper ***/ - private class OrderKeeper { - private final BlockingQueue>> queue = new LinkedBlockingQueue<>(); - private final ReentrantLock pickUpLock = new ReentrantLock(); - private final AtomicBoolean done = new AtomicBoolean(); - private final AtomicBoolean stop = new AtomicBoolean(); - private int timesOfOver; - private int lastTimes; - private Throwable serverErr; - - void receive(List data, int times) { - receivedCount.addAndGet(data.size()); - this.queue.offer(() -> data); - receipt(times); - - this.lastTimes = times; - } - - private List pickUp() { - Supplier> res; - - pickUpLock.lock(); - try { - - if (this.done.get()) { - if (this.stop.get()) { - log.warn("Invoking pickUp method after OrderKeeper has bean closing."); - } - res = this.queue.poll(); - if (res == null) { - res = () -> null; - } - } else { - res = this.queue.poll(HAVE_NEXT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - if (res == null) { - if (this.done.get()) { - res = () -> null; - } else { - throw HgStoreClientException.of( - "Timeout, max time: " + HAVE_NEXT_TIMEOUT_SECONDS + - " seconds" + - ", isOver: " + this.done.get() + - ", isStop: " + this.stop.get() + - ", last-times: " + this.lastTimes + - ", over-times: " + this.timesOfOver); - } - } - - } - } catch (InterruptedException e) { - log.error( - "Failed to receive List from queue because of interruption of " + - "current thread [" - + Thread.currentThread().getName() + "]"); - - Thread.currentThread().interrupt(); - - throw HgStoreClientException.of( - "Failed to receive List from queue, cause by:", e); - } finally { - pickUpLock.unlock(); - } - - checkServerErr(); - return res.get(); - - } - - void done(int times) { - this.timesOfOver = times; - this.done.set(true); - this.queue.offer(() -> null); - } - - void shout(Throwable t) { - this.serverErr = t; - log.error("Failed to receive from sever", t); - this.queue.offer(() -> null); - } - - private void checkServerErr() { - if (this.serverErr != null) { - throw HgStoreClientException.of(this.serverErr); - } - } - } - - } - - - /* iterator */ - private static class OrderConsumer implements KvCloseableIterator, HgPageSize { - private final OrderBroker broker; - private Iterator dataIterator; - private long tookCount = 0; - private final String consumerId; - - OrderConsumer(OrderBroker broker) { - this.broker = broker; - consumerId = broker.brokerId; - } - - private Iterator getIterator() { - List list = this.broker.oneMore(); - - if (log.isDebugEnabled()) { - if (list != null && list.isEmpty()) { - log.debug("[ANALYSIS EMPTY] [{}] , tookCount: {}", consumerId, tookCount); - } - } - - if (list == null || list.isEmpty()) { - return null; - } else { - return list.iterator(); - } - } - - @Override - public void close() { - this.broker.finish(this.tookCount); - } - - @Override - public long getPageSize() { - return PAGE_SIZE; - } - - @Override - public boolean hasNext() { - - if (this.dataIterator == null) { - this.dataIterator = this.getIterator(); - } else { - if (this.dataIterator.hasNext()) { - return true; - } else { - this.dataIterator = this.getIterator(); - } - } - - if (this.dataIterator == null) { - if (log.isDebugEnabled()) { - log.debug("[ANALYSIS NULL -> FALSE] [{}] , tookCount: {}", consumerId, - tookCount); - } - return false; - } else { - if (log.isDebugEnabled()) { - if (!this.dataIterator.hasNext()) { - log.debug("[ANALYSIS hasNext -> FALSE] [{}] , tookCount: {}", consumerId, - tookCount); - } - } - return this.dataIterator.hasNext(); - } - - } - - @Override - public Kv next() { - if (this.dataIterator == null) { - if (!this.hasNext()) { - throw new NoSuchElementException(); - } - } - - if (log.isDebugEnabled()) { - tookCount++; - if (tookCount % 10000 == 0) { - log.debug("[ANALYSIS NEXT] [{}] , tookCount: {}", consumerId, tookCount); - } - } - return this.dataIterator.next(); - } - - } - +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Supplier; + +import javax.annotation.concurrent.NotThreadSafe; + +import org.apache.hugegraph.store.HgPageSize; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.client.type.HgStoreClientException; +import org.apache.hugegraph.store.client.util.Base58; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.grpc.common.Kv; +import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc; + +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; + +/** + * @author lynn.bond@hotmail.com created on 2022/03/23 + * @version 5.0.0 + */ +@Slf4j +@NotThreadSafe +class KvBatchScanner5 { + private final static HgStoreClientConfig storeClientConfig = HgStoreClientConfig.of(); + //private final static int HAVE_NEXT_TIMEOUT_SECONDS = storeClientConfig + // .getNetKvScannerHaveNextTimeout(); + private final static int HAVE_NEXT_TIMEOUT_SECONDS = 60; + private final static long PAGE_SIZE = storeClientConfig.getNetKvScannerPageSize(); + + private enum OrderState { + NEW(0), + WORKING(1), + COMPLETED(10); + + int value; + + OrderState(int value) { + this.value = value; + } + } + + public static KvCloseableIterator scan(HgStoreNodeSession nodeSession, + HgStoreStreamGrpc.HgStoreStreamStub stub, + HgScanQuery scanQuery) { + return new OrderConsumer(new OrderBroker(stub, scanQuery, nodeSession)); + } + + /*** Broker ***/ + private static class OrderBroker { + private final HgScanQuery scanQuery; + private final StreamObserver requestObserver; + private final ScanStreamBatchReq.Builder reqBuilder; + private final ReentrantLock senderLock = new ReentrantLock(); + private final AtomicBoolean serverFinished = new AtomicBoolean(); + private final AtomicBoolean clientFinished = new AtomicBoolean(); + + private final ScanReceiptRequest.Builder receiptReqBuilder = + ScanReceiptRequest.newBuilder(); + private final ScanCancelRequest cancelReq = ScanCancelRequest.newBuilder().build(); + private final HgStoreNodeSession nodeSession; + private final OrderAgent agent; + + public final OrderKeeper keeper = new OrderKeeper(); + + private OrderState state = OrderState.NEW; + + private final AtomicLong receivedCount = new AtomicLong(); + private final AtomicInteger receivedLastTimes = new AtomicInteger(); + private final BlockingQueue timesQueue = new LinkedBlockingQueue(); + String brokerId = ""; + + OrderBroker(HgStoreStreamGrpc.HgStoreStreamStub stub, + HgScanQuery scanQuery, + HgStoreNodeSession nodeSession) { + + if (log.isDebugEnabled()) { + if (scanQuery.getPrefixList() != null && scanQuery.getPrefixList().size() > 0) { + brokerId = Base58.encode(scanQuery.getPrefixList().get(0).getKey()); + + log.debug( + "[ANALYSIS START] [{}] firstKey: {}, keyLength: {}, table: {}, node: {}" + , brokerId + , scanQuery.getPrefixList().get(0) + , scanQuery.getPrefixList().size() + , scanQuery.getTable() + , nodeSession.getStoreNode().getAddress()); + } + } + + this.scanQuery = scanQuery; + this.reqBuilder = KvBatchUtil.getRequestBuilder(nodeSession); + this.nodeSession = nodeSession; + this.agent = new OrderAgent(brokerId); + this.requestObserver = stub.scanBatch(agent); + + } + + List oneMore() { + + if (this.state == OrderState.NEW) { + synchronized (this.state) { + if (this.state == OrderState.NEW) { + this.makeADeal(); + this.state = OrderState.WORKING; + } + } + } else { + this.sendReceipt(); + } + + return this.keeper.pickUp(); + } + + void receipt(int times) { + this.timesQueue.offer(times); + receivedLastTimes.set(times); + } + + void sendReceipt() { + Integer buf = this.timesQueue.poll(); + + if (buf == null) { + buf = this.receivedLastTimes.get(); + } + + AtomicInteger timesBuf = new AtomicInteger(buf); + + if (!this.clientFinished.get()) { + this.send(() -> + getReqBuilder().setReceiptRequest( + this.receiptReqBuilder.setTimes(timesBuf.get()).build()) + .build() + ); + } + } + + private void makeADeal() { + this.send(() -> getReqBuilder() + .setQueryRequest(KvBatchUtil.createQueryReq(scanQuery, PAGE_SIZE)).build() + ); + } + + private void finish(long tookAmt) { + this.clientFinished.set(true); + if (log.isDebugEnabled()) { + log.debug("[ANALYSIS END] [{}] times: {}, received: {}, took: {}" + , this.brokerId + , this.receivedLastTimes.get() + , this.receivedCount.get() + , tookAmt + ); + } + if (this.receivedCount.get() != tookAmt) { + if (log.isDebugEnabled()) { + log.debug("[ANALYSIS END] [{}] times: {}, received: {}, took: {}" + , this.brokerId + , this.receivedLastTimes.get() + , this.receivedCount.get() + , tookAmt + ); + } + } + synchronized (this.state) { + if (this.state.value < OrderState.COMPLETED.value) { + this.send(() -> getReqBuilder().setCancelRequest(this.cancelReq).build()); + this.state = OrderState.COMPLETED; + } + } + } + + private ScanStreamBatchReq.Builder getReqBuilder() { + return this.reqBuilder.clearQueryRequest(); + } + + private void send(Supplier supplier) { + this.senderLock.lock(); + try { + if (!this.serverFinished.get()) { + this.requestObserver.onNext(supplier.get()); + } + Thread.yield(); + } finally { + this.senderLock.unlock(); + } + } + + private class OrderAgent implements StreamObserver { + private final AtomicInteger count = new AtomicInteger(0); + private final AtomicBoolean over = new AtomicBoolean(false); + private final String agentId; + + OrderAgent(String agentId) { + this.agentId = agentId; + } + + @Override + public void onNext(KvPageRes value) { + if (log.isDebugEnabled()) { + log.debug("Scan [ {} ] [ {} ] times, received: [ {} ]" + , nodeSession.getStoreNode().getAddress(), value.getTimes(), + value.getDataList().size()); + } + + serverFinished.set(value.getOver()); + + List buffer = value.getDataList(); + count.addAndGet(buffer.size()); + if (log.isDebugEnabled()) { + if (value.getOver()) { + log.debug("[ANALYSIS OVER] [{}] count: {}", agentId, count); + } + } + keeper.receive(buffer, value.getTimes()); + this.over.set(value.getOver()); + this.checkOver(value.getTimes()); + } + + private void checkOver(int times) { + if (this.over.get()) { + requestObserver.onCompleted(); + keeper.done(times); + } + } + + @Override + public void onError(Throwable t) { + log.error("received server onError event, Throwable:", t); + keeper.shout(t); + } + + @Override + public void onCompleted() { + if (log.isDebugEnabled()) log.debug("received sever completed event."); + serverFinished.set(true); + + } + + } + + /*** Inventory Keeper ***/ + private class OrderKeeper { + private final BlockingQueue>> queue = new LinkedBlockingQueue<>(); + private final ReentrantLock pickUpLock = new ReentrantLock(); + private final AtomicBoolean done = new AtomicBoolean(); + private final AtomicBoolean stop = new AtomicBoolean(); + private int timesOfOver; + private int lastTimes; + private Throwable serverErr; + + void receive(List data, int times) { + receivedCount.addAndGet(data.size()); + this.queue.offer(() -> data); + receipt(times); + + this.lastTimes = times; + } + + private List pickUp() { + Supplier> res; + + pickUpLock.lock(); + try { + + if (this.done.get()) { + if (this.stop.get()) { + log.warn("Invoking pickUp method after OrderKeeper has bean closing."); + } + res = this.queue.poll(); + if (res == null) { + res = () -> null; + } + } else { + res = this.queue.poll(HAVE_NEXT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + if (res == null) { + if (this.done.get()) { + res = () -> null; + } else { + throw HgStoreClientException.of( + "Timeout, max time: " + HAVE_NEXT_TIMEOUT_SECONDS + + " seconds" + + ", isOver: " + this.done.get() + + ", isStop: " + this.stop.get() + + ", last-times: " + this.lastTimes + + ", over-times: " + this.timesOfOver); + } + } + + } + } catch (InterruptedException e) { + log.error( + "Failed to receive List from queue because of interruption of " + + "current thread [" + + Thread.currentThread().getName() + "]"); + + Thread.currentThread().interrupt(); + + throw HgStoreClientException.of( + "Failed to receive List from queue, cause by:", e); + } finally { + pickUpLock.unlock(); + } + + checkServerErr(); + return res.get(); + + } + + void done(int times) { + this.timesOfOver = times; + this.done.set(true); + this.queue.offer(() -> null); + } + + void shout(Throwable t) { + this.serverErr = t; + log.error("Failed to receive from sever", t); + this.queue.offer(() -> null); + } + + private void checkServerErr() { + if (this.serverErr != null) { + throw HgStoreClientException.of(this.serverErr); + } + } + } + + } + + + /* iterator */ + private static class OrderConsumer implements KvCloseableIterator, HgPageSize { + private final OrderBroker broker; + private Iterator dataIterator; + private long tookCount = 0; + private final String consumerId; + + OrderConsumer(OrderBroker broker) { + this.broker = broker; + consumerId = broker.brokerId; + } + + private Iterator getIterator() { + List list = this.broker.oneMore(); + + if (log.isDebugEnabled()) { + if (list != null && list.isEmpty()) { + log.debug("[ANALYSIS EMPTY] [{}] , tookCount: {}", consumerId, tookCount); + } + } + + if (list == null || list.isEmpty()) { + return null; + } else { + return list.iterator(); + } + } + + @Override + public void close() { + this.broker.finish(this.tookCount); + } + + @Override + public long getPageSize() { + return PAGE_SIZE; + } + + @Override + public boolean hasNext() { + + if (this.dataIterator == null) { + this.dataIterator = this.getIterator(); + } else { + if (this.dataIterator.hasNext()) { + return true; + } else { + this.dataIterator = this.getIterator(); + } + } + + if (this.dataIterator == null) { + if (log.isDebugEnabled()) { + log.debug("[ANALYSIS NULL -> FALSE] [{}] , tookCount: {}", consumerId, + tookCount); + } + return false; + } else { + if (log.isDebugEnabled()) { + if (!this.dataIterator.hasNext()) { + log.debug("[ANALYSIS hasNext -> FALSE] [{}] , tookCount: {}", consumerId, + tookCount); + } + } + return this.dataIterator.hasNext(); + } + + } + + @Override + public Kv next() { + if (this.dataIterator == null) { + if (!this.hasNext()) { + throw new NoSuchElementException(); + } + } + + if (log.isDebugEnabled()) { + tookCount++; + if (tookCount % 10000 == 0) { + log.debug("[ANALYSIS NEXT] [{}] , tookCount: {}", consumerId, tookCount); + } + } + return this.dataIterator.next(); + } + + } + } \ No newline at end of file diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchUtil.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchUtil.java index c325c4bf8..62f24ca12 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchUtil.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvBatchUtil.java @@ -1,152 +1,152 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client.grpc; - -import java.util.List; - -import javax.annotation.concurrent.NotThreadSafe; - -import org.apache.hugegraph.store.HgOwnerKey; -import org.apache.hugegraph.store.HgScanQuery; -import org.apache.hugegraph.store.client.HgStoreNodeSession; -import org.apache.hugegraph.store.client.util.HgStoreClientConst; -import org.apache.hugegraph.store.grpc.common.Header; -import org.apache.hugegraph.store.grpc.common.ScanMethod; - -import com.google.protobuf.ByteString; - -import lombok.extern.slf4j.Slf4j; - -/** - * @author lynn.bond@hotmail.com created on 2022/04/23 - */ -@Slf4j -@NotThreadSafe -class KvBatchUtil { - static final byte[] EMPTY_POSITION = HgStoreClientConst.EMPTY_BYTES; - - static ScanStreamBatchReq.Builder getRequestBuilder(HgStoreNodeSession nodeSession) { - return ScanStreamBatchReq.newBuilder().setHeader(getHeader(nodeSession)); - } - - static ScanQueryRequest createQueryReq(HgScanQuery scanQuery, long pageSize) { - - ScanQueryRequest.Builder qb = ScanQueryRequest.newBuilder(); - ScanCondition.Builder cb = ScanCondition.newBuilder(); - - qb.setLimit(getLimit(scanQuery.getLimit())); - qb.setPerKeyLimit(getLimit(scanQuery.getPerKeyLimit())); - qb.setPerKeyMax(getLimit(scanQuery.getPerKeyMax())); - - switch (scanQuery.getScanMethod()) { - case ALL: - qb.setMethod(ScanMethod.ALL); - break; - case PREFIX: - qb.setMethod(ScanMethod.PREFIX); - addPrefixCondition(scanQuery, qb, cb); - break; - case RANGE: - qb.setMethod(ScanMethod.RANGE); - addRangeCondition(scanQuery, qb, cb); - break; - default: - throw new RuntimeException("Unsupported ScanType: " + scanQuery.getScanMethod()); - } - - qb.setTable(scanQuery.getTable()); - qb.setPageSize(pageSize); - qb.setQuery(toBs(scanQuery.getQuery())); - qb.setScanType(scanQuery.getScanType()); - qb.setOrderType(scanQuery.getOrderType()); - qb.setSkipDegree(scanQuery.getSkipDegree()); - - return qb.build(); - } - - static long getLimit(long limit) { - return limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE : limit; - } - - static Header getHeader(HgStoreNodeSession nodeSession) { - return Header.newBuilder().setGraph(nodeSession.getGraphName()).build(); - } - - static void addPrefixCondition(HgScanQuery scanQuery, ScanQueryRequest.Builder qb, - ScanCondition.Builder cb) { - List prefixList = scanQuery.getPrefixList(); - - if (prefixList == null || prefixList.isEmpty()) { - throw new RuntimeException( - "The start-list of ScanQuery shouldn't to be invalid in ScanMethod.PREFIX " + - "mode."); - } - - prefixList.forEach((e) -> { - qb.addCondition(cb.clear() - .setPrefix(toBs(e.getKey())) - .setCode(e.getKeyCode()) - .setSerialNo(e.getSerialNo()) - .build() - ); - }); - - } - - static void addRangeCondition(HgScanQuery scanQuery, ScanQueryRequest.Builder qb, - ScanCondition.Builder cb) { - List startList = scanQuery.getStartList(); - List endList = scanQuery.getEndList(); - - if (startList == null || startList.isEmpty()) { - throw new RuntimeException( - "The start-list of ScanQuery shouldn't to be invalid in ScanMethod.RANGE mode" + - "."); - } - - if (endList == null || endList.isEmpty()) { - throw new RuntimeException( - "The end-list of ScanQuery shouldn't to be invalid in ScanMethod.RANGE mode."); - } - - if (startList.size() != endList.size()) { - throw new RuntimeException("The size of start-list not equals end-list's."); - } - - for (int i = 0, s = startList.size(); i < s; i++) { - HgOwnerKey start = startList.get(i); - HgOwnerKey end = endList.get(i); - qb.addCondition(cb.clear().setCode(start.getKeyCode()) - .setStart(toBs(start.getKey())) - .setEnd(toBs(end.getKey())) - .setSerialNo(start.getSerialNo()) - .build() - ); - } - - } - - static HgOwnerKey toOk(HgOwnerKey key) { - return key == null ? HgStoreClientConst.EMPTY_OWNER_KEY : key; - } - - static ByteString toBs(byte[] bytes) { - return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES); - } - +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.util.List; + +import javax.annotation.concurrent.NotThreadSafe; + +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.grpc.common.Header; +import org.apache.hugegraph.store.grpc.common.ScanMethod; + +import com.google.protobuf.ByteString; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author lynn.bond@hotmail.com created on 2022/04/23 + */ +@Slf4j +@NotThreadSafe +class KvBatchUtil { + static final byte[] EMPTY_POSITION = HgStoreClientConst.EMPTY_BYTES; + + static ScanStreamBatchReq.Builder getRequestBuilder(HgStoreNodeSession nodeSession) { + return ScanStreamBatchReq.newBuilder().setHeader(getHeader(nodeSession)); + } + + static ScanQueryRequest createQueryReq(HgScanQuery scanQuery, long pageSize) { + + ScanQueryRequest.Builder qb = ScanQueryRequest.newBuilder(); + ScanCondition.Builder cb = ScanCondition.newBuilder(); + + qb.setLimit(getLimit(scanQuery.getLimit())); + qb.setPerKeyLimit(getLimit(scanQuery.getPerKeyLimit())); + qb.setPerKeyMax(getLimit(scanQuery.getPerKeyMax())); + + switch (scanQuery.getScanMethod()) { + case ALL: + qb.setMethod(ScanMethod.ALL); + break; + case PREFIX: + qb.setMethod(ScanMethod.PREFIX); + addPrefixCondition(scanQuery, qb, cb); + break; + case RANGE: + qb.setMethod(ScanMethod.RANGE); + addRangeCondition(scanQuery, qb, cb); + break; + default: + throw new RuntimeException("Unsupported ScanType: " + scanQuery.getScanMethod()); + } + + qb.setTable(scanQuery.getTable()); + qb.setPageSize(pageSize); + qb.setQuery(toBs(scanQuery.getQuery())); + qb.setScanType(scanQuery.getScanType()); + qb.setOrderType(scanQuery.getOrderType()); + qb.setSkipDegree(scanQuery.getSkipDegree()); + + return qb.build(); + } + + static long getLimit(long limit) { + return limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE : limit; + } + + static Header getHeader(HgStoreNodeSession nodeSession) { + return Header.newBuilder().setGraph(nodeSession.getGraphName()).build(); + } + + static void addPrefixCondition(HgScanQuery scanQuery, ScanQueryRequest.Builder qb, + ScanCondition.Builder cb) { + List prefixList = scanQuery.getPrefixList(); + + if (prefixList == null || prefixList.isEmpty()) { + throw new RuntimeException( + "The start-list of ScanQuery shouldn't to be invalid in ScanMethod.PREFIX " + + "mode."); + } + + prefixList.forEach((e) -> { + qb.addCondition(cb.clear() + .setPrefix(toBs(e.getKey())) + .setCode(e.getKeyCode()) + .setSerialNo(e.getSerialNo()) + .build() + ); + }); + + } + + static void addRangeCondition(HgScanQuery scanQuery, ScanQueryRequest.Builder qb, + ScanCondition.Builder cb) { + List startList = scanQuery.getStartList(); + List endList = scanQuery.getEndList(); + + if (startList == null || startList.isEmpty()) { + throw new RuntimeException( + "The start-list of ScanQuery shouldn't to be invalid in ScanMethod.RANGE mode" + + "."); + } + + if (endList == null || endList.isEmpty()) { + throw new RuntimeException( + "The end-list of ScanQuery shouldn't to be invalid in ScanMethod.RANGE mode."); + } + + if (startList.size() != endList.size()) { + throw new RuntimeException("The size of start-list not equals end-list's."); + } + + for (int i = 0, s = startList.size(); i < s; i++) { + HgOwnerKey start = startList.get(i); + HgOwnerKey end = endList.get(i); + qb.addCondition(cb.clear().setCode(start.getKeyCode()) + .setStart(toBs(start.getKey())) + .setEnd(toBs(end.getKey())) + .setSerialNo(start.getSerialNo()) + .build() + ); + } + + } + + static HgOwnerKey toOk(HgOwnerKey key) { + return key == null ? HgStoreClientConst.EMPTY_OWNER_KEY : key; + } + + static ByteString toBs(byte[] bytes) { + return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES); + } + } \ No newline at end of file diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvCloseableIterator.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvCloseableIterator.java index d70879831..fd20d7bb0 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvCloseableIterator.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvCloseableIterator.java @@ -1,28 +1,28 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client.grpc; - -import java.io.Closeable; -import java.util.Iterator; - -/** - * @author lynn.bond@hotmail.com on 2022/3/16 - */ -public interface KvCloseableIterator extends Iterator, Closeable { - void close(); -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.io.Closeable; +import java.util.Iterator; + +/** + * @author lynn.bond@hotmail.com on 2022/3/16 + */ +public interface KvCloseableIterator extends Iterator, Closeable { + void close(); +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvListIterator.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvListIterator.java index 3ef8be0f7..943bd3924 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvListIterator.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvListIterator.java @@ -1,48 +1,48 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client.grpc; - -import java.util.Iterator; -import java.util.List; - -/** - * @author lynn.bond@hotmail.com on 2022/4/6 - */ -class KvListIterator implements KvCloseableIterator { - - private final Iterator iterator; - - KvListIterator(List list) { - this.iterator = list.iterator(); - } - - @Override - public void close() { - /*Nothing to do.*/ - } - - @Override - public boolean hasNext() { - return this.iterator.hasNext(); - } - - @Override - public T next() { - return this.iterator.next(); - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.util.Iterator; +import java.util.List; + +/** + * @author lynn.bond@hotmail.com on 2022/4/6 + */ +class KvListIterator implements KvCloseableIterator { + + private final Iterator iterator; + + KvListIterator(List list) { + this.iterator = list.iterator(); + } + + @Override + public void close() { + /*Nothing to do.*/ + } + + @Override + public boolean hasNext() { + return this.iterator.hasNext(); + } + + @Override + public T next() { + return this.iterator.next(); + } +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvOneShotScanner.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvOneShotScanner.java index b78d20815..02a416590 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvOneShotScanner.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/KvOneShotScanner.java @@ -1,213 +1,213 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client.grpc; - -import java.util.Iterator; -import java.util.List; - -import javax.annotation.concurrent.NotThreadSafe; - -import org.apache.hugegraph.store.HgKvStore; -import org.apache.hugegraph.store.HgOwnerKey; -import org.apache.hugegraph.store.HgPageSize; -import org.apache.hugegraph.store.HgSeekAble; -import org.apache.hugegraph.store.client.HgStoreNodeSession; -import org.apache.hugegraph.store.client.util.HgStoreClientConfig; -import org.apache.hugegraph.store.client.util.HgStoreClientConst; -import org.apache.hugegraph.store.client.util.HgStoreClientUtil; -import org.apache.hugegraph.store.grpc.common.Header; -import org.apache.hugegraph.store.grpc.common.Kv; -import org.apache.hugegraph.store.grpc.common.ScanMethod; -import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamBlockingStub; -import org.apache.hugegraph.store.grpc.stream.ScanStreamReq; - -import com.google.protobuf.ByteString; - -import lombok.extern.slf4j.Slf4j; - -/** - * @author lynn.bond@hotmail.com created on 2021/12/1 - */ -@Slf4j -@NotThreadSafe -class KvOneShotScanner implements KvCloseableIterator, HgPageSize, HgSeekAble { - private static final HgStoreClientConfig storeClientConfig = HgStoreClientConfig.of(); - private final HgStoreNodeSession session; - private final HgStoreStreamBlockingStub stub; - private final ScanStreamReq.Builder reqBuilder = ScanStreamReq.newBuilder(); - private ScanStreamReq req; - private Iterator iterator; - private List list = null; - private final String table; - private final HgOwnerKey startKey; - private final HgOwnerKey endKey; - private final HgOwnerKey prefix; - private final ScanMethod scanMethod; - private final long limit; - private final int partition; - private final int scanType; - private final byte[] query; - private final int pageSize; - private boolean in = true; - private byte[] nodePosition = HgStoreClientConst.EMPTY_BYTES; - - public static KvCloseableIterator scanAll(HgStoreNodeSession session, - HgStoreStreamBlockingStub stub, - String table, long limit, byte[] query) { - return new KvOneShotScanner(ScanMethod.ALL, session, stub, table, null, null, null, limit, - -1, HgKvStore.SCAN_ANY, - query); - } - - public static KvCloseableIterator scanPrefix(HgStoreNodeSession session, - HgStoreStreamBlockingStub stub, - String table, HgOwnerKey prefix, long limit, - byte[] query) { - return new KvOneShotScanner(ScanMethod.PREFIX, session, stub, table, prefix, null, null, - limit, - prefix.getKeyCode(), HgKvStore.SCAN_PREFIX_BEGIN, query); - } - - public static KvCloseableIterator scanRange(HgStoreNodeSession nodeSession, - HgStoreStreamBlockingStub stub, - String table, HgOwnerKey startKey, - HgOwnerKey endKey, long limit, - int scanType, byte[] query) { - return new KvOneShotScanner(ScanMethod.RANGE, nodeSession, stub, table, null, startKey, - endKey, limit, - startKey.getKeyCode(), scanType, query); - } - - - private KvOneShotScanner(ScanMethod scanMethod, HgStoreNodeSession session, - HgStoreStreamBlockingStub stub, - String table, HgOwnerKey prefix, HgOwnerKey startKey, - HgOwnerKey endKey, long limit, - int partition, int scanType, byte[] query) { - this.scanMethod = scanMethod; - this.session = session; - this.stub = stub; - this.table = table; - this.startKey = toOk(startKey); - this.endKey = toOk(endKey); - this.prefix = toOk(prefix); - this.partition = partition; - this.scanType = scanType; - this.query = query != null ? query : HgStoreClientConst.EMPTY_BYTES; - this.limit = limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE : - limit; // <=0 means no limit - this.pageSize = storeClientConfig.getNetKvScannerPageSize(); - - } - - private Header getHeader(HgStoreNodeSession nodeSession) { - return Header.newBuilder().setGraph(nodeSession.getGraphName()).build(); - } - - private void createReq() { - this.req = this.reqBuilder - .setHeader(this.getHeader(this.session)) - .setMethod(this.scanMethod) - .setTable(this.table) - .setStart(toBs(this.startKey.getKey())) - .setEnd(toBs(this.endKey.getKey())) - .setLimit(this.limit) - .setPrefix(toBs(this.prefix.getKey())) - .setCode(this.partition) - .setScanType(this.scanType) - .setQuery(toBs(this.query)) - .setPageSize(pageSize) - .setPosition(toBs(this.nodePosition)) - .build(); - } - - private void init() { - - if (this.iterator == null) { - this.createReq(); - this.list = this.stub.scanOneShot(this.req).getDataList(); - this.iterator = this.list.iterator(); - } - - } - - @Override - public boolean hasNext() { - if (!this.in) { - return false; - } - if (this.iterator == null) { - this.init(); - } - return this.iterator.hasNext(); - } - - @Override - public Kv next() { - if (this.iterator == null) { - this.init(); - } - return this.iterator.next(); - } - - @Override - public long getPageSize() { - return this.limit; - } - - @Override - public boolean isPageEmpty() { - return !this.iterator.hasNext(); - } - - @Override - public byte[] position() { - return HgStoreClientUtil.toBytes(this.session.getStoreNode().getNodeId().longValue()); - } - - @Override - public void seek(byte[] position) { - if (position == null || position.length < Long.BYTES) { - return; - } - byte[] nodeIdBytes = new byte[Long.BYTES]; - System.arraycopy(position, 0, nodeIdBytes, 0, Long.BYTES); - long nodeId = this.session.getStoreNode().getNodeId().longValue(); - long pId = HgStoreClientUtil.toLong(nodeIdBytes); - this.in = nodeId >= pId; - if (this.in && nodeId == pId) { - this.nodePosition = new byte[position.length - Long.BYTES]; - System.arraycopy(position, Long.BYTES, this.nodePosition, 0, this.nodePosition.length); - } else { - this.nodePosition = HgStoreClientConst.EMPTY_BYTES; - } - } - - static HgOwnerKey toOk(HgOwnerKey key) { - return key == null ? HgStoreClientConst.EMPTY_OWNER_KEY : key; - } - - static ByteString toBs(byte[] bytes) { - return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES); - } - - @Override - public void close() { - //TODO: implements - } +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.util.Iterator; +import java.util.List; + +import javax.annotation.concurrent.NotThreadSafe; + +import org.apache.hugegraph.store.HgKvStore; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgPageSize; +import org.apache.hugegraph.store.HgSeekAble; +import org.apache.hugegraph.store.client.HgStoreNodeSession; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.HgStoreClientUtil; +import org.apache.hugegraph.store.grpc.common.Header; +import org.apache.hugegraph.store.grpc.common.Kv; +import org.apache.hugegraph.store.grpc.common.ScanMethod; +import org.apache.hugegraph.store.grpc.stream.HgStoreStreamGrpc.HgStoreStreamBlockingStub; +import org.apache.hugegraph.store.grpc.stream.ScanStreamReq; + +import com.google.protobuf.ByteString; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author lynn.bond@hotmail.com created on 2021/12/1 + */ +@Slf4j +@NotThreadSafe +class KvOneShotScanner implements KvCloseableIterator, HgPageSize, HgSeekAble { + private static final HgStoreClientConfig storeClientConfig = HgStoreClientConfig.of(); + private final HgStoreNodeSession session; + private final HgStoreStreamBlockingStub stub; + private final ScanStreamReq.Builder reqBuilder = ScanStreamReq.newBuilder(); + private ScanStreamReq req; + private Iterator iterator; + private List list = null; + private final String table; + private final HgOwnerKey startKey; + private final HgOwnerKey endKey; + private final HgOwnerKey prefix; + private final ScanMethod scanMethod; + private final long limit; + private final int partition; + private final int scanType; + private final byte[] query; + private final int pageSize; + private boolean in = true; + private byte[] nodePosition = HgStoreClientConst.EMPTY_BYTES; + + public static KvCloseableIterator scanAll(HgStoreNodeSession session, + HgStoreStreamBlockingStub stub, + String table, long limit, byte[] query) { + return new KvOneShotScanner(ScanMethod.ALL, session, stub, table, null, null, null, limit, + -1, HgKvStore.SCAN_ANY, + query); + } + + public static KvCloseableIterator scanPrefix(HgStoreNodeSession session, + HgStoreStreamBlockingStub stub, + String table, HgOwnerKey prefix, long limit, + byte[] query) { + return new KvOneShotScanner(ScanMethod.PREFIX, session, stub, table, prefix, null, null, + limit, + prefix.getKeyCode(), HgKvStore.SCAN_PREFIX_BEGIN, query); + } + + public static KvCloseableIterator scanRange(HgStoreNodeSession nodeSession, + HgStoreStreamBlockingStub stub, + String table, HgOwnerKey startKey, + HgOwnerKey endKey, long limit, + int scanType, byte[] query) { + return new KvOneShotScanner(ScanMethod.RANGE, nodeSession, stub, table, null, startKey, + endKey, limit, + startKey.getKeyCode(), scanType, query); + } + + + private KvOneShotScanner(ScanMethod scanMethod, HgStoreNodeSession session, + HgStoreStreamBlockingStub stub, + String table, HgOwnerKey prefix, HgOwnerKey startKey, + HgOwnerKey endKey, long limit, + int partition, int scanType, byte[] query) { + this.scanMethod = scanMethod; + this.session = session; + this.stub = stub; + this.table = table; + this.startKey = toOk(startKey); + this.endKey = toOk(endKey); + this.prefix = toOk(prefix); + this.partition = partition; + this.scanType = scanType; + this.query = query != null ? query : HgStoreClientConst.EMPTY_BYTES; + this.limit = limit <= HgStoreClientConst.NO_LIMIT ? Integer.MAX_VALUE : + limit; // <=0 means no limit + this.pageSize = storeClientConfig.getNetKvScannerPageSize(); + + } + + private Header getHeader(HgStoreNodeSession nodeSession) { + return Header.newBuilder().setGraph(nodeSession.getGraphName()).build(); + } + + private void createReq() { + this.req = this.reqBuilder + .setHeader(this.getHeader(this.session)) + .setMethod(this.scanMethod) + .setTable(this.table) + .setStart(toBs(this.startKey.getKey())) + .setEnd(toBs(this.endKey.getKey())) + .setLimit(this.limit) + .setPrefix(toBs(this.prefix.getKey())) + .setCode(this.partition) + .setScanType(this.scanType) + .setQuery(toBs(this.query)) + .setPageSize(pageSize) + .setPosition(toBs(this.nodePosition)) + .build(); + } + + private void init() { + + if (this.iterator == null) { + this.createReq(); + this.list = this.stub.scanOneShot(this.req).getDataList(); + this.iterator = this.list.iterator(); + } + + } + + @Override + public boolean hasNext() { + if (!this.in) { + return false; + } + if (this.iterator == null) { + this.init(); + } + return this.iterator.hasNext(); + } + + @Override + public Kv next() { + if (this.iterator == null) { + this.init(); + } + return this.iterator.next(); + } + + @Override + public long getPageSize() { + return this.limit; + } + + @Override + public boolean isPageEmpty() { + return !this.iterator.hasNext(); + } + + @Override + public byte[] position() { + return HgStoreClientUtil.toBytes(this.session.getStoreNode().getNodeId().longValue()); + } + + @Override + public void seek(byte[] position) { + if (position == null || position.length < Long.BYTES) { + return; + } + byte[] nodeIdBytes = new byte[Long.BYTES]; + System.arraycopy(position, 0, nodeIdBytes, 0, Long.BYTES); + long nodeId = this.session.getStoreNode().getNodeId().longValue(); + long pId = HgStoreClientUtil.toLong(nodeIdBytes); + this.in = nodeId >= pId; + if (this.in && nodeId == pId) { + this.nodePosition = new byte[position.length - Long.BYTES]; + System.arraycopy(position, Long.BYTES, this.nodePosition, 0, this.nodePosition.length); + } else { + this.nodePosition = HgStoreClientConst.EMPTY_BYTES; + } + } + + static HgOwnerKey toOk(HgOwnerKey key) { + return key == null ? HgStoreClientConst.EMPTY_OWNER_KEY : key; + } + + static ByteString toBs(byte[] bytes) { + return ByteString.copyFrom((bytes != null) ? bytes : HgStoreClientConst.EMPTY_BYTES); + } + + @Override + public void close() { + //TODO: implements + } } \ No newline at end of file diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/SeekAbleIterator.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/SeekAbleIterator.java index 39b5b7c21..a0aef416e 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/SeekAbleIterator.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/SeekAbleIterator.java @@ -1,69 +1,69 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client.grpc; - -import java.util.Iterator; -import java.util.function.Consumer; -import java.util.function.Supplier; - -import org.apache.hugegraph.store.HgSeekAble; -import org.apache.hugegraph.store.client.util.HgAssert; - -/** - * @author lynn.bond@hotmail.com on 2022/3/11 - */ -class SeekAbleIterator implements Iterator, HgSeekAble { - private final Iterator iterator; - private final Consumer seeker; - private final Supplier positioner; - - public static SeekAbleIterator of(Iterator iterator, Supplier positioner, - Consumer seeker) { - HgAssert.isArgumentNotNull(iterator, "iterator"); - HgAssert.isArgumentNotNull(positioner, "positioner"); - HgAssert.isArgumentNotNull(seeker, "seeker"); - return new SeekAbleIterator(iterator, positioner, seeker); - } - - private SeekAbleIterator(Iterator iterator, Supplier positioner, - Consumer seeker) { - this.iterator = iterator; - this.positioner = positioner; - this.seeker = seeker; - } - - @Override - public byte[] position() { - return this.positioner.get(); - } - - @Override - public void seek(byte[] position) { - this.seeker.accept(position); - } - - @Override - public boolean hasNext() { - return this.iterator.hasNext(); - } - - @Override - public E next() { - return this.iterator.next(); - } +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.grpc; + +import java.util.Iterator; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import org.apache.hugegraph.store.HgSeekAble; +import org.apache.hugegraph.store.client.util.HgAssert; + +/** + * @author lynn.bond@hotmail.com on 2022/3/11 + */ +class SeekAbleIterator implements Iterator, HgSeekAble { + private final Iterator iterator; + private final Consumer seeker; + private final Supplier positioner; + + public static SeekAbleIterator of(Iterator iterator, Supplier positioner, + Consumer seeker) { + HgAssert.isArgumentNotNull(iterator, "iterator"); + HgAssert.isArgumentNotNull(positioner, "positioner"); + HgAssert.isArgumentNotNull(seeker, "seeker"); + return new SeekAbleIterator(iterator, positioner, seeker); + } + + private SeekAbleIterator(Iterator iterator, Supplier positioner, + Consumer seeker) { + this.iterator = iterator; + this.positioner = positioner; + this.seeker = seeker; + } + + @Override + public byte[] position() { + return this.positioner.get(); + } + + @Override + public void seek(byte[] position) { + this.seeker.accept(position); + } + + @Override + public boolean hasNext() { + return this.iterator.hasNext(); + } + + @Override + public E next() { + return this.iterator.next(); + } } \ No newline at end of file diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/Base58.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/Base58.java index c9ec738bb..e332b40cd 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/Base58.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/Base58.java @@ -1,177 +1,177 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client.util; - -import java.io.UnsupportedEncodingException; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; - -/** - * Created with IntelliJ IDEA. - * User: noah - * Date: 8/2/13 - * Time: 10:36 AM - * To change this template use File | Settings | File Templates. - */ -public class Base58 { - - public static final char[] ALPHABET = - "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray(); - private static final int[] INDEXES = new int[128]; - - static { - for (int i = 0; i < INDEXES.length; i++) { - INDEXES[i] = -1; - } - for (int i = 0; i < ALPHABET.length; i++) { - INDEXES[ALPHABET[i]] = i; - } - } - - /** - * Encodes the given bytes in base58. No checksum is appended. - */ - public static String encode(byte[] input) { - if (input.length == 0) { - return ""; - } - input = copyOfRange(input, 0, input.length); - // Count leading zeroes. - int zeroCount = 0; - while (zeroCount < input.length && input[zeroCount] == 0) { - ++zeroCount; - } - // The actual encoding. - byte[] temp = new byte[input.length * 2]; - int j = temp.length; - - int startAt = zeroCount; - while (startAt < input.length) { - byte mod = divmod58(input, startAt); - if (input[startAt] == 0) { - ++startAt; - } - temp[--j] = (byte) ALPHABET[mod]; - } - - // Strip extra '1' if there are some after decoding. - while (j < temp.length && temp[j] == ALPHABET[0]) { - ++j; - } - // Add as many leading '1' as there were leading zeros. - while (--zeroCount >= 0) { - temp[--j] = (byte) ALPHABET[0]; - } - - byte[] output = copyOfRange(temp, j, temp.length); - return new String(output, StandardCharsets.US_ASCII); - } - - public static byte[] decode(String input) throws IllegalArgumentException { - if (input.length() == 0) { - return new byte[0]; - } - byte[] input58 = new byte[input.length()]; - // Transform the String to a base58 byte sequence - for (int i = 0; i < input.length(); ++i) { - char c = input.charAt(i); - - int digit58 = -1; - if (c >= 0 && c < 128) { - digit58 = INDEXES[c]; - } - if (digit58 < 0) { - throw new IllegalArgumentException("Illegal character " + c + " at " + i); - } - - input58[i] = (byte) digit58; - } - // Count leading zeroes - int zeroCount = 0; - while (zeroCount < input58.length && input58[zeroCount] == 0) { - ++zeroCount; - } - // The encoding - byte[] temp = new byte[input.length()]; - int j = temp.length; - - int startAt = zeroCount; - while (startAt < input58.length) { - byte mod = divmod256(input58, startAt); - if (input58[startAt] == 0) { - ++startAt; - } - - temp[--j] = mod; - } - // Do no add extra leading zeroes, move j to first non null byte. - while (j < temp.length && temp[j] == 0) { - ++j; - } - - return copyOfRange(temp, j - zeroCount, temp.length); - } - - public static BigInteger decodeToBigInteger(String input) throws IllegalArgumentException { - return new BigInteger(1, decode(input)); - } - - // - // number -> number / 58, returns number % 58 - // - private static byte divmod58(byte[] number, int startAt) { - int remainder = 0; - for (int i = startAt; i < number.length; i++) { - int digit256 = (int) number[i] & 0xFF; - int temp = remainder * 256 + digit256; - - number[i] = (byte) (temp / 58); - - remainder = temp % 58; - } - - return (byte) remainder; - } - - // - // number -> number / 256, returns number % 256 - // - private static byte divmod256(byte[] number58, int startAt) { - int remainder = 0; - for (int i = startAt; i < number58.length; i++) { - int digit58 = (int) number58[i] & 0xFF; - int temp = remainder * 58 + digit58; - - number58[i] = (byte) (temp / 256); - - remainder = temp % 256; - } - - return (byte) remainder; - } - - private static byte[] copyOfRange(byte[] source, int from, int to) { - byte[] range = new byte[to - from]; - System.arraycopy(source, from, range, 0, range.length); - - return range; - } - - -} - +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.util; + +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; + +/** + * Created with IntelliJ IDEA. + * User: noah + * Date: 8/2/13 + * Time: 10:36 AM + * To change this template use File | Settings | File Templates. + */ +public class Base58 { + + public static final char[] ALPHABET = + "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray(); + private static final int[] INDEXES = new int[128]; + + static { + for (int i = 0; i < INDEXES.length; i++) { + INDEXES[i] = -1; + } + for (int i = 0; i < ALPHABET.length; i++) { + INDEXES[ALPHABET[i]] = i; + } + } + + /** + * Encodes the given bytes in base58. No checksum is appended. + */ + public static String encode(byte[] input) { + if (input.length == 0) { + return ""; + } + input = copyOfRange(input, 0, input.length); + // Count leading zeroes. + int zeroCount = 0; + while (zeroCount < input.length && input[zeroCount] == 0) { + ++zeroCount; + } + // The actual encoding. + byte[] temp = new byte[input.length * 2]; + int j = temp.length; + + int startAt = zeroCount; + while (startAt < input.length) { + byte mod = divmod58(input, startAt); + if (input[startAt] == 0) { + ++startAt; + } + temp[--j] = (byte) ALPHABET[mod]; + } + + // Strip extra '1' if there are some after decoding. + while (j < temp.length && temp[j] == ALPHABET[0]) { + ++j; + } + // Add as many leading '1' as there were leading zeros. + while (--zeroCount >= 0) { + temp[--j] = (byte) ALPHABET[0]; + } + + byte[] output = copyOfRange(temp, j, temp.length); + return new String(output, StandardCharsets.US_ASCII); + } + + public static byte[] decode(String input) throws IllegalArgumentException { + if (input.length() == 0) { + return new byte[0]; + } + byte[] input58 = new byte[input.length()]; + // Transform the String to a base58 byte sequence + for (int i = 0; i < input.length(); ++i) { + char c = input.charAt(i); + + int digit58 = -1; + if (c >= 0 && c < 128) { + digit58 = INDEXES[c]; + } + if (digit58 < 0) { + throw new IllegalArgumentException("Illegal character " + c + " at " + i); + } + + input58[i] = (byte) digit58; + } + // Count leading zeroes + int zeroCount = 0; + while (zeroCount < input58.length && input58[zeroCount] == 0) { + ++zeroCount; + } + // The encoding + byte[] temp = new byte[input.length()]; + int j = temp.length; + + int startAt = zeroCount; + while (startAt < input58.length) { + byte mod = divmod256(input58, startAt); + if (input58[startAt] == 0) { + ++startAt; + } + + temp[--j] = mod; + } + // Do no add extra leading zeroes, move j to first non null byte. + while (j < temp.length && temp[j] == 0) { + ++j; + } + + return copyOfRange(temp, j - zeroCount, temp.length); + } + + public static BigInteger decodeToBigInteger(String input) throws IllegalArgumentException { + return new BigInteger(1, decode(input)); + } + + // + // number -> number / 58, returns number % 58 + // + private static byte divmod58(byte[] number, int startAt) { + int remainder = 0; + for (int i = startAt; i < number.length; i++) { + int digit256 = (int) number[i] & 0xFF; + int temp = remainder * 256 + digit256; + + number[i] = (byte) (temp / 58); + + remainder = temp % 58; + } + + return (byte) remainder; + } + + // + // number -> number / 256, returns number % 256 + // + private static byte divmod256(byte[] number58, int startAt) { + int remainder = 0; + for (int i = startAt; i < number58.length; i++) { + int digit58 = (int) number58[i] & 0xFF; + int temp = remainder * 58 + digit58; + + number58[i] = (byte) (temp / 256); + + remainder = temp % 256; + } + + return (byte) remainder; + } + + private static byte[] copyOfRange(byte[] source, int from, int to) { + byte[] range = new byte[to - from]; + System.arraycopy(source, from, range, 0, range.length); + + return range; + } + + +} + diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgBufferProxy.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgBufferProxy.java index 7231f0548..09dc4d1e8 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgBufferProxy.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgBufferProxy.java @@ -1,160 +1,160 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client.util; - -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; -import java.util.function.Supplier; - -import javax.annotation.CheckForNull; - -import org.apache.hugegraph.store.client.type.HgStoreClientException; - -import lombok.extern.slf4j.Slf4j; - -/** - * @author lynn.bond@hotmail.com on 2022/3/15 - * @version 0.1.0 - */ -@Slf4j -public final class HgBufferProxy { - private final BlockingQueue> queue; - private final AtomicBoolean closed = new AtomicBoolean(false); - private final ReentrantLock lock = new ReentrantLock(); - private final Runnable task; - private Throwable err; - - public static HgBufferProxy of(Runnable task) { - HgAssert.isArgumentNotNull(task, "task"); - return new HgBufferProxy(task); - } - - private HgBufferProxy(Runnable task) { - this.task = task; - this.queue = new LinkedBlockingQueue<>(); - } - - public void send(T t) { - if (t == null) { - throw new IllegalArgumentException("the argument t is null"); - } - if (this.closed.get()) { - return; - } - this.lock.lock(); - try { - this.queue.offer(() -> t); - } finally { - lock.unlock(); - } - } - - private void apply() { - this.lock.lock(); - try { - if (!this.closed.get()) { - this.task.run(); - Thread.yield(); - } - } finally { - this.lock.unlock(); - } - } - - /** - * return an item from the chan - * - * @return null when the chan has been closed - * @throws RuntimeException - */ - @CheckForNull - public T receive(int time, Consumer callback) { - Supplier s; - if (this.closed.get()) { - s = this.queue.poll(); - this.checkErr(); - return s != null ? s.get() : null; - } - if (this.queue.size() <= 1) { - this.apply(); - } - lock.lock(); - try { - if (this.isClosed()) { - s = this.queue.poll(); - this.checkErr(); - return s != null ? s.get() : null; - } - } finally { - lock.unlock(); - } - try { - s = this.queue.poll(time, TimeUnit.SECONDS); - } catch (Throwable t) { - log.error("failed to receive a item from channel, cause by: ", t); - throw HgStoreClientException.of(t); - } - if (s == null) { - if (this.closed.get()) { - s = this.queue.poll(); - } else { - if (callback == null) { - throw new RuntimeException("timeout, max time: " + time + " seconds"); - } else { - callback.accept(time); - } - } - } - this.checkErr(); - return s != null ? s.get() : null; - } - - public boolean isClosed() { - return this.closed.get(); - } - - /** - * @throws RuntimeException when fail to close the chan - */ - public void close() { - if (this.closed.get()) { - return; - } - lock.lock(); - this.closed.set(true); - try { - this.queue.offer(() -> null); - } finally { - lock.unlock(); - } - } - - public void setError(Throwable streamErr) { - this.err = streamErr; - } - - private void checkErr() { - if (this.err != null) { - throw HgStoreClientException.of(this.err); - } - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.util; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import javax.annotation.CheckForNull; + +import org.apache.hugegraph.store.client.type.HgStoreClientException; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author lynn.bond@hotmail.com on 2022/3/15 + * @version 0.1.0 + */ +@Slf4j +public final class HgBufferProxy { + private final BlockingQueue> queue; + private final AtomicBoolean closed = new AtomicBoolean(false); + private final ReentrantLock lock = new ReentrantLock(); + private final Runnable task; + private Throwable err; + + public static HgBufferProxy of(Runnable task) { + HgAssert.isArgumentNotNull(task, "task"); + return new HgBufferProxy(task); + } + + private HgBufferProxy(Runnable task) { + this.task = task; + this.queue = new LinkedBlockingQueue<>(); + } + + public void send(T t) { + if (t == null) { + throw new IllegalArgumentException("the argument t is null"); + } + if (this.closed.get()) { + return; + } + this.lock.lock(); + try { + this.queue.offer(() -> t); + } finally { + lock.unlock(); + } + } + + private void apply() { + this.lock.lock(); + try { + if (!this.closed.get()) { + this.task.run(); + Thread.yield(); + } + } finally { + this.lock.unlock(); + } + } + + /** + * return an item from the chan + * + * @return null when the chan has been closed + * @throws RuntimeException + */ + @CheckForNull + public T receive(int time, Consumer callback) { + Supplier s; + if (this.closed.get()) { + s = this.queue.poll(); + this.checkErr(); + return s != null ? s.get() : null; + } + if (this.queue.size() <= 1) { + this.apply(); + } + lock.lock(); + try { + if (this.isClosed()) { + s = this.queue.poll(); + this.checkErr(); + return s != null ? s.get() : null; + } + } finally { + lock.unlock(); + } + try { + s = this.queue.poll(time, TimeUnit.SECONDS); + } catch (Throwable t) { + log.error("failed to receive a item from channel, cause by: ", t); + throw HgStoreClientException.of(t); + } + if (s == null) { + if (this.closed.get()) { + s = this.queue.poll(); + } else { + if (callback == null) { + throw new RuntimeException("timeout, max time: " + time + " seconds"); + } else { + callback.accept(time); + } + } + } + this.checkErr(); + return s != null ? s.get() : null; + } + + public boolean isClosed() { + return this.closed.get(); + } + + /** + * @throws RuntimeException when fail to close the chan + */ + public void close() { + if (this.closed.get()) { + return; + } + lock.lock(); + this.closed.set(true); + try { + this.queue.offer(() -> null); + } finally { + lock.unlock(); + } + } + + public void setError(Throwable streamErr) { + this.err = streamErr; + } + + private void checkErr() { + if (this.err != null) { + throw HgStoreClientException.of(this.err); + } + } +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgUuid.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgUuid.java index 76c77ca53..79c35670d 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgUuid.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/HgUuid.java @@ -1,44 +1,44 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client.util; - -import java.nio.ByteBuffer; -import java.util.UUID; - -/** - * @author lynn.bond@hotmail.com - */ -public final class HgUuid { - - private static String encode(UUID uuid) { - ByteBuffer bb = ByteBuffer.wrap(new byte[16]); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - return Base58.encode(bb.array()); - } - - /** - * Get a UUID in Base58 FORM - * - * @return - */ - public static String newUUID() { - return encode(UUID.randomUUID()); - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.util; + +import java.nio.ByteBuffer; +import java.util.UUID; + +/** + * @author lynn.bond@hotmail.com + */ +public final class HgUuid { + + private static String encode(UUID uuid) { + ByteBuffer bb = ByteBuffer.wrap(new byte[16]); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + return Base58.encode(bb.array()); + } + + /** + * Get a UUID in Base58 FORM + * + * @return + */ + public static String newUUID() { + return encode(UUID.randomUUID()); + } + +} diff --git a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/MetricX.java b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/MetricX.java index 8f8c6809e..027961ec1 100644 --- a/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/MetricX.java +++ b/hg-store-client/src/main/java/org/apache/hugegraph/store/client/util/MetricX.java @@ -1,103 +1,103 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.client.util; - -import java.util.concurrent.atomic.AtomicLong; - -/** - * @author lynn.bond@hotmail.com on 2022/1/29 - */ -public class MetricX { - public static AtomicLong iteratorSum = new AtomicLong(); - public static AtomicLong iteratorCount = new AtomicLong(); - public static AtomicLong iteratorMax = new AtomicLong(); - - public AtomicLong failureCount = new AtomicLong(); - - private long start; - private long end; - - public static MetricX ofStart() { - return new MetricX(System.currentTimeMillis()); - } - - private MetricX(long start) { - this.start = start; - } - - public long start() { - return this.start = System.currentTimeMillis(); - } - - public long end() { - return this.end = System.currentTimeMillis(); - } - - public long past() { - return this.end - this.start; - } - - public static void plusIteratorWait(long nanoSeconds) { - iteratorSum.addAndGet(nanoSeconds); - iteratorCount.getAndIncrement(); - if (iteratorMax.get() < nanoSeconds) { - iteratorMax.set(nanoSeconds); - } - } - - /** - * amount of waiting - * - * @return millisecond - */ - public static long getIteratorWait() { - return iteratorSum.get() / 1_000_000; - } - - /** - * average of waiting - * - * @return millisecond - */ - public static long getIteratorWaitAvg() { - if (iteratorCount.get() == 0) return -1; - return getIteratorWait() / iteratorCount.get(); - } - - /** - * maximum of waiting - * - * @return millisecond - */ - public static long getIteratorWaitMax() { - return iteratorMax.get() / 1_000_000; - } - - public static long getIteratorCount() { - return iteratorCount.get(); - } - - public void countFail() { - this.failureCount.getAndIncrement(); - } - - public long getFailureCount() { - return this.failureCount.get(); - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.client.util; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author lynn.bond@hotmail.com on 2022/1/29 + */ +public class MetricX { + public static AtomicLong iteratorSum = new AtomicLong(); + public static AtomicLong iteratorCount = new AtomicLong(); + public static AtomicLong iteratorMax = new AtomicLong(); + + public AtomicLong failureCount = new AtomicLong(); + + private long start; + private long end; + + public static MetricX ofStart() { + return new MetricX(System.currentTimeMillis()); + } + + private MetricX(long start) { + this.start = start; + } + + public long start() { + return this.start = System.currentTimeMillis(); + } + + public long end() { + return this.end = System.currentTimeMillis(); + } + + public long past() { + return this.end - this.start; + } + + public static void plusIteratorWait(long nanoSeconds) { + iteratorSum.addAndGet(nanoSeconds); + iteratorCount.getAndIncrement(); + if (iteratorMax.get() < nanoSeconds) { + iteratorMax.set(nanoSeconds); + } + } + + /** + * amount of waiting + * + * @return millisecond + */ + public static long getIteratorWait() { + return iteratorSum.get() / 1_000_000; + } + + /** + * average of waiting + * + * @return millisecond + */ + public static long getIteratorWaitAvg() { + if (iteratorCount.get() == 0) return -1; + return getIteratorWait() / iteratorCount.get(); + } + + /** + * maximum of waiting + * + * @return millisecond + */ + public static long getIteratorWaitMax() { + return iteratorMax.get() / 1_000_000; + } + + public static long getIteratorCount() { + return iteratorCount.get(); + } + + public void countFail() { + this.failureCount.getAndIncrement(); + } + + public long getFailureCount() { + return this.failureCount.get(); + } + +} diff --git a/hg-store-client/src/main/resources/log4j2.xml b/hg-store-client/src/main/resources/log4j2.xml index fab1b79c1..54096d41f 100644 --- a/hg-store-client/src/main/resources/log4j2.xml +++ b/hg-store-client/src/main/resources/log4j2.xml @@ -1,102 +1,102 @@ - - - - - - - - logs - hg-store-client - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + logs + hg-store-client + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerTest.java b/hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerTest.java index af0961a4e..1ba678ea7 100644 --- a/hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerTest.java +++ b/hg-store-client/src/test/java/org/apache/hugegraph/store/HgSessionManagerTest.java @@ -1,1125 +1,1125 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store; - -import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; -import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; - -import org.apache.hugegraph.pd.common.PartitionUtils; -import org.apache.hugegraph.store.client.HgStoreNodeManager; -import org.apache.hugegraph.store.client.util.ExecutorPool; -import org.apache.hugegraph.store.client.util.HgStoreClientConfig; -import org.apache.hugegraph.store.client.util.HgStoreClientConst; -import org.apache.hugegraph.store.client.util.MetricX; -import org.apache.hugegraph.store.util.HgStoreTestUtil; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HgSessionManagerTest { - private static final Map leaderMap = new ConcurrentHashMap<>(); - private static final Map storeMap = new ConcurrentHashMap<>(); - - private static final ExecutorService pool = Executors.newFixedThreadPool(100, - ExecutorPool.newThreadFactory( - "unit-test")); - - private static final int partitionCount = 10; - // 需要与store的application.yml的fake-pd.partition-count保持一致 - - //private static String[] storeAddress = {"127.0.0.1:8500"}; - private static final String[] storeAddress = - {"127.0.0.1:8501", "127.0.0.1:8502", "127.0.0.1:8503"}; - - private static final int PARTITION_LENGTH = getPartitionLength(); - - private static int getPartitionLength() { - return PartitionUtils.MAX_VALUE / - (partitionCount == 0 ? storeAddress.length : partitionCount) + 1; - } - - @BeforeClass - public static void init() { - for (String address : storeAddress) { - storeMap.put((long) address.hashCode(), address); - } - for (int i = 0; i < partitionCount; i++) { - leaderMap.put(i, storeMap.keySet().iterator().next()); - } - - HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); - nodeManager.setNodePartitioner((builder, graphName, startKey, endKey) -> { - int startCode = PartitionUtils.calcHashcode(startKey); - int endCode = PartitionUtils.calcHashcode(endKey); - if (ALL_PARTITION_OWNER == startKey) { - storeMap.forEach((k, v) -> { - builder.add(k, -1); - }); - } else if (endKey == HgStoreClientConst.EMPTY_BYTES || startKey == endKey || - Arrays.equals(startKey, endKey)) { - //log.info("leader-> {}",leaderMap.get(startCode / PARTITION_LENGTH)); - builder.add(leaderMap.get(startCode / PARTITION_LENGTH), startCode); - } else { - Assert.fail("OwnerKey转成HashCode后已经无序了， 按照OwnerKey范围查询没意义"); - builder.add(leaderMap.get(startCode / PARTITION_LENGTH), startCode); - builder.add(leaderMap.get(endCode / PARTITION_LENGTH), endCode); - } - return 0; - }); - nodeManager.setNodeProvider((graphName, nodeId) -> { - // System.out.println("HgStoreNodeProvider apply " + graphName + " " + nodeId + - // " " + storeMap.get(nodeId)); - return nodeManager.getNodeBuilder().setNodeId(nodeId) - .setAddress(storeMap.get(nodeId)).build(); - }); - nodeManager.setNodeNotifier((graphName, storeNotice) -> { - System.out.println("recv node notifier " + storeNotice); - if (storeNotice.getPartitionLeaders().size() > 0) { - leaderMap.putAll(storeNotice.getPartitionLeaders()); - System.out.println("leader changed "); - leaderMap.forEach((k, v) -> { - System.out.print(" " + k + " " + v + ","); - }); - System.out.println(); - } - return 0; - }); - } - - protected static HgStoreSession getStoreSession() { - return HgSessionManager.getInstance().openSession(HgStoreTestUtil.GRAPH_NAME); - } - - protected static HgStoreSession getStoreSession(String graph) { - return HgSessionManager.getInstance().openSession(graph); - } - - @Test - public void put_get() { - System.out.println("--- test put & get ---"); - - HgStoreSession session = getStoreSession(); - - HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); - //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); - byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); - - Assert.assertTrue(session.put(HgStoreTestUtil.TABLE_NAME, key, value)); - Assert.assertEquals(HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get( - HgStoreTestUtil.TABLE_NAME, key))); - } - - @Test - public void batchPrefix() { - System.out.println("--- test batchGet ---"); - HgStoreSession session = getStoreSession(); - String keyPrefix = "UNIT-BATCH-GET"; - - Map map = HgStoreTestUtil.batchPut(session, keyPrefix); - List keyList = - map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); - - Map keyMap = new HashMap<>(); - map.entrySet().forEach(e -> { - keyMap.put(Arrays.toString(e.getKey().getKey()), e.getValue()); - }); - - //printOwner(keyList); - HgKvIterator iterator = session.batchPrefix(HgStoreTestUtil.TABLE_NAME, keyList); - int amount = HgStoreTestUtil.println(iterator); - Assert.assertEquals(amount, keyList.size()); -/* - - println("--- batch-get result ---"); - iterator = session.batchGet(TABLE_NAME, keyList); - while(iterator.hasNext()){ - HgKvEntry kv=iterator.next(); - Assert.assertEquals(true,keyMap.containsKey(Arrays.toString(kv.key()))); - } -*/ - - } - - @Test - //CAUTION: ONLY FOR LONG！ - //注意：目前只可以对long类型value进行Merge操作。 - public void merge() { - System.out.println("--- test merge (1+1=2)---"); - HgStoreSession session = getStoreSession(); - String mergeKey = "merge-key"; - HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey); - byte[] value = HgStoreTestUtil.toBytes(1L); - - System.out.println("- put " + mergeKey + ":1 -"); - session.put(HgStoreTestUtil.TABLE_NAME, key, value); - System.out.println("- merge " + mergeKey + ":1 -"); - session.merge(HgStoreTestUtil.TABLE_NAME, key, value); - long res = HgStoreTestUtil.toLong(session.get(HgStoreTestUtil.TABLE_NAME, key)); - System.out.printf("after merge " + mergeKey + "=%s%n", res); - Assert.assertEquals(2L, res); - - String putStr = "19"; - session.put(HgStoreTestUtil.TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr)); - byte[] b1 = session.get(HgStoreTestUtil.TABLE_NAME, key); - Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1)); - } - - @Test - public void delete() { - System.out.println("--- test delete ---"); - HgStoreSession session = getStoreSession(); - - String delKey = "del-key"; - String delValue = "del-value"; - HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); - byte[] value = HgStoreTestUtil.toBytes(delValue); - - HgStoreTestUtil.println("- put " + delKey + ": " + delValue); - session.put(HgStoreTestUtil.TABLE_NAME, key, value); - HgStoreTestUtil.println("- delete " + delKey); - session.delete(HgStoreTestUtil.TABLE_NAME, key); - value = session.get(HgStoreTestUtil.TABLE_NAME, key); - HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value)); - Assert.assertEquals(EMPTY_BYTES, value); - } - - // @Test - public void deleteSingle() { - System.out.println("--- test deleteSingle ---"); - HgStoreSession session = getStoreSession(); - - String delKey = "del-single-key"; - String delValue = "del-single-value"; - HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); - byte[] value = HgStoreTestUtil.toBytes(delValue); - - HgStoreTestUtil.println("- put [" + delKey + "] = " + delValue); - session.put(HgStoreTestUtil.TABLE_NAME, key, value); - - value = session.get(HgStoreTestUtil.TABLE_NAME, key); - HgStoreTestUtil.println( - "- before del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); - Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value)); - - HgStoreTestUtil.println("- delete-single : [" + delKey + "]"); - session.deleteSingle(HgStoreTestUtil.TABLE_NAME, key); - value = session.get(HgStoreTestUtil.TABLE_NAME, key); - HgStoreTestUtil.println( - "- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); - Assert.assertEquals("", HgStoreTestUtil.toStr(value)); - - } - - // @Test - public void deleteRange() { - HgStoreTestUtil.println("--- test deleteRange ---"); - HgStoreSession session = getStoreSession(); - - String rangePrefix = "DEL-RANGE-KEY"; - String owner = "batch-delete-owner"; - Map map = - HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, rangePrefix, 10, - key -> HgStoreTestUtil.toOwnerKey(owner, key) - ); - - HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00"); - HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05"); - - Assert.assertTrue(session.deleteRange(HgStoreTestUtil.TABLE_NAME, startKey, endKey)); - - HgStoreTestUtil.println("- after delete range from [" - + HgStoreTestUtil.toStr(startKey.getKey()) + "] to [" - + HgStoreTestUtil.toStr(endKey.getKey()) + "]"); - - for (int i = 0; i < 10; i++) { - HgOwnerKey key = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" + - HgStoreTestUtil.padLeftZeros( - String.valueOf(i), 2)); - String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); - HgStoreTestUtil.println( - "- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); - - // TODO: [start,end)? - if (i < 5) { - Assert.assertEquals("", value); - } else { - //println(value); - Assert.assertNotEquals("", value); - } - } - - } - - // @Test - public void deletePrefix() { - System.out.println("--- test deletePrefix ---"); - HgStoreSession session = getStoreSession(); - - String prefixStr = "DEL-PREFIX-KEY"; - String owner = "batch-delete-owner"; - Map map = - HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, prefixStr, 10, - key -> { - return HgStoreTestUtil.toOwnerKey(owner, key); - }); - - //printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList())); - - HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr); - - Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator( - HgStoreTestUtil.TABLE_NAME, prefix))); - session.deletePrefix(HgStoreTestUtil.TABLE_NAME, prefix); - Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator( - HgStoreTestUtil.TABLE_NAME, prefix))); - - HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]"); - - for (int i = 0; i < 10; i++) { - HgOwnerKey key = - HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2)); - String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); - System.out.println("- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); - Assert.assertEquals("", value); - } - - } - - // @Test - public void scan() { - HgStoreTestUtil.println("--- test scanIterator ---"); - String tableName = "UNIT_SCAN_ITERATOR"; - String keyName = "SCAN-ITER"; - int keyAmt = 100; - - HgStoreSession session = getStoreSession(); - if (HgStoreTestUtil.amountOf( - session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) < - 10) { - HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); - } - int count = 0; - int limit = 0; - int max = 99999; - HgKvIterator iterator = null; - - // println("-- test scan all --"); - iterator = session.scanIterator(tableName); - count = 0; - while (iterator.hasNext()) { - count++; - HgKvEntry entry = iterator.next(); - if (count % 10 == 0) { - // println(entry); - } - if (count >= max) break; - } - iterator.close(); - Assert.assertEquals(keyAmt, count); - - // println("-- test 0 element --"); - iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"), - HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0); - Assert.assertFalse(iterator.hasNext()); - try { - iterator.next(); - Assert.fail(); - } catch (Throwable t) { - // println("-- test NoSuchElementException --"); - Assert.assertTrue(t instanceof NoSuchElementException); - } - iterator.close(); - - // println("-- test limit 1 to 10 --"); - for (int i = 1; i <= 10; i++) { - // println("- limit " + i + " -"); - limit = i; - iterator = session.scanIterator(tableName, - HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), - HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), - limit); - count = 0; - while (iterator.hasNext()) { - count++; - HgKvEntry entry = iterator.next(); - // println(entry); - } - iterator.close(); - Assert.assertEquals(limit, count); - } - - // println("-- test limit 1 to 10 not enough --"); - for (int i = 1; i <= 10; i++) { - // println("- limit " + i + " -"); - limit = i; - iterator = session.scanIterator(tableName - , HgStoreTestUtil.toAllPartitionKey(keyName + "-001") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-005"), limit); - count = 0; - while (iterator.hasNext()) { - count++; - HgKvEntry entry = iterator.next(); - // println(entry); - } - iterator.close(); - if (i <= 5) { - Assert.assertEquals(limit, count); - } else { - Assert.assertEquals(5, count); - } - - } - - // println("-- test limit 0 (no limit) --"); - limit = 0; - iterator = - session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), - HgStoreTestUtil.toAllPartitionKey(keyName + "-2"), limit); - - count = 0; - while (iterator.hasNext()) { - count++; - HgKvEntry entry = iterator.next(); - if (count % 10 == 0) { - // println(entry); - } - if (count >= max) break; - } - iterator.close(); - Assert.assertEquals(keyAmt, count); - - // println("-- test scan prefix --"); - iterator = - session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-")); - count = 0; - while (iterator.hasNext()) { - count++; - HgKvEntry entry = iterator.next(); - if (count % 10 == 0) { - // println(entry); - } - if (count >= max) break; - } - iterator.close(); - Assert.assertEquals(keyAmt, count); - - // println("-- test scan via hash code --"); - iterator = session.scanIterator(tableName, 0, 65535, HgKvStore.SCAN_HASHCODE, EMPTY_BYTES); - count = 0; - - while (iterator.hasNext()) { - count++; - HgKvEntry entry = iterator.next(); - if (count % 10 == 0) { - // println(entry); - } - if (count >= max) break; - } - iterator.close(); - Assert.assertEquals(keyAmt, count); - - // println("-- test range limit scan type --"); - iterator = session.scanIterator(tableName - , HgStoreTestUtil.toAllPartitionKey("WWWWWWW") - , HgStoreClientConst.EMPTY_OWNER_KEY - , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES - ); - iterator.close(); - // println("-- test range limit scan type -session.scanIterator over--"); - Assert.assertEquals(0, HgStoreTestUtil.amountOf(iterator)); - // println("-- test range limit scan type -session.assertEquals over--"); - - // println("-- test range limit scan type -100"); - iterator = session.scanIterator(tableName - , HgStoreTestUtil.toAllPartitionKey(keyName + "-100") - , HgStoreClientConst.EMPTY_OWNER_KEY - , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES - ); - - // println("-- test range limit scan type -100 over"); - Assert.assertEquals(1, HgStoreTestUtil.amountOf(iterator)); - // println("-- test range limit scan type -100 assertEquals 1"); - - // println("-- test range limit scan type -51 "); - iterator = session.scanIterator(tableName - , HgStoreTestUtil.toAllPartitionKey(keyName + "-051") - , HgStoreClientConst.EMPTY_OWNER_KEY - ); - - // println("-- test range limit scan type -51 over"); - //HgStoreTestUtil.println(iterator); - Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator)); - // println("-- test range limit scan type -51 assertEquals"); - //TODO: add more... - HgStoreTestUtil.println("-- test scanIterator end"); - } - - // @Test - public void scan_close() { - /*************** test scan close **************/ - HgStoreTestUtil.println("--- test scan close ---"); - String tableName = "UNIT_ITER_CLOSE"; - String keyName = "UNIT_ITER_CLOSE"; - - int amount = 1_000_000; - HgStoreSession session = getStoreSession(); - if (HgStoreTestUtil.amountOf( - session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < - 1) { - HgStoreTestUtil.batchPut(session, tableName, keyName, amount); - } - - HgKvIterator iterator = null; - - /* server over, all -> close */ - iterator = session.scanIterator(tableName); - Assert.assertEquals(amount, HgStoreTestUtil.amountOf(iterator)); - - /* server over, one page, all -> close */ - iterator = session.scanIterator(tableName, 100_000); - Assert.assertEquals(100_000, HgStoreTestUtil.amountOf(iterator)); - - /* server over, two page, all -> close */ - iterator = session.scanIterator(tableName, 200_000); - Assert.assertEquals(200_000, HgStoreTestUtil.amountOf(iterator)); - - /* server not over, enough -> close */ - iterator = session.scanIterator(tableName); - iterator.next(); - iterator.close(); - - /* server not over, one page, enough -> close */ - iterator = session.scanIterator(tableName, 100_000); - iterator.next(); - iterator.close(); - - /* server not over, two page, enough -> close */ - iterator = session.scanIterator(tableName, 200_000); - iterator.next(); - iterator.close(); - - /* server not over, enough -> close */ - iterator = session.scanIterator(tableName); - iterator.next(); - iterator.close(); - iterator = session.scanIterator(tableName); - for (int i = 0; iterator.hasNext() && i < 500_000; i++) { - iterator.next(); - } - iterator.close(); - - /* server timeout, enough -> close */ - iterator = session.scanIterator(tableName); - iterator.next(); - // HgStoreTestUtil.sleeping(11000); - - try { - HgStoreTestUtil.amountOf(iterator); - } catch (Throwable t) { - HgStoreTestUtil.println("-- passed server waiting timeout --"); - } - - iterator.close(); - - } - - // @Test - public void paging() { - HgStoreTestUtil.println("--- test scanIterator_range ---"); - String graph = "UNIT/paging"; - String tableName = "UNIT_SCAN_PAGING"; - String keyName = "SCAN-PAGING"; - int keyAmt = 100; - HgStoreSession session = getStoreSession(graph); - - if (HgStoreTestUtil.amountOf( - session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < - 1) { - HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); - } - - int count = 0; - int limit = 0; - int max = 99999; - HgKvIterator iterator = null; - - count = 0; - - iterator = session.scanIterator(tableName - , HgStoreTestUtil.toAllPartitionKey(keyName + "-000") - , HgStoreClientConst.EMPTY_OWNER_KEY - , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES - ); - //HgStoreTestUtil.println(iterator); - // Assert.assertEquals(100, HgStoreTestUtil.amountOf(iterator)); - List positionList = new LinkedList<>(); - while (iterator.hasNext()) { - HgStoreTestUtil.print((count++) + " "); - HgKvEntry entry = iterator.next(); - HgStoreTestUtil.print(entry); - HgStoreTestUtil.print(" " + Arrays.toString(iterator.position()) + "\n"); - positionList.add(iterator.position()); - if (count >= max) break; - } - - - iterator = session.scanIterator(tableName - , HgStoreTestUtil.toAllPartitionKey(keyName + "-000") - , HgStoreClientConst.EMPTY_OWNER_KEY - , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES - ); - - byte[] position = positionList.get(50); - HgStoreTestUtil.println("seek: " + Arrays.toString(position)); - iterator.seek(position); - //println("amt after seek: "+HgStoreTestUtil.println(iterator)); - Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator)); - - } - - @Test - public void scanBatch() { - HgStoreTestUtil.println("--- test scanBatch ---"); - String tableName = "UNIT_SCAN_BATCH_1"; - String keyName = "SCAN-BATCH"; - int keyAmt = 10_000; - - HgStoreSession session = getStoreSession(); - - if (HgStoreTestUtil.amountOf( - session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), - keyAmt)) < keyAmt) { - HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); - } - - Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); - - List> iterators = null; - List prefixes = null; - - List startList = Arrays.asList( - HgStoreTestUtil.toAllPartitionKey(keyName + "-001") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-003") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-005") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-007") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-009") - ); - - List endList = Arrays.asList( - HgStoreTestUtil.toAllPartitionKey(keyName + "-002") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-004") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-006") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-008") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-010") - ); - - List prefixList = Arrays.asList( - HgStoreTestUtil.toAllPartitionKey(keyName + "-001") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-002") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-003") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-004") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-005") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-006") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-007") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-008") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-009") - ); - - HgStoreTestUtil.println("-- test scan-batch all --"); - - HgScanQuery scanQuery = HgScanQuery.tableOf(tableName); - iterators = session.scanBatch(scanQuery); - Assert.assertEquals(3, iterators.size()); - - Assert.assertEquals(keyAmt, HgStoreTestUtil.println(iterators)); - - HgStoreTestUtil.println("-- test scan-batch prefix --"); - - - iterators = session.scanBatch( - HgScanQuery.prefixOf(tableName, prefixList) - ); - Assert.assertEquals(3, iterators.size()); - Assert.assertEquals(900, - iterators.parallelStream().mapToInt(e -> HgStoreTestUtil.amountOf(e)) - .sum() - ); - - HgStoreTestUtil.println("-- test scan-batch range --"); - - iterators = session.scanBatch(HgScanQuery.rangeOf(tableName, startList, endList)); - Assert.assertEquals(3, iterators.size()); - Assert.assertEquals( - HgStoreTestUtil.amountOf( - session.scanIterator(tableName - , HgStoreTestUtil.toAllPartitionKey(keyName + "-001") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-010") - ) - ) - , - iterators.parallelStream().mapToInt(e -> HgStoreTestUtil.amountOf(e)).sum() - ); - - HgStoreTestUtil.println("-- test scan-batch limit --"); - - int limit = 1; - iterators = session.scanBatch( - HgScanQuery.rangeOf(tableName, startList, endList) - .builder() - .setLimit(limit) - .build() - ); - - //HgStoreTestUtil.println(iterators); - Assert.assertEquals(iterators.size() * limit, - iterators.parallelStream().mapToInt(e -> HgStoreTestUtil.amountOf(e)) - .sum() - ); - - HgStoreTestUtil.println("-- test scan-batch multi-table --"); - if (HgStoreTestUtil.amountOf( - session.scanIterator("g+oe", HgStoreTestUtil.toAllPartitionKey(keyName), keyAmt)) < - keyAmt) { - HgStoreTestUtil.batchPut(session, "g+oe", keyName, keyAmt); - HgStoreTestUtil.batchPut(session, "g+ie", keyName, keyAmt); - } - - prefixes = Collections.singletonList(HgStoreTestUtil.toAllPartitionKey(keyName)); - - tableName = "g+oe,g+ie"; - iterators = session.scanBatch( - HgScanQuery.prefixOf(tableName, prefixes) - ); - - //HgStoreTestUtil.println(iterators); - Assert.assertEquals(keyAmt * 2, HgStoreTestUtil.amountIn(iterators)); - iterators = session.scanBatch( - HgScanQuery.prefixOf(tableName, prefixList) - ); - Assert.assertEquals(900 * 2, HgStoreTestUtil.amountIn(iterators)); - - HgStoreTestUtil.println("-- test scan-batch per-key-limit --"); - - tableName = "PER_KEY_LIMIT_TABLE"; - keyName = "PER_KEY_LIMIT"; - if (HgStoreTestUtil.amountOf( - session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), - keyAmt)) < keyAmt) { - HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); - } - prefixes = Arrays.asList( - HgStoreTestUtil.toAllPartitionKey(keyName + "-01") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-02") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-03") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-04") - , HgStoreTestUtil.toAllPartitionKey(keyName + "-05") - ); - - iterators = session.scanBatch( - HgScanQuery.prefixOf(tableName, prefixes).builder().setPerKeyLimit(1).build() - ); - //HgStoreTestUtil.println(iterators); - Assert.assertEquals(prefixes.size() * 3, HgStoreTestUtil.amountIn(iterators)); - - } - - // @Test - public void truncate() { - HgStoreTestUtil.println("--- test truncate ---"); - String graph = "graph_truncate"; - String tableName = "UNIT_TRUNCATE_1"; - String keyName = "KEY_TRUNCATE"; - - HgStoreSession session = getStoreSession(graph); - HgStoreTestUtil.batchPut(session, tableName, keyName, 100); - Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); - - String tableName2 = "UNIT_TRUNCATE_2"; - HgStoreTestUtil.batchPut(session, tableName2, keyName, 100); - Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); - - session.truncate(); - Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); - Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); - } - - // @Test - public void handle_table() { - HgStoreTestUtil.println("--- test table ---"); - - String tableName = "UNIT_TABLE_" + System.currentTimeMillis(); - HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); - byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); - HgStoreSession session = getStoreSession(); - - HgStoreTestUtil.println("-- test createTable --"); - session.createTable(tableName); - Assert.assertTrue(session.existsTable(tableName)); - - HgStoreTestUtil.println("-- test deleteTable --"); - session.put(tableName, key, value); - Assert.assertEquals(HgStoreTestUtil.toStr(value), - HgStoreTestUtil.toStr(session.get(tableName, key))); - session.deleteTable(tableName); - Assert.assertNotEquals(HgStoreTestUtil.toStr(value), - HgStoreTestUtil.toStr(session.get(tableName, key))); - Assert.assertTrue(session.existsTable(tableName)); - - - HgStoreTestUtil.println("-- test dropTable --"); - Assert.assertTrue(session.dropTable(tableName)); - Assert.assertFalse(session.existsTable(tableName)); - - - HgStoreTestUtil.println("-- test existsTable --"); - Assert.assertFalse(session.existsTable(tableName)); - - } - - // @Test - public void tx() { - HgStoreTestUtil.println("--- test tx ---"); - HgStoreSession session = getStoreSession(); - String tableName = "UNIT_TABLE_TX"; - String keyPrefix = "TX"; - int keyAmt = 100; - - HgStoreTestUtil.println("-- unsupported tx operation --"); - HgStoreTestUtil.println("- tx deleteTable -"); - HgStoreTestUtil.batchPut(session, tableName, keyPrefix, keyAmt); - Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); - session.beginTx(); - session.deleteTable(tableName); - session.commit(); - Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); - - HgStoreTestUtil.println("- tx dropTable -"); - HgStoreTestUtil.batchPut(session, tableName, keyPrefix, keyAmt); - Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); - session.beginTx(); - session.dropTable(tableName); - session.commit(); - Assert.assertFalse(session.existsTable(tableName)); - Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); - - HgStoreTestUtil.println("- tx truncate -"); - HgStoreTestUtil.batchPut(session, tableName, keyPrefix, keyAmt); - Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); - session.beginTx(); - session.truncate(); - session.commit(); - Assert.assertFalse(session.existsTable(tableName)); - Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); - - //TODO : add others - } - - //// @Test - public void scanIterator_WithNonePartition() { - HgStoreTestUtil.println("--- test scanIterator with none partition ---"); - int count = 0; - HgStoreSession session = getStoreSession(); - - for (int i = 0; i < 100; i++) { - HgKvIterator iterator = session.scanIterator("XXXXXXXX"); - while (iterator.hasNext()) { - count++; - HgKvEntry entry = iterator.next(); - HgStoreTestUtil.println(entry); - } - } - - Assert.assertEquals(0, count); - - } - - //// @Test - public void repeatedly_parallel_scan() { - MetricX metrics = MetricX.ofStart(); - HgStoreTestUtil.repeatedly_test(100, () -> parallel_scan()); - metrics.end(); - - log.info("*************************************************"); - log.info("************* Scanning Completed **************"); - log.info("Total: {} sec.", metrics.past() / 1000); - log.info("Sum: {} sec.", MetricX.getIteratorWait() / 1000); - log.info("Amt: {} scans.", MetricX.getIteratorCount()); - log.info("Avg: {} ms.", MetricX.getIteratorWaitAvg()); - log.info("Max: {} ms.", MetricX.getIteratorWaitMax()); - log.info("Fail: {} times.", metrics.getFailureCount()); - log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize()); - log.info("*************************************************"); - - // runWaiting(); - } - - // @Test - public void parallel_scan() { - HgStoreTestUtil.println("--- test scanIterator in parallel ---"); - HgStoreTestUtil.parallel_test(100, () -> { - this.scan(); - }, t -> t.printStackTrace()); - } - - //// @Test - public void put_Benchmark() { - /*************** Put Benchmark **************/ - String tableName = "UNIT_PUT_BENCHMARK"; - String keyPrefix = "PUT-BENCHMARK"; - int amount = 1_000_000; - HgStoreSession session = getStoreSession(); - - int length = String.valueOf(amount).length(); - - session.beginTx(); - - long start = System.currentTimeMillis(); - for (int i = 0; i < amount; i++) { - HgOwnerKey key = HgStoreTestUtil.toOwnerKey( - keyPrefix + "-" + HgStoreTestUtil.padLeftZeros(String.valueOf(i), length)); - byte[] value = HgStoreTestUtil.toBytes(keyPrefix + "-V-" + i); - - session.put(tableName, key, value); - - if ((i + 1) % 100_000 == 0) { - HgStoreTestUtil.println("---------- " + (i + 1) + " --------"); - HgStoreTestUtil.println( - "Preparing took: " + (System.currentTimeMillis() - start) + " ms."); - session.commit(); - HgStoreTestUtil.println( - "Committing took: " + (System.currentTimeMillis() - start) + " ms."); - start = System.currentTimeMillis(); - session.beginTx(); - } - } - - if (session.isTx()) { - session.commit(); - } - - Assert.assertEquals(amount, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); - - - } - - //// @Test - public void put_Benchmark_Parallel() { - int threadsAmount = 100; - CountDownLatch countDownLatch = new CountDownLatch(threadsAmount); - - for (int j = 0; j < threadsAmount; j++) { - for (int i = 0; i < threadsAmount; i++) { - pool.submit(() -> { - this.put_Benchmark(); - // this.scanIterator_WithNonePartition(); - countDownLatch.countDown(); - }); - } - } - try { - countDownLatch.await(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - } - - //// @Test - public void parallel_scanBatch() { - int threadsAmount = 50; - CountDownLatch countDownLatch = new CountDownLatch(threadsAmount); - - for (int j = 0; j < threadsAmount; j++) { - for (int i = 0; i < threadsAmount; i++) { - pool.submit(() -> { - try { - this.scanBatch(); - } catch (Throwable t) { - t.printStackTrace(); - } finally { - countDownLatch.countDown(); - } - }); - } - } - try { - countDownLatch.await(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - } - - //// @Test - public void benchmark_scanBatch() { - HgStoreTestUtil.println("--- Benchmark scanBatch ---"); - String tableName = "Benchmark_SCAN_BATCH"; - String keyName = "SCAN-BATCH"; - int keyAmt = 10_000_000; - - HgStoreSession session = getStoreSession(); - - if (HgStoreTestUtil.amountOf( - session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < - 1) { - HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); - } - HgStoreTestUtil.println("-- Starting scan --"); - // while (true) { - - MetricX metrics = MetricX.ofStart(); - //HgStoreTestUtil.println(session.scanIterator(tableName)); - List> iterators = session.scanBatch(HgScanQuery.tableOf(tableName)); - //HgStoreTestUtil.println(iterators); - //Assert.assertEquals(keyAmt, HgStoreTestUtil.amountIn(iterators)); - HgStoreTestUtil.amountIn(iterators); - metrics.end(); - log.info("*************************************************"); - log.info("*********** Batch Scan Completed **************"); - log.info("Total : {} (sec)", metrics.past() / 1000); - log.info(" Sum : {} (sec)", MetricX.getIteratorWait() / 1000); - log.info(" Amt : {} (scans).", MetricX.getIteratorCount()); - log.info(" Avg : {} (ms)", MetricX.getIteratorWaitAvg()); - log.info(" Max : {} (ms)", MetricX.getIteratorWaitMax()); - log.info(" Fail : {} (times)", metrics.getFailureCount()); - log.info(" Page : {} (KVs)", HgStoreClientConfig.of().getNetKvScannerPageSize()); - log.info("Iters : {}", iterators.size()); - log.info("*************************************************"); - HgStoreTestUtil.sleeping(100); - - // } - } - - //// @Test - public void benchmark_scan() { - /*************** test no limit, with 10 millions **************/ - String tableName = "UNIT_HUGE"; - String keyName = "SCAN-HUGE"; - int amount = 10_000_000; - int max = 10_000_000; - HgStoreSession session = getStoreSession(); - - /*Initialization*/ - if (HgStoreTestUtil.amountOf( - session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < - 1) { - HgStoreTestUtil.batchPut(session, tableName, keyName, amount); - } - - MetricX metricX = MetricX.ofStart(); - - int count = 0; - HgKvIterator iterator = session.scanIterator(tableName); - //HgStoreTestUtil.println(iterator, e -> (e % (amount / 100) == 0)); - while (iterator.hasNext()) { - count++; - HgKvEntry entry = iterator.next(); - if (count % (amount / 10) == 0) { - HgStoreTestUtil.println(entry); - } - if (count >= max) break; - } - - metricX.end(); - - log.info("*************************************************"); - log.info("************* Benchmark Completed *************"); - log.info("Keys: {}", count); - log.info("Waiting: {} ms.", MetricX.getIteratorWait()); - log.info("Total: {} ms.", metricX.past()); - log.info("Iterator: [{}]", iterator.getClass().getSimpleName()); - log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize()); - log.info("*************************************************"); - - Assert.assertEquals(amount, count); - } - - - //// @Test - public void extreme_scan_close() { - /*************** test close **************/ - String tableName = "UNIT_ITER_CLOSE_EXTREME"; - String keyName = "UNIT_ITER_CLOSE_EXTREME"; - int amount = 1_000_000; - HgStoreSession session = getStoreSession(); - - if (HgStoreTestUtil.amountOf( - session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < - 1) { - HgStoreTestUtil.batchPut(session, tableName, keyName, amount); - } - HgKvIterator iterator = null; - /* server not over, enough, extreme -> close */ - for (int i = 0; i <= 10_000; i++) { - iterator = session.scanIterator(tableName); - iterator.next(); - iterator.close(); - HgStoreTestUtil.println("extreme loop: " + i); - } - HgStoreTestUtil.runWaiting(); - } - - //// @Test - public void parallel_scan_close() { - HgStoreTestUtil.parallel_test(10, () -> this.scan_close(), t -> t.printStackTrace()); - } - - //// @Test - public void repeat_parallel_scan_close() { - HgStoreTestUtil.repeatedly_test(1000, () -> this.parallel_scan_close()); - HgStoreTestUtil.runWaiting(); - } - - //// @Test - public void parallel_huge_scan() { - int threadsAmount = 3; - CountDownLatch countDownLatch = new CountDownLatch(threadsAmount); - ExecutorService poll = Executors.newFixedThreadPool(threadsAmount); - - for (int i = 0; i < threadsAmount; i++) { - poll.submit(() -> { - this.benchmark_scan(); - countDownLatch.countDown(); - }); - } - - try { - countDownLatch.await(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - } - +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store; + +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +import org.apache.hugegraph.pd.common.PartitionUtils; +import org.apache.hugegraph.store.client.HgStoreNodeManager; +import org.apache.hugegraph.store.client.util.ExecutorPool; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.MetricX; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class HgSessionManagerTest { + private static final Map leaderMap = new ConcurrentHashMap<>(); + private static final Map storeMap = new ConcurrentHashMap<>(); + + private static final ExecutorService pool = Executors.newFixedThreadPool(100, + ExecutorPool.newThreadFactory( + "unit-test")); + + private static final int partitionCount = 10; + // 需要与store的application.yml的fake-pd.partition-count保持一致 + + //private static String[] storeAddress = {"127.0.0.1:8500"}; + private static final String[] storeAddress = + {"127.0.0.1:8501", "127.0.0.1:8502", "127.0.0.1:8503"}; + + private static final int PARTITION_LENGTH = getPartitionLength(); + + private static int getPartitionLength() { + return PartitionUtils.MAX_VALUE / + (partitionCount == 0 ? storeAddress.length : partitionCount) + 1; + } + + @BeforeClass + public static void init() { + for (String address : storeAddress) { + storeMap.put((long) address.hashCode(), address); + } + for (int i = 0; i < partitionCount; i++) { + leaderMap.put(i, storeMap.keySet().iterator().next()); + } + + HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); + nodeManager.setNodePartitioner((builder, graphName, startKey, endKey) -> { + int startCode = PartitionUtils.calcHashcode(startKey); + int endCode = PartitionUtils.calcHashcode(endKey); + if (ALL_PARTITION_OWNER == startKey) { + storeMap.forEach((k, v) -> { + builder.add(k, -1); + }); + } else if (endKey == HgStoreClientConst.EMPTY_BYTES || startKey == endKey || + Arrays.equals(startKey, endKey)) { + //log.info("leader-> {}",leaderMap.get(startCode / PARTITION_LENGTH)); + builder.add(leaderMap.get(startCode / PARTITION_LENGTH), startCode); + } else { + Assert.fail("OwnerKey转成HashCode后已经无序了， 按照OwnerKey范围查询没意义"); + builder.add(leaderMap.get(startCode / PARTITION_LENGTH), startCode); + builder.add(leaderMap.get(endCode / PARTITION_LENGTH), endCode); + } + return 0; + }); + nodeManager.setNodeProvider((graphName, nodeId) -> { + // System.out.println("HgStoreNodeProvider apply " + graphName + " " + nodeId + + // " " + storeMap.get(nodeId)); + return nodeManager.getNodeBuilder().setNodeId(nodeId) + .setAddress(storeMap.get(nodeId)).build(); + }); + nodeManager.setNodeNotifier((graphName, storeNotice) -> { + System.out.println("recv node notifier " + storeNotice); + if (storeNotice.getPartitionLeaders().size() > 0) { + leaderMap.putAll(storeNotice.getPartitionLeaders()); + System.out.println("leader changed "); + leaderMap.forEach((k, v) -> { + System.out.print(" " + k + " " + v + ","); + }); + System.out.println(); + } + return 0; + }); + } + + protected static HgStoreSession getStoreSession() { + return HgSessionManager.getInstance().openSession(HgStoreTestUtil.GRAPH_NAME); + } + + protected static HgStoreSession getStoreSession(String graph) { + return HgSessionManager.getInstance().openSession(graph); + } + + @Test + public void put_get() { + System.out.println("--- test put & get ---"); + + HgStoreSession session = getStoreSession(); + + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + + Assert.assertTrue(session.put(HgStoreTestUtil.TABLE_NAME, key, value)); + Assert.assertEquals(HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get( + HgStoreTestUtil.TABLE_NAME, key))); + } + + @Test + public void batchPrefix() { + System.out.println("--- test batchGet ---"); + HgStoreSession session = getStoreSession(); + String keyPrefix = "UNIT-BATCH-GET"; + + Map map = HgStoreTestUtil.batchPut(session, keyPrefix); + List keyList = + map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); + + Map keyMap = new HashMap<>(); + map.entrySet().forEach(e -> { + keyMap.put(Arrays.toString(e.getKey().getKey()), e.getValue()); + }); + + //printOwner(keyList); + HgKvIterator iterator = session.batchPrefix(HgStoreTestUtil.TABLE_NAME, keyList); + int amount = HgStoreTestUtil.println(iterator); + Assert.assertEquals(amount, keyList.size()); +/* + + println("--- batch-get result ---"); + iterator = session.batchGet(TABLE_NAME, keyList); + while(iterator.hasNext()){ + HgKvEntry kv=iterator.next(); + Assert.assertEquals(true,keyMap.containsKey(Arrays.toString(kv.key()))); + } +*/ + + } + + @Test + //CAUTION: ONLY FOR LONG！ + //注意：目前只可以对long类型value进行Merge操作。 + public void merge() { + System.out.println("--- test merge (1+1=2)---"); + HgStoreSession session = getStoreSession(); + String mergeKey = "merge-key"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey); + byte[] value = HgStoreTestUtil.toBytes(1L); + + System.out.println("- put " + mergeKey + ":1 -"); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + System.out.println("- merge " + mergeKey + ":1 -"); + session.merge(HgStoreTestUtil.TABLE_NAME, key, value); + long res = HgStoreTestUtil.toLong(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.printf("after merge " + mergeKey + "=%s%n", res); + Assert.assertEquals(2L, res); + + String putStr = "19"; + session.put(HgStoreTestUtil.TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr)); + byte[] b1 = session.get(HgStoreTestUtil.TABLE_NAME, key); + Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1)); + } + + @Test + public void delete() { + System.out.println("--- test delete ---"); + HgStoreSession session = getStoreSession(); + + String delKey = "del-key"; + String delValue = "del-value"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); + + HgStoreTestUtil.println("- put " + delKey + ": " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + HgStoreTestUtil.println("- delete " + delKey); + session.delete(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(EMPTY_BYTES, value); + } + + // @Test + public void deleteSingle() { + System.out.println("--- test deleteSingle ---"); + HgStoreSession session = getStoreSession(); + + String delKey = "del-single-key"; + String delValue = "del-single-value"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); + + HgStoreTestUtil.println("- put [" + delKey + "] = " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- before del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value)); + + HgStoreTestUtil.println("- delete-single : [" + delKey + "]"); + session.deleteSingle(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals("", HgStoreTestUtil.toStr(value)); + + } + + // @Test + public void deleteRange() { + HgStoreTestUtil.println("--- test deleteRange ---"); + HgStoreSession session = getStoreSession(); + + String rangePrefix = "DEL-RANGE-KEY"; + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, rangePrefix, 10, + key -> HgStoreTestUtil.toOwnerKey(owner, key) + ); + + HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00"); + HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05"); + + Assert.assertTrue(session.deleteRange(HgStoreTestUtil.TABLE_NAME, startKey, endKey)); + + HgStoreTestUtil.println("- after delete range from [" + + HgStoreTestUtil.toStr(startKey.getKey()) + "] to [" + + HgStoreTestUtil.toStr(endKey.getKey()) + "]"); + + for (int i = 0; i < 10; i++) { + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" + + HgStoreTestUtil.padLeftZeros( + String.valueOf(i), 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + HgStoreTestUtil.println( + "- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); + + // TODO: [start,end)? + if (i < 5) { + Assert.assertEquals("", value); + } else { + //println(value); + Assert.assertNotEquals("", value); + } + } + + } + + // @Test + public void deletePrefix() { + System.out.println("--- test deletePrefix ---"); + HgStoreSession session = getStoreSession(); + + String prefixStr = "DEL-PREFIX-KEY"; + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, prefixStr, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + + //printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList())); + + HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr); + + Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + session.deletePrefix(HgStoreTestUtil.TABLE_NAME, prefix); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + + HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]"); + + for (int i = 0; i < 10; i++) { + HgOwnerKey key = + HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.println("- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); + Assert.assertEquals("", value); + } + + } + + // @Test + public void scan() { + HgStoreTestUtil.println("--- test scanIterator ---"); + String tableName = "UNIT_SCAN_ITERATOR"; + String keyName = "SCAN-ITER"; + int keyAmt = 100; + + HgStoreSession session = getStoreSession(); + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) < + 10) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); + } + int count = 0; + int limit = 0; + int max = 99999; + HgKvIterator iterator = null; + + // println("-- test scan all --"); + iterator = session.scanIterator(tableName); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 10 == 0) { + // println(entry); + } + if (count >= max) break; + } + iterator.close(); + Assert.assertEquals(keyAmt, count); + + // println("-- test 0 element --"); + iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"), + HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0); + Assert.assertFalse(iterator.hasNext()); + try { + iterator.next(); + Assert.fail(); + } catch (Throwable t) { + // println("-- test NoSuchElementException --"); + Assert.assertTrue(t instanceof NoSuchElementException); + } + iterator.close(); + + // println("-- test limit 1 to 10 --"); + for (int i = 1; i <= 10; i++) { + // println("- limit " + i + " -"); + limit = i; + iterator = session.scanIterator(tableName, + HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), + limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + // println(entry); + } + iterator.close(); + Assert.assertEquals(limit, count); + } + + // println("-- test limit 1 to 10 not enough --"); + for (int i = 1; i <= 10; i++) { + // println("- limit " + i + " -"); + limit = i; + iterator = session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey(keyName + "-001") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-005"), limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + // println(entry); + } + iterator.close(); + if (i <= 5) { + Assert.assertEquals(limit, count); + } else { + Assert.assertEquals(5, count); + } + + } + + // println("-- test limit 0 (no limit) --"); + limit = 0; + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-2"), limit); + + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 10 == 0) { + // println(entry); + } + if (count >= max) break; + } + iterator.close(); + Assert.assertEquals(keyAmt, count); + + // println("-- test scan prefix --"); + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-")); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 10 == 0) { + // println(entry); + } + if (count >= max) break; + } + iterator.close(); + Assert.assertEquals(keyAmt, count); + + // println("-- test scan via hash code --"); + iterator = session.scanIterator(tableName, 0, 65535, HgKvStore.SCAN_HASHCODE, EMPTY_BYTES); + count = 0; + + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 10 == 0) { + // println(entry); + } + if (count >= max) break; + } + iterator.close(); + Assert.assertEquals(keyAmt, count); + + // println("-- test range limit scan type --"); + iterator = session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey("WWWWWWW") + , HgStoreClientConst.EMPTY_OWNER_KEY + , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES + ); + iterator.close(); + // println("-- test range limit scan type -session.scanIterator over--"); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(iterator)); + // println("-- test range limit scan type -session.assertEquals over--"); + + // println("-- test range limit scan type -100"); + iterator = session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey(keyName + "-100") + , HgStoreClientConst.EMPTY_OWNER_KEY + , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES + ); + + // println("-- test range limit scan type -100 over"); + Assert.assertEquals(1, HgStoreTestUtil.amountOf(iterator)); + // println("-- test range limit scan type -100 assertEquals 1"); + + // println("-- test range limit scan type -51 "); + iterator = session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey(keyName + "-051") + , HgStoreClientConst.EMPTY_OWNER_KEY + ); + + // println("-- test range limit scan type -51 over"); + //HgStoreTestUtil.println(iterator); + Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator)); + // println("-- test range limit scan type -51 assertEquals"); + //TODO: add more... + HgStoreTestUtil.println("-- test scanIterator end"); + } + + // @Test + public void scan_close() { + /*************** test scan close **************/ + HgStoreTestUtil.println("--- test scan close ---"); + String tableName = "UNIT_ITER_CLOSE"; + String keyName = "UNIT_ITER_CLOSE"; + + int amount = 1_000_000; + HgStoreSession session = getStoreSession(); + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); + } + + HgKvIterator iterator = null; + + /* server over, all -> close */ + iterator = session.scanIterator(tableName); + Assert.assertEquals(amount, HgStoreTestUtil.amountOf(iterator)); + + /* server over, one page, all -> close */ + iterator = session.scanIterator(tableName, 100_000); + Assert.assertEquals(100_000, HgStoreTestUtil.amountOf(iterator)); + + /* server over, two page, all -> close */ + iterator = session.scanIterator(tableName, 200_000); + Assert.assertEquals(200_000, HgStoreTestUtil.amountOf(iterator)); + + /* server not over, enough -> close */ + iterator = session.scanIterator(tableName); + iterator.next(); + iterator.close(); + + /* server not over, one page, enough -> close */ + iterator = session.scanIterator(tableName, 100_000); + iterator.next(); + iterator.close(); + + /* server not over, two page, enough -> close */ + iterator = session.scanIterator(tableName, 200_000); + iterator.next(); + iterator.close(); + + /* server not over, enough -> close */ + iterator = session.scanIterator(tableName); + iterator.next(); + iterator.close(); + iterator = session.scanIterator(tableName); + for (int i = 0; iterator.hasNext() && i < 500_000; i++) { + iterator.next(); + } + iterator.close(); + + /* server timeout, enough -> close */ + iterator = session.scanIterator(tableName); + iterator.next(); + // HgStoreTestUtil.sleeping(11000); + + try { + HgStoreTestUtil.amountOf(iterator); + } catch (Throwable t) { + HgStoreTestUtil.println("-- passed server waiting timeout --"); + } + + iterator.close(); + + } + + // @Test + public void paging() { + HgStoreTestUtil.println("--- test scanIterator_range ---"); + String graph = "UNIT/paging"; + String tableName = "UNIT_SCAN_PAGING"; + String keyName = "SCAN-PAGING"; + int keyAmt = 100; + HgStoreSession session = getStoreSession(graph); + + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); + } + + int count = 0; + int limit = 0; + int max = 99999; + HgKvIterator iterator = null; + + count = 0; + + iterator = session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey(keyName + "-000") + , HgStoreClientConst.EMPTY_OWNER_KEY + , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES + ); + //HgStoreTestUtil.println(iterator); + // Assert.assertEquals(100, HgStoreTestUtil.amountOf(iterator)); + List positionList = new LinkedList<>(); + while (iterator.hasNext()) { + HgStoreTestUtil.print((count++) + " "); + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.print(entry); + HgStoreTestUtil.print(" " + Arrays.toString(iterator.position()) + "\n"); + positionList.add(iterator.position()); + if (count >= max) break; + } + + + iterator = session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey(keyName + "-000") + , HgStoreClientConst.EMPTY_OWNER_KEY + , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES + ); + + byte[] position = positionList.get(50); + HgStoreTestUtil.println("seek: " + Arrays.toString(position)); + iterator.seek(position); + //println("amt after seek: "+HgStoreTestUtil.println(iterator)); + Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator)); + + } + + @Test + public void scanBatch() { + HgStoreTestUtil.println("--- test scanBatch ---"); + String tableName = "UNIT_SCAN_BATCH_1"; + String keyName = "SCAN-BATCH"; + int keyAmt = 10_000; + + HgStoreSession session = getStoreSession(); + + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), + keyAmt)) < keyAmt) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); + } + + Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + List> iterators = null; + List prefixes = null; + + List startList = Arrays.asList( + HgStoreTestUtil.toAllPartitionKey(keyName + "-001") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-003") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-005") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-007") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-009") + ); + + List endList = Arrays.asList( + HgStoreTestUtil.toAllPartitionKey(keyName + "-002") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-004") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-006") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-008") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-010") + ); + + List prefixList = Arrays.asList( + HgStoreTestUtil.toAllPartitionKey(keyName + "-001") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-002") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-003") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-004") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-005") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-006") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-007") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-008") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-009") + ); + + HgStoreTestUtil.println("-- test scan-batch all --"); + + HgScanQuery scanQuery = HgScanQuery.tableOf(tableName); + iterators = session.scanBatch(scanQuery); + Assert.assertEquals(3, iterators.size()); + + Assert.assertEquals(keyAmt, HgStoreTestUtil.println(iterators)); + + HgStoreTestUtil.println("-- test scan-batch prefix --"); + + + iterators = session.scanBatch( + HgScanQuery.prefixOf(tableName, prefixList) + ); + Assert.assertEquals(3, iterators.size()); + Assert.assertEquals(900, + iterators.parallelStream().mapToInt(e -> HgStoreTestUtil.amountOf(e)) + .sum() + ); + + HgStoreTestUtil.println("-- test scan-batch range --"); + + iterators = session.scanBatch(HgScanQuery.rangeOf(tableName, startList, endList)); + Assert.assertEquals(3, iterators.size()); + Assert.assertEquals( + HgStoreTestUtil.amountOf( + session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey(keyName + "-001") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-010") + ) + ) + , + iterators.parallelStream().mapToInt(e -> HgStoreTestUtil.amountOf(e)).sum() + ); + + HgStoreTestUtil.println("-- test scan-batch limit --"); + + int limit = 1; + iterators = session.scanBatch( + HgScanQuery.rangeOf(tableName, startList, endList) + .builder() + .setLimit(limit) + .build() + ); + + //HgStoreTestUtil.println(iterators); + Assert.assertEquals(iterators.size() * limit, + iterators.parallelStream().mapToInt(e -> HgStoreTestUtil.amountOf(e)) + .sum() + ); + + HgStoreTestUtil.println("-- test scan-batch multi-table --"); + if (HgStoreTestUtil.amountOf( + session.scanIterator("g+oe", HgStoreTestUtil.toAllPartitionKey(keyName), keyAmt)) < + keyAmt) { + HgStoreTestUtil.batchPut(session, "g+oe", keyName, keyAmt); + HgStoreTestUtil.batchPut(session, "g+ie", keyName, keyAmt); + } + + prefixes = Collections.singletonList(HgStoreTestUtil.toAllPartitionKey(keyName)); + + tableName = "g+oe,g+ie"; + iterators = session.scanBatch( + HgScanQuery.prefixOf(tableName, prefixes) + ); + + //HgStoreTestUtil.println(iterators); + Assert.assertEquals(keyAmt * 2, HgStoreTestUtil.amountIn(iterators)); + iterators = session.scanBatch( + HgScanQuery.prefixOf(tableName, prefixList) + ); + Assert.assertEquals(900 * 2, HgStoreTestUtil.amountIn(iterators)); + + HgStoreTestUtil.println("-- test scan-batch per-key-limit --"); + + tableName = "PER_KEY_LIMIT_TABLE"; + keyName = "PER_KEY_LIMIT"; + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), + keyAmt)) < keyAmt) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); + } + prefixes = Arrays.asList( + HgStoreTestUtil.toAllPartitionKey(keyName + "-01") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-02") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-03") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-04") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-05") + ); + + iterators = session.scanBatch( + HgScanQuery.prefixOf(tableName, prefixes).builder().setPerKeyLimit(1).build() + ); + //HgStoreTestUtil.println(iterators); + Assert.assertEquals(prefixes.size() * 3, HgStoreTestUtil.amountIn(iterators)); + + } + + // @Test + public void truncate() { + HgStoreTestUtil.println("--- test truncate ---"); + String graph = "graph_truncate"; + String tableName = "UNIT_TRUNCATE_1"; + String keyName = "KEY_TRUNCATE"; + + HgStoreSession session = getStoreSession(graph); + HgStoreTestUtil.batchPut(session, tableName, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + String tableName2 = "UNIT_TRUNCATE_2"; + HgStoreTestUtil.batchPut(session, tableName2, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); + + session.truncate(); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); + } + + // @Test + public void handle_table() { + HgStoreTestUtil.println("--- test table ---"); + + String tableName = "UNIT_TABLE_" + System.currentTimeMillis(); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + HgStoreSession session = getStoreSession(); + + HgStoreTestUtil.println("-- test createTable --"); + session.createTable(tableName); + Assert.assertTrue(session.existsTable(tableName)); + + HgStoreTestUtil.println("-- test deleteTable --"); + session.put(tableName, key, value); + Assert.assertEquals(HgStoreTestUtil.toStr(value), + HgStoreTestUtil.toStr(session.get(tableName, key))); + session.deleteTable(tableName); + Assert.assertNotEquals(HgStoreTestUtil.toStr(value), + HgStoreTestUtil.toStr(session.get(tableName, key))); + Assert.assertTrue(session.existsTable(tableName)); + + + HgStoreTestUtil.println("-- test dropTable --"); + Assert.assertTrue(session.dropTable(tableName)); + Assert.assertFalse(session.existsTable(tableName)); + + + HgStoreTestUtil.println("-- test existsTable --"); + Assert.assertFalse(session.existsTable(tableName)); + + } + + // @Test + public void tx() { + HgStoreTestUtil.println("--- test tx ---"); + HgStoreSession session = getStoreSession(); + String tableName = "UNIT_TABLE_TX"; + String keyPrefix = "TX"; + int keyAmt = 100; + + HgStoreTestUtil.println("-- unsupported tx operation --"); + HgStoreTestUtil.println("- tx deleteTable -"); + HgStoreTestUtil.batchPut(session, tableName, keyPrefix, keyAmt); + Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + session.beginTx(); + session.deleteTable(tableName); + session.commit(); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + HgStoreTestUtil.println("- tx dropTable -"); + HgStoreTestUtil.batchPut(session, tableName, keyPrefix, keyAmt); + Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + session.beginTx(); + session.dropTable(tableName); + session.commit(); + Assert.assertFalse(session.existsTable(tableName)); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + HgStoreTestUtil.println("- tx truncate -"); + HgStoreTestUtil.batchPut(session, tableName, keyPrefix, keyAmt); + Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + session.beginTx(); + session.truncate(); + session.commit(); + Assert.assertFalse(session.existsTable(tableName)); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + //TODO : add others + } + + //// @Test + public void scanIterator_WithNonePartition() { + HgStoreTestUtil.println("--- test scanIterator with none partition ---"); + int count = 0; + HgStoreSession session = getStoreSession(); + + for (int i = 0; i < 100; i++) { + HgKvIterator iterator = session.scanIterator("XXXXXXXX"); + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.println(entry); + } + } + + Assert.assertEquals(0, count); + + } + + //// @Test + public void repeatedly_parallel_scan() { + MetricX metrics = MetricX.ofStart(); + HgStoreTestUtil.repeatedly_test(100, () -> parallel_scan()); + metrics.end(); + + log.info("*************************************************"); + log.info("************* Scanning Completed **************"); + log.info("Total: {} sec.", metrics.past() / 1000); + log.info("Sum: {} sec.", MetricX.getIteratorWait() / 1000); + log.info("Amt: {} scans.", MetricX.getIteratorCount()); + log.info("Avg: {} ms.", MetricX.getIteratorWaitAvg()); + log.info("Max: {} ms.", MetricX.getIteratorWaitMax()); + log.info("Fail: {} times.", metrics.getFailureCount()); + log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize()); + log.info("*************************************************"); + + // runWaiting(); + } + + // @Test + public void parallel_scan() { + HgStoreTestUtil.println("--- test scanIterator in parallel ---"); + HgStoreTestUtil.parallel_test(100, () -> { + this.scan(); + }, t -> t.printStackTrace()); + } + + //// @Test + public void put_Benchmark() { + /*************** Put Benchmark **************/ + String tableName = "UNIT_PUT_BENCHMARK"; + String keyPrefix = "PUT-BENCHMARK"; + int amount = 1_000_000; + HgStoreSession session = getStoreSession(); + + int length = String.valueOf(amount).length(); + + session.beginTx(); + + long start = System.currentTimeMillis(); + for (int i = 0; i < amount; i++) { + HgOwnerKey key = HgStoreTestUtil.toOwnerKey( + keyPrefix + "-" + HgStoreTestUtil.padLeftZeros(String.valueOf(i), length)); + byte[] value = HgStoreTestUtil.toBytes(keyPrefix + "-V-" + i); + + session.put(tableName, key, value); + + if ((i + 1) % 100_000 == 0) { + HgStoreTestUtil.println("---------- " + (i + 1) + " --------"); + HgStoreTestUtil.println( + "Preparing took: " + (System.currentTimeMillis() - start) + " ms."); + session.commit(); + HgStoreTestUtil.println( + "Committing took: " + (System.currentTimeMillis() - start) + " ms."); + start = System.currentTimeMillis(); + session.beginTx(); + } + } + + if (session.isTx()) { + session.commit(); + } + + Assert.assertEquals(amount, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + + } + + //// @Test + public void put_Benchmark_Parallel() { + int threadsAmount = 100; + CountDownLatch countDownLatch = new CountDownLatch(threadsAmount); + + for (int j = 0; j < threadsAmount; j++) { + for (int i = 0; i < threadsAmount; i++) { + pool.submit(() -> { + this.put_Benchmark(); + // this.scanIterator_WithNonePartition(); + countDownLatch.countDown(); + }); + } + } + try { + countDownLatch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + } + + //// @Test + public void parallel_scanBatch() { + int threadsAmount = 50; + CountDownLatch countDownLatch = new CountDownLatch(threadsAmount); + + for (int j = 0; j < threadsAmount; j++) { + for (int i = 0; i < threadsAmount; i++) { + pool.submit(() -> { + try { + this.scanBatch(); + } catch (Throwable t) { + t.printStackTrace(); + } finally { + countDownLatch.countDown(); + } + }); + } + } + try { + countDownLatch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + } + + //// @Test + public void benchmark_scanBatch() { + HgStoreTestUtil.println("--- Benchmark scanBatch ---"); + String tableName = "Benchmark_SCAN_BATCH"; + String keyName = "SCAN-BATCH"; + int keyAmt = 10_000_000; + + HgStoreSession session = getStoreSession(); + + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); + } + HgStoreTestUtil.println("-- Starting scan --"); + // while (true) { + + MetricX metrics = MetricX.ofStart(); + //HgStoreTestUtil.println(session.scanIterator(tableName)); + List> iterators = session.scanBatch(HgScanQuery.tableOf(tableName)); + //HgStoreTestUtil.println(iterators); + //Assert.assertEquals(keyAmt, HgStoreTestUtil.amountIn(iterators)); + HgStoreTestUtil.amountIn(iterators); + metrics.end(); + log.info("*************************************************"); + log.info("*********** Batch Scan Completed **************"); + log.info("Total : {} (sec)", metrics.past() / 1000); + log.info(" Sum : {} (sec)", MetricX.getIteratorWait() / 1000); + log.info(" Amt : {} (scans).", MetricX.getIteratorCount()); + log.info(" Avg : {} (ms)", MetricX.getIteratorWaitAvg()); + log.info(" Max : {} (ms)", MetricX.getIteratorWaitMax()); + log.info(" Fail : {} (times)", metrics.getFailureCount()); + log.info(" Page : {} (KVs)", HgStoreClientConfig.of().getNetKvScannerPageSize()); + log.info("Iters : {}", iterators.size()); + log.info("*************************************************"); + HgStoreTestUtil.sleeping(100); + + // } + } + + //// @Test + public void benchmark_scan() { + /*************** test no limit, with 10 millions **************/ + String tableName = "UNIT_HUGE"; + String keyName = "SCAN-HUGE"; + int amount = 10_000_000; + int max = 10_000_000; + HgStoreSession session = getStoreSession(); + + /*Initialization*/ + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); + } + + MetricX metricX = MetricX.ofStart(); + + int count = 0; + HgKvIterator iterator = session.scanIterator(tableName); + //HgStoreTestUtil.println(iterator, e -> (e % (amount / 100) == 0)); + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % (amount / 10) == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + + metricX.end(); + + log.info("*************************************************"); + log.info("************* Benchmark Completed *************"); + log.info("Keys: {}", count); + log.info("Waiting: {} ms.", MetricX.getIteratorWait()); + log.info("Total: {} ms.", metricX.past()); + log.info("Iterator: [{}]", iterator.getClass().getSimpleName()); + log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize()); + log.info("*************************************************"); + + Assert.assertEquals(amount, count); + } + + + //// @Test + public void extreme_scan_close() { + /*************** test close **************/ + String tableName = "UNIT_ITER_CLOSE_EXTREME"; + String keyName = "UNIT_ITER_CLOSE_EXTREME"; + int amount = 1_000_000; + HgStoreSession session = getStoreSession(); + + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); + } + HgKvIterator iterator = null; + /* server not over, enough, extreme -> close */ + for (int i = 0; i <= 10_000; i++) { + iterator = session.scanIterator(tableName); + iterator.next(); + iterator.close(); + HgStoreTestUtil.println("extreme loop: " + i); + } + HgStoreTestUtil.runWaiting(); + } + + //// @Test + public void parallel_scan_close() { + HgStoreTestUtil.parallel_test(10, () -> this.scan_close(), t -> t.printStackTrace()); + } + + //// @Test + public void repeat_parallel_scan_close() { + HgStoreTestUtil.repeatedly_test(1000, () -> this.parallel_scan_close()); + HgStoreTestUtil.runWaiting(); + } + + //// @Test + public void parallel_huge_scan() { + int threadsAmount = 3; + CountDownLatch countDownLatch = new CountDownLatch(threadsAmount); + ExecutorService poll = Executors.newFixedThreadPool(threadsAmount); + + for (int i = 0; i < threadsAmount; i++) { + poll.submit(() -> { + this.benchmark_scan(); + countDownLatch.countDown(); + }); + } + + try { + countDownLatch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + } + } \ No newline at end of file diff --git a/hg-store-client/src/test/resources/log4j2.xml b/hg-store-client/src/test/resources/log4j2.xml index bd88bec75..0c37f08ea 100644 --- a/hg-store-client/src/test/resources/log4j2.xml +++ b/hg-store-client/src/test/resources/log4j2.xml @@ -1,102 +1,102 @@ - - - - - - - - logs - hg-store-client - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + logs + hg-store-client + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchGrpcClosure.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchGrpcClosure.java index dca4c4417..80ea6bdcf 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchGrpcClosure.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchGrpcClosure.java @@ -1,212 +1,212 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.grpc; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; -import java.util.function.Function; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.hugegraph.store.grpc.common.ResCode; -import org.apache.hugegraph.store.grpc.common.ResStatus; -import org.apache.hugegraph.store.grpc.session.FeedbackRes; -import org.apache.hugegraph.store.grpc.session.PartitionFaultResponse; -import org.apache.hugegraph.store.grpc.session.PartitionFaultType; -import org.apache.hugegraph.store.grpc.session.PartitionLeader; -import org.apache.hugegraph.store.raft.RaftClosure; -import org.apache.hugegraph.store.util.HgRaftError; - -import com.alipay.sofa.jraft.Status; - -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; - -/** - * 批量处理的grpc回调封装类 - * - * @param - */ -@Slf4j -class BatchGrpcClosure { - private final CountDownLatch countDownLatch; - private final List errorStatus; - private final List results; - private final Map leaderMap; - - public BatchGrpcClosure(int count) { - countDownLatch = new CountDownLatch(count); - errorStatus = Collections.synchronizedList(new ArrayList<>()); - results = Collections.synchronizedList(new ArrayList<>()); - leaderMap = new ConcurrentHashMap<>(); - } - - public RaftClosure newRaftClosure() { - return new GrpcClosure() { - @Override - public void run(Status status) { - countDownLatch.countDown(); - if (status.isOk()) { - results.add(this.getResult()); - } else { - leaderMap.putAll(this.getLeaderMap()); - errorStatus.add(status); - } - } - }; - } - - public RaftClosure newRaftClosure(Consumer ok) { - return new GrpcClosure() { - @Override - public void run(Status status) { - countDownLatch.countDown(); - if (status.isOk()) { - results.add(this.getResult()); - } else { - leaderMap.putAll(this.getLeaderMap()); - errorStatus.add(status); - } - ok.accept(status); - } - }; - } - - /** - * 不使用计数器latch - * - * @return - */ - public RaftClosure newClosureNoLatch() { - return new GrpcClosure() { - @Override - public void run(Status status) { - if (status.isOk()) { - results.add(this.getResult()); - } else { - leaderMap.putAll(this.getLeaderMap()); - errorStatus.add(status); - } - } - }; - } - - - public PartitionFaultResponse getErrorResponse() { - PartitionFaultResponse errorResponse; - - if (leaderMap.size() > 0) { - PartitionFaultResponse.Builder partitionFault = - PartitionFaultResponse.newBuilder().setFaultType( - PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LEADER); - leaderMap.forEach((k, v) -> { - partitionFault.addPartitionLeaders(PartitionLeader.newBuilder() - .setPartitionId(k) - .setLeaderId(v).build()); - }); - errorResponse = partitionFault.build(); - } else { - PartitionFaultType faultType = PartitionFaultType.PARTITION_FAULT_TYPE_UNKNOWN; - switch (HgRaftError.forNumber(errorStatus.get(0).getCode())) { - case NOT_LEADER: - faultType = PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LEADER; - break; - case WAIT_LEADER_TIMEOUT: - faultType = PartitionFaultType.PARTITION_FAULT_TYPE_WAIT_LEADER_TIMEOUT; - break; - case NOT_LOCAL: - faultType = PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LOCAL; - break; - default: - log.error("Unmatchable errorStatus: " + errorStatus); - } - errorResponse = PartitionFaultResponse.newBuilder().setFaultType(faultType).build(); - } - return errorResponse; - } - - public String getErrorMsg() { - StringBuilder builder = new StringBuilder(); - errorStatus.forEach(status -> { - if (!status.isOk()) { - builder.append(status.getErrorMsg()); - builder.append("\n"); - } - }); - return builder.toString(); - } - - /** - * 等待raft执行结束，返回结果给grpc - */ - public void waitFinish(StreamObserver observer, Function, V> ok, long timeout) { - try { - countDownLatch.await(timeout, TimeUnit.MILLISECONDS); - - if (errorStatus.isEmpty()) { // 没有错误时，合并结果 - observer.onNext(ok.apply(results)); - } else { - observer.onNext((V) FeedbackRes.newBuilder() - .setStatus(ResStatus.newBuilder() - .setCode(ResCode.RES_CODE_FAIL) - .setMsg(getErrorMsg())) - .setPartitionFaultResponse(this.getErrorResponse()) - .build()); - } - } catch (InterruptedException e) { - log.error("waitFinish exception: ", e); - observer.onNext((V) FeedbackRes.newBuilder() - .setStatus(ResStatus.newBuilder() - .setCode(ResCode.RES_CODE_FAIL) - .setMsg(e.getLocalizedMessage()) - .build()).build()); - } - observer.onCompleted(); - } - - /** - * 从多个结果中选择一个错误的结果返回，如果没有错误，返回第一个 - */ - public FeedbackRes selectError(List results) { - if (!CollectionUtils.isEmpty(results)) { - AtomicReference res = new AtomicReference<>(results.get(0)); - results.forEach(e -> { - try { - if (e.getStatus().getCode() != ResCode.RES_CODE_OK) { - res.set(e); - } - } catch (Exception ex) { - log.error("{}", ex); - } - }); - return res.get(); - } else { - return FeedbackRes.newBuilder() - .setStatus(ResStatus.newBuilder() - .setCode(ResCode.RES_CODE_OK).build()) - .build(); - } - - } +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.hugegraph.store.grpc.common.ResCode; +import org.apache.hugegraph.store.grpc.common.ResStatus; +import org.apache.hugegraph.store.grpc.session.FeedbackRes; +import org.apache.hugegraph.store.grpc.session.PartitionFaultResponse; +import org.apache.hugegraph.store.grpc.session.PartitionFaultType; +import org.apache.hugegraph.store.grpc.session.PartitionLeader; +import org.apache.hugegraph.store.raft.RaftClosure; +import org.apache.hugegraph.store.util.HgRaftError; + +import com.alipay.sofa.jraft.Status; + +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; + +/** + * 批量处理的grpc回调封装类 + * + * @param + */ +@Slf4j +class BatchGrpcClosure { + private final CountDownLatch countDownLatch; + private final List errorStatus; + private final List results; + private final Map leaderMap; + + public BatchGrpcClosure(int count) { + countDownLatch = new CountDownLatch(count); + errorStatus = Collections.synchronizedList(new ArrayList<>()); + results = Collections.synchronizedList(new ArrayList<>()); + leaderMap = new ConcurrentHashMap<>(); + } + + public RaftClosure newRaftClosure() { + return new GrpcClosure() { + @Override + public void run(Status status) { + countDownLatch.countDown(); + if (status.isOk()) { + results.add(this.getResult()); + } else { + leaderMap.putAll(this.getLeaderMap()); + errorStatus.add(status); + } + } + }; + } + + public RaftClosure newRaftClosure(Consumer ok) { + return new GrpcClosure() { + @Override + public void run(Status status) { + countDownLatch.countDown(); + if (status.isOk()) { + results.add(this.getResult()); + } else { + leaderMap.putAll(this.getLeaderMap()); + errorStatus.add(status); + } + ok.accept(status); + } + }; + } + + /** + * 不使用计数器latch + * + * @return + */ + public RaftClosure newClosureNoLatch() { + return new GrpcClosure() { + @Override + public void run(Status status) { + if (status.isOk()) { + results.add(this.getResult()); + } else { + leaderMap.putAll(this.getLeaderMap()); + errorStatus.add(status); + } + } + }; + } + + + public PartitionFaultResponse getErrorResponse() { + PartitionFaultResponse errorResponse; + + if (leaderMap.size() > 0) { + PartitionFaultResponse.Builder partitionFault = + PartitionFaultResponse.newBuilder().setFaultType( + PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LEADER); + leaderMap.forEach((k, v) -> { + partitionFault.addPartitionLeaders(PartitionLeader.newBuilder() + .setPartitionId(k) + .setLeaderId(v).build()); + }); + errorResponse = partitionFault.build(); + } else { + PartitionFaultType faultType = PartitionFaultType.PARTITION_FAULT_TYPE_UNKNOWN; + switch (HgRaftError.forNumber(errorStatus.get(0).getCode())) { + case NOT_LEADER: + faultType = PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LEADER; + break; + case WAIT_LEADER_TIMEOUT: + faultType = PartitionFaultType.PARTITION_FAULT_TYPE_WAIT_LEADER_TIMEOUT; + break; + case NOT_LOCAL: + faultType = PartitionFaultType.PARTITION_FAULT_TYPE_NOT_LOCAL; + break; + default: + log.error("Unmatchable errorStatus: " + errorStatus); + } + errorResponse = PartitionFaultResponse.newBuilder().setFaultType(faultType).build(); + } + return errorResponse; + } + + public String getErrorMsg() { + StringBuilder builder = new StringBuilder(); + errorStatus.forEach(status -> { + if (!status.isOk()) { + builder.append(status.getErrorMsg()); + builder.append("\n"); + } + }); + return builder.toString(); + } + + /** + * 等待raft执行结束，返回结果给grpc + */ + public void waitFinish(StreamObserver observer, Function, V> ok, long timeout) { + try { + countDownLatch.await(timeout, TimeUnit.MILLISECONDS); + + if (errorStatus.isEmpty()) { // 没有错误时，合并结果 + observer.onNext(ok.apply(results)); + } else { + observer.onNext((V) FeedbackRes.newBuilder() + .setStatus(ResStatus.newBuilder() + .setCode(ResCode.RES_CODE_FAIL) + .setMsg(getErrorMsg())) + .setPartitionFaultResponse(this.getErrorResponse()) + .build()); + } + } catch (InterruptedException e) { + log.error("waitFinish exception: ", e); + observer.onNext((V) FeedbackRes.newBuilder() + .setStatus(ResStatus.newBuilder() + .setCode(ResCode.RES_CODE_FAIL) + .setMsg(e.getLocalizedMessage()) + .build()).build()); + } + observer.onCompleted(); + } + + /** + * 从多个结果中选择一个错误的结果返回，如果没有错误，返回第一个 + */ + public FeedbackRes selectError(List results) { + if (!CollectionUtils.isEmpty(results)) { + AtomicReference res = new AtomicReference<>(results.get(0)); + results.forEach(e -> { + try { + if (e.getStatus().getCode() != ResCode.RES_CODE_OK) { + res.set(e); + } + } catch (Exception ex) { + log.error("{}", ex); + } + }); + return res.get(); + } else { + return FeedbackRes.newBuilder() + .setStatus(ResStatus.newBuilder() + .setCode(ResCode.RES_CODE_OK).build()) + .build(); + } + + } } \ No newline at end of file diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchScanIterator.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchScanIterator.java index 516a20164..978d6ac53 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchScanIterator.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/BatchScanIterator.java @@ -1,160 +1,160 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.grpc; - -import java.util.NoSuchElementException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Supplier; - -import javax.annotation.concurrent.NotThreadSafe; - -import org.apache.hugegraph.pd.common.KVPair; -import org.apache.hugegraph.rocksdb.access.ScanIterator; -import org.apache.hugegraph.store.node.util.HgAssert; -import org.apache.hugegraph.store.node.util.HgStoreConst; - -/** - * @author lynn.bond@hotmail.com on 2022/2/28 - * @version 0.3.0 added limit support - */ -@NotThreadSafe -public final class BatchScanIterator implements ScanIterator { - private final Supplier> batchSupplier; - private final Supplier limitSupplier; - - private ScanIterator iterator; - private boolean hasNext = false; - private long curCount; - private long curLimit; - private final AtomicBoolean closed = new AtomicBoolean(); - - public static BatchScanIterator of( - Supplier> iteratorSupplier, - Supplier limitSupplier) { - HgAssert.isArgumentNotNull(iteratorSupplier, "iteratorSupplier"); - HgAssert.isArgumentNotNull(limitSupplier, "limitSupplier"); - return new BatchScanIterator(iteratorSupplier, limitSupplier); - } - - private BatchScanIterator(Supplier> iteratorSupplier, - Supplier limitSupplier) { - this.batchSupplier = iteratorSupplier; - this.limitSupplier = limitSupplier; - } - - private ScanIterator getIterator() { - ScanIterator buf; - int count = 0; - this.curCount = 0L; - - do { - buf = this.batchSupplier.get().getValue(); - - if (buf == null) { - break; - } - - if (!buf.hasNext()) { - buf.close(); - buf = null; - } - - if (++count == Integer.MAX_VALUE) { - throw new RuntimeException("Do loop times more than Integer.MAX_VALUE"); - } - - } while (buf == null); - - if (buf != null) { - Long limit = this.limitSupplier.get(); - - if (limit == null || limit <= 0) { - this.curLimit = Integer.MAX_VALUE; - } else { - this.curLimit = limit; - } - - } - - return buf; - - } - - @Override - public boolean hasNext() { - - if (this.iterator == null) { - this.iterator = this.getIterator(); - } else if (!this.iterator.hasNext()) { - this.iterator.close(); - this.iterator = this.getIterator(); - } else if (this.curCount == this.curLimit) { - this.iterator.close(); - this.iterator = this.getIterator(); - } - - if (this.iterator == null) { - return false; - } else { - this.hasNext = true; - return true; - } - } - - @Override - public T next() { - if (this.hasNext) { - this.hasNext = false; - } else { - if (!this.hasNext()) { - throw new NoSuchElementException(); - } - } - - this.curCount++; - - return this.iterator.next(); - } - - @Override - public void close() { - if (!this.closed.getAndSet(true)) { - if (this.iterator != null) { - this.iterator.close(); - } - } - } - - @Override - public boolean isValid() { - throw new UnsupportedOperationException(); - } - - @Override - public long count() { - throw new UnsupportedOperationException(); - } - - @Override - public byte[] position() { - if (this.iterator != null) { - return this.iterator.position(); - } - return HgStoreConst.EMPTY_BYTES; - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +import java.util.NoSuchElementException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; + +import javax.annotation.concurrent.NotThreadSafe; + +import org.apache.hugegraph.pd.common.KVPair; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.node.util.HgAssert; +import org.apache.hugegraph.store.node.util.HgStoreConst; + +/** + * @author lynn.bond@hotmail.com on 2022/2/28 + * @version 0.3.0 added limit support + */ +@NotThreadSafe +public final class BatchScanIterator implements ScanIterator { + private final Supplier> batchSupplier; + private final Supplier limitSupplier; + + private ScanIterator iterator; + private boolean hasNext = false; + private long curCount; + private long curLimit; + private final AtomicBoolean closed = new AtomicBoolean(); + + public static BatchScanIterator of( + Supplier> iteratorSupplier, + Supplier limitSupplier) { + HgAssert.isArgumentNotNull(iteratorSupplier, "iteratorSupplier"); + HgAssert.isArgumentNotNull(limitSupplier, "limitSupplier"); + return new BatchScanIterator(iteratorSupplier, limitSupplier); + } + + private BatchScanIterator(Supplier> iteratorSupplier, + Supplier limitSupplier) { + this.batchSupplier = iteratorSupplier; + this.limitSupplier = limitSupplier; + } + + private ScanIterator getIterator() { + ScanIterator buf; + int count = 0; + this.curCount = 0L; + + do { + buf = this.batchSupplier.get().getValue(); + + if (buf == null) { + break; + } + + if (!buf.hasNext()) { + buf.close(); + buf = null; + } + + if (++count == Integer.MAX_VALUE) { + throw new RuntimeException("Do loop times more than Integer.MAX_VALUE"); + } + + } while (buf == null); + + if (buf != null) { + Long limit = this.limitSupplier.get(); + + if (limit == null || limit <= 0) { + this.curLimit = Integer.MAX_VALUE; + } else { + this.curLimit = limit; + } + + } + + return buf; + + } + + @Override + public boolean hasNext() { + + if (this.iterator == null) { + this.iterator = this.getIterator(); + } else if (!this.iterator.hasNext()) { + this.iterator.close(); + this.iterator = this.getIterator(); + } else if (this.curCount == this.curLimit) { + this.iterator.close(); + this.iterator = this.getIterator(); + } + + if (this.iterator == null) { + return false; + } else { + this.hasNext = true; + return true; + } + } + + @Override + public T next() { + if (this.hasNext) { + this.hasNext = false; + } else { + if (!this.hasNext()) { + throw new NoSuchElementException(); + } + } + + this.curCount++; + + return this.iterator.next(); + } + + @Override + public void close() { + if (!this.closed.getAndSet(true)) { + if (this.iterator != null) { + this.iterator.close(); + } + } + } + + @Override + public boolean isValid() { + throw new UnsupportedOperationException(); + } + + @Override + public long count() { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] position() { + if (this.iterator != null) { + return this.iterator.position(); + } + return HgStoreConst.EMPTY_BYTES; + } +} diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GRpcServerConfig.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GRpcServerConfig.java index 93549daa0..4519b3450 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GRpcServerConfig.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GRpcServerConfig.java @@ -1,46 +1,46 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.grpc; - -import org.apache.hugegraph.store.node.AppConfig; -import org.apache.hugegraph.store.node.util.HgExecutorUtil; -import org.lognet.springboot.grpc.GRpcServerBuilderConfigurer; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import io.grpc.ServerBuilder; - -/** - * @author lynn.bond@hotmail.com on 2022/3/4 - */ -@Component -public class GRpcServerConfig extends GRpcServerBuilderConfigurer { - public final static String EXECUTOR_NAME = "hg-grpc"; - @Autowired - private AppConfig appConfig; - - @Override - public void configure(ServerBuilder serverBuilder) { - AppConfig.ThreadPoolGrpc grpc = appConfig.getThreadPoolGrpc(); - serverBuilder.executor( - HgExecutorUtil.createExecutor(EXECUTOR_NAME, grpc.getCore(), grpc.getMax(), - grpc.getQueue()) - ); - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +import org.apache.hugegraph.store.node.AppConfig; +import org.apache.hugegraph.store.node.util.HgExecutorUtil; +import org.lognet.springboot.grpc.GRpcServerBuilderConfigurer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import io.grpc.ServerBuilder; + +/** + * @author lynn.bond@hotmail.com on 2022/3/4 + */ +@Component +public class GRpcServerConfig extends GRpcServerBuilderConfigurer { + public final static String EXECUTOR_NAME = "hg-grpc"; + @Autowired + private AppConfig appConfig; + + @Override + public void configure(ServerBuilder serverBuilder) { + AppConfig.ThreadPoolGrpc grpc = appConfig.getThreadPoolGrpc(); + serverBuilder.executor( + HgExecutorUtil.createExecutor(EXECUTOR_NAME, grpc.getCore(), grpc.getMax(), + grpc.getQueue()) + ); + } + +} diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcClosure.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcClosure.java index 021a0ad81..c054c973d 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcClosure.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcClosure.java @@ -1,72 +1,72 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.grpc; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.hugegraph.store.grpc.session.FeedbackRes; -import org.apache.hugegraph.store.raft.RaftClosure; - -import io.grpc.stub.StreamObserver; - -/** - * @author lynn.bond@hotmail.com on 2022/1/27 - */ - -abstract class GrpcClosure implements RaftClosure { - private V result; - private final Map leaderMap = new HashMap<>(); - - public V getResult() { - return result; - } - - public Map getLeaderMap() { - return leaderMap; - } - - @Override - public void onLeaderChanged(Integer partId, Long storeId) { - leaderMap.put(partId, storeId); - } - - public void setResult(V result) { - this.result = result; - } - - /** - * 设置输出结果给raftClosure，对于Follower来说，raftClosure为空 - */ - public static void setResult(RaftClosure raftClosure, V result) { - GrpcClosure closure = (GrpcClosure) raftClosure; - if (closure != null) { - closure.setResult(result); - } - } - - public static RaftClosure newRaftClosure(StreamObserver observer) { - BatchGrpcClosure wrap = new BatchGrpcClosure(0); - return wrap.newRaftClosure(s -> { - wrap.waitFinish(observer, r -> { - return (V) wrap.selectError((List) r); - }, 0); - }); - } +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hugegraph.store.grpc.session.FeedbackRes; +import org.apache.hugegraph.store.raft.RaftClosure; + +import io.grpc.stub.StreamObserver; + +/** + * @author lynn.bond@hotmail.com on 2022/1/27 + */ + +abstract class GrpcClosure implements RaftClosure { + private V result; + private final Map leaderMap = new HashMap<>(); + + public V getResult() { + return result; + } + + public Map getLeaderMap() { + return leaderMap; + } + + @Override + public void onLeaderChanged(Integer partId, Long storeId) { + leaderMap.put(partId, storeId); + } + + public void setResult(V result) { + this.result = result; + } + + /** + * 设置输出结果给raftClosure，对于Follower来说，raftClosure为空 + */ + public static void setResult(RaftClosure raftClosure, V result) { + GrpcClosure closure = (GrpcClosure) raftClosure; + if (closure != null) { + closure.setResult(result); + } + } + + public static RaftClosure newRaftClosure(StreamObserver observer) { + BatchGrpcClosure wrap = new BatchGrpcClosure(0); + return wrap.newRaftClosure(s -> { + wrap.waitFinish(observer, r -> { + return (V) wrap.selectError((List) r); + }, 0); + }); + } } \ No newline at end of file diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java index 689a30caa..de26adae4 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java @@ -1,409 +1,409 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.grpc; - -import java.util.List; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantLock; - -import javax.annotation.concurrent.NotThreadSafe; - -import org.apache.hugegraph.rocksdb.access.RocksDBSession; -import org.apache.hugegraph.rocksdb.access.ScanIterator; -import org.apache.hugegraph.store.grpc.common.Kv; -import org.apache.hugegraph.store.grpc.stream.KvPageRes; -import org.apache.hugegraph.store.grpc.stream.ScanCondition; -import org.apache.hugegraph.store.grpc.stream.ScanQueryRequest; -import org.apache.hugegraph.store.grpc.stream.ScanStreamBatchReq; -import org.apache.hugegraph.store.node.util.Base58; -import org.apache.hugegraph.store.node.util.HgAssert; -import org.apache.hugegraph.store.node.util.HgGrpc; -import org.apache.hugegraph.store.node.util.HgStoreConst; -import org.apache.hugegraph.store.node.util.HgStoreNodeUtil; - -import com.google.protobuf.ByteString; - -import io.grpc.Status; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; - -/** - * @author lynn.bond@hotmail.com created on 2022/03/27 - * @version 3.6.0 - */ -@Slf4j -public class ScanBatchResponse3 { - private final static long DEFAULT_PACKAGE_SIZE = 10_000; - private final static int MAX_NOT_RECEIPT = 10; - - public static StreamObserver of(StreamObserver responseObserver, - HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) { - HgAssert.isArgumentNotNull(responseObserver, "responseObserver"); - HgAssert.isArgumentNotNull(wrapper, "wrapper"); - return new Broker(responseObserver, wrapper, executor); - } - - private enum OrderState { - NEW(0), - WORKING(1);//, PAUSE(2), COMPLETE(10); - int value; - - OrderState(int value) { - this.value = value; - } - } - - /*** Broker ***/ - private static class Broker implements StreamObserver { - private final StreamObserver responseObserver; - private final HgStoreWrapperEx wrapper; - private final ThreadPoolExecutor executor; - private String graph; - - private final OrderManager manager = new OrderManager(); - - Broker(StreamObserver responseObserver, HgStoreWrapperEx wrapper, - ThreadPoolExecutor executor) { - this.responseObserver = responseObserver; - this.wrapper = wrapper; - this.executor = executor; - } - - @Override - public void onNext(ScanStreamBatchReq request) { - this.handleHeader(request); - switch (request.getQueryCase()) { - case QUERY_REQUEST: - this.makeADeal(request.getQueryRequest()); - break; - case RECEIPT_REQUEST: - this.manager.receipt(request.getReceiptRequest().getTimes()); - break; - case CANCEL_REQUEST: - this.manager.finished(); - break; - default: - responseObserver.onError( - HgGrpc.toErr("Unsupported sub-request: [ " + request + " ]")); - } - } - - @Override - public void onError(Throwable t) { - log.warn(t.getMessage()); - this.manager.breakdown(); - } - - @Override - public void onCompleted() { - this.manager.finished(); - } - - private void handleHeader(ScanStreamBatchReq request) { - if (this.graph == null) { - this.graph = request.getHeader().getGraph(); - } - } - - private void makeADeal(ScanQueryRequest request) { - String deliverId = ""; - if (log.isDebugEnabled()) { - List conditions = request.getConditionList(); - if (conditions.size() > 0) { - ScanCondition c = conditions.get(0); - if (c.getPrefix() != null && c.getPrefix().size() > 0) { - deliverId = Base58.encode(c.getPrefix().toByteArray()); - log.info("[ANALYSIS DEAL] [{}] prefixLength: {}", deliverId, - conditions.size()); - } - - } - } - - OrderDeliverer deliverer = new OrderDeliverer(deliverId, this.responseObserver); - OrderWorker worker = new OrderWorker( - request.getLimit(), - request.getPageSize(), - ScanUtil.getIterator(this.graph, request, this.wrapper), - deliverer, - this.executor); - - this.manager.deal(worker, deliverer); - } - - } - - @NotThreadSafe - private static class OrderManager { - OrderState state = OrderState.NEW; - OrderWorker worker; - OrderDeliverer deliverer; - - synchronized void deal(OrderWorker worker, OrderDeliverer deliverer) { - if (log.isDebugEnabled()) log.debug("Receiving query request."); - if (this.state == OrderState.NEW) { - this.worker = worker; - this.deliverer = deliverer; - this.worker.hereWeGo(); - this.state = OrderState.WORKING; - } - } - - synchronized void receipt(int receiptTimes) { - if (log.isDebugEnabled()) log.debug("Receiving receipt request."); - this.worker.setReceipt(receiptTimes); - } - - synchronized void finished() { - if (log.isDebugEnabled()) log.debug("Receiving finished request."); -/* if (this.state.value > OrderState.NEW.value - && this.state.value < OrderState.COMPLETE.value) { - this.state = OrderState.COMPLETE; - }*/ - this.breakdown(); - } - - synchronized void breakdown() { - if (this.worker != null) { - this.worker.breakdown(); - } - } - } - - private static class OrderDeliverer { - private final StreamObserver responseObserver; - private final AtomicBoolean finishFlag = new AtomicBoolean(); - private final String delivererId; - private final AtomicLong count = new AtomicLong(); - - OrderDeliverer(String delivererId, StreamObserver responseObserver) { - this.responseObserver = responseObserver; - this.delivererId = delivererId; - } - - void deliver(KvPageRes.Builder dataBuilder, int times, boolean isOver) { - if (this.finishFlag.get()) { - return; - } - count.addAndGet(dataBuilder.getDataCount()); - this.responseObserver.onNext(dataBuilder.setOver(isOver).setTimes(times).build()); - if (log.isDebugEnabled()) log.debug("deliver times : {}, over: {}", times, isOver); - - if (isOver) { - if (log.isDebugEnabled()) { - if (delivererId != null && !delivererId.isEmpty()) { - log.debug("[ANALYSIS OVER] [{}] count: {}, times: {}", delivererId, count, - times); - } - } - this.finish(); - } - } - - void finish() { - if (!finishFlag.getAndSet(true)) { - this.responseObserver.onCompleted(); - } - } - - void error(String msg) { - if (!finishFlag.getAndSet(true)) { - this.responseObserver.onError(HgGrpc.toErr(msg)); - } - } - - void error(String msg, Throwable t) { - if (!finishFlag.getAndSet(true)) { - this.responseObserver.onError(HgGrpc.toErr(Status.INTERNAL, - msg, t)); - } - } - } - - /*** Worker ***/ - private static class OrderWorker { - private final ScanIterator iterator; - private final OrderDeliverer deliverer; - private final AtomicBoolean pauseFlag = new AtomicBoolean(); - private final AtomicBoolean completeFlag = new AtomicBoolean(); - private final ReentrantLock workingLock = new ReentrantLock(); - private final AtomicBoolean isWorking = new AtomicBoolean(); - private final AtomicBoolean breakdown = new AtomicBoolean(); - private final AtomicInteger receiptTimes = new AtomicInteger(); - private final AtomicInteger curTimes = new AtomicInteger(); - private final ThreadPoolExecutor executor; - private final long limit; - private long packageSize; - private long counter; - - OrderWorker(long limit, long packageSize, ScanIterator iterator, OrderDeliverer deliverer, - ThreadPoolExecutor executor) { - this.limit = limit; - this.packageSize = packageSize; - this.iterator = iterator; - this.deliverer = deliverer; - this.executor = executor; - - if (this.packageSize <= 0) { - this.packageSize = DEFAULT_PACKAGE_SIZE; - log.warn( - "As page-Size is less than or equals 0, default package-size was " + - "effective.[ {} ]", - DEFAULT_PACKAGE_SIZE); - } - - } - - void hereWeGo() { - if (this.completeFlag.get()) { - log.warn("job complete."); - return; - } - if (this.isWorking.get()) { - log.warn("has been working."); - return; - } - - if (this.workingLock.isLocked()) { - log.warn("working now"); - return; - } - - executor.execute(() -> working()); - Thread.yield(); - } - - void setReceipt(int times) { - this.receiptTimes.set(times); - this.continueWorking(); - } - - boolean checkContinue() { - return (this.curTimes.get() - this.receiptTimes.get() < MAX_NOT_RECEIPT); - } - - void continueWorking() { - if (this.checkContinue()) { - synchronized (this.iterator) { - this.iterator.notify(); - } - } - } - - void breakdown() { - this.breakdown.set(true); - synchronized (this.iterator) { - this.iterator.notify(); - } - } - - private void working() { - if (this.isWorking.getAndSet(true)) { - return; - } - - this.workingLock.lock(); - - try { - synchronized (this.iterator) { - KvPageRes.Builder dataBuilder = KvPageRes.newBuilder(); - Kv.Builder kvBuilder = Kv.newBuilder(); - long packageCount = 0; - - while (iterator.hasNext()) { - if (++this.counter > limit) { - this.completeFlag.set(true); - break; - } - - if (++packageCount > packageSize) { - - if (this.breakdown.get()) { - break; - } - - deliverer.deliver(dataBuilder, curTimes.incrementAndGet(), false); - Thread.yield(); - - if (!this.checkContinue()) { - long start = System.currentTimeMillis(); - iterator.wait( - HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * - 1000); - - if (System.currentTimeMillis() - start - >= - HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * 1000) { - throw new TimeoutException("Waiting continue more than " - + - HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS + - " seconds."); - } - - if (this.breakdown.get()) { - break; - } - - } - - packageCount = 1; - dataBuilder = KvPageRes.newBuilder(); - } - - RocksDBSession.BackendColumn col = iterator.next(); - - dataBuilder.addData(kvBuilder - .setKey(ByteString.copyFrom(col.name)) - .setValue(ByteString.copyFrom(col.value)) - .setCode(HgStoreNodeUtil.toInt( - iterator.position())) -//position == partition-id. - ); - - } - - this.completeFlag.set(true); - - deliverer.deliver(dataBuilder, curTimes.incrementAndGet(), true); - - } - - } catch (InterruptedException e) { - log.error("Interrupted waiting of iterator, canceled while.", e); - this.deliverer.error("Failed to finish scanning, cause by InterruptedException."); - } catch (TimeoutException t) { - log.info(t.getMessage()); - this.deliverer.error("Sever waiting exceeded [" - + HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS + - "] seconds."); - } catch (Throwable t) { - log.error("Failed to do while for scanning, cause by:", t); - this.deliverer.error("Failed to finish scanning ", t); - } finally { - this.workingLock.unlock(); - this.iterator.close(); - } - } - - } - +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; + +import javax.annotation.concurrent.NotThreadSafe; + +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.grpc.common.Kv; +import org.apache.hugegraph.store.grpc.stream.KvPageRes; +import org.apache.hugegraph.store.grpc.stream.ScanCondition; +import org.apache.hugegraph.store.grpc.stream.ScanQueryRequest; +import org.apache.hugegraph.store.grpc.stream.ScanStreamBatchReq; +import org.apache.hugegraph.store.node.util.Base58; +import org.apache.hugegraph.store.node.util.HgAssert; +import org.apache.hugegraph.store.node.util.HgGrpc; +import org.apache.hugegraph.store.node.util.HgStoreConst; +import org.apache.hugegraph.store.node.util.HgStoreNodeUtil; + +import com.google.protobuf.ByteString; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; + +/** + * @author lynn.bond@hotmail.com created on 2022/03/27 + * @version 3.6.0 + */ +@Slf4j +public class ScanBatchResponse3 { + private final static long DEFAULT_PACKAGE_SIZE = 10_000; + private final static int MAX_NOT_RECEIPT = 10; + + public static StreamObserver of(StreamObserver responseObserver, + HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) { + HgAssert.isArgumentNotNull(responseObserver, "responseObserver"); + HgAssert.isArgumentNotNull(wrapper, "wrapper"); + return new Broker(responseObserver, wrapper, executor); + } + + private enum OrderState { + NEW(0), + WORKING(1);//, PAUSE(2), COMPLETE(10); + int value; + + OrderState(int value) { + this.value = value; + } + } + + /*** Broker ***/ + private static class Broker implements StreamObserver { + private final StreamObserver responseObserver; + private final HgStoreWrapperEx wrapper; + private final ThreadPoolExecutor executor; + private String graph; + + private final OrderManager manager = new OrderManager(); + + Broker(StreamObserver responseObserver, HgStoreWrapperEx wrapper, + ThreadPoolExecutor executor) { + this.responseObserver = responseObserver; + this.wrapper = wrapper; + this.executor = executor; + } + + @Override + public void onNext(ScanStreamBatchReq request) { + this.handleHeader(request); + switch (request.getQueryCase()) { + case QUERY_REQUEST: + this.makeADeal(request.getQueryRequest()); + break; + case RECEIPT_REQUEST: + this.manager.receipt(request.getReceiptRequest().getTimes()); + break; + case CANCEL_REQUEST: + this.manager.finished(); + break; + default: + responseObserver.onError( + HgGrpc.toErr("Unsupported sub-request: [ " + request + " ]")); + } + } + + @Override + public void onError(Throwable t) { + log.warn(t.getMessage()); + this.manager.breakdown(); + } + + @Override + public void onCompleted() { + this.manager.finished(); + } + + private void handleHeader(ScanStreamBatchReq request) { + if (this.graph == null) { + this.graph = request.getHeader().getGraph(); + } + } + + private void makeADeal(ScanQueryRequest request) { + String deliverId = ""; + if (log.isDebugEnabled()) { + List conditions = request.getConditionList(); + if (conditions.size() > 0) { + ScanCondition c = conditions.get(0); + if (c.getPrefix() != null && c.getPrefix().size() > 0) { + deliverId = Base58.encode(c.getPrefix().toByteArray()); + log.info("[ANALYSIS DEAL] [{}] prefixLength: {}", deliverId, + conditions.size()); + } + + } + } + + OrderDeliverer deliverer = new OrderDeliverer(deliverId, this.responseObserver); + OrderWorker worker = new OrderWorker( + request.getLimit(), + request.getPageSize(), + ScanUtil.getIterator(this.graph, request, this.wrapper), + deliverer, + this.executor); + + this.manager.deal(worker, deliverer); + } + + } + + @NotThreadSafe + private static class OrderManager { + OrderState state = OrderState.NEW; + OrderWorker worker; + OrderDeliverer deliverer; + + synchronized void deal(OrderWorker worker, OrderDeliverer deliverer) { + if (log.isDebugEnabled()) log.debug("Receiving query request."); + if (this.state == OrderState.NEW) { + this.worker = worker; + this.deliverer = deliverer; + this.worker.hereWeGo(); + this.state = OrderState.WORKING; + } + } + + synchronized void receipt(int receiptTimes) { + if (log.isDebugEnabled()) log.debug("Receiving receipt request."); + this.worker.setReceipt(receiptTimes); + } + + synchronized void finished() { + if (log.isDebugEnabled()) log.debug("Receiving finished request."); +/* if (this.state.value > OrderState.NEW.value + && this.state.value < OrderState.COMPLETE.value) { + this.state = OrderState.COMPLETE; + }*/ + this.breakdown(); + } + + synchronized void breakdown() { + if (this.worker != null) { + this.worker.breakdown(); + } + } + } + + private static class OrderDeliverer { + private final StreamObserver responseObserver; + private final AtomicBoolean finishFlag = new AtomicBoolean(); + private final String delivererId; + private final AtomicLong count = new AtomicLong(); + + OrderDeliverer(String delivererId, StreamObserver responseObserver) { + this.responseObserver = responseObserver; + this.delivererId = delivererId; + } + + void deliver(KvPageRes.Builder dataBuilder, int times, boolean isOver) { + if (this.finishFlag.get()) { + return; + } + count.addAndGet(dataBuilder.getDataCount()); + this.responseObserver.onNext(dataBuilder.setOver(isOver).setTimes(times).build()); + if (log.isDebugEnabled()) log.debug("deliver times : {}, over: {}", times, isOver); + + if (isOver) { + if (log.isDebugEnabled()) { + if (delivererId != null && !delivererId.isEmpty()) { + log.debug("[ANALYSIS OVER] [{}] count: {}, times: {}", delivererId, count, + times); + } + } + this.finish(); + } + } + + void finish() { + if (!finishFlag.getAndSet(true)) { + this.responseObserver.onCompleted(); + } + } + + void error(String msg) { + if (!finishFlag.getAndSet(true)) { + this.responseObserver.onError(HgGrpc.toErr(msg)); + } + } + + void error(String msg, Throwable t) { + if (!finishFlag.getAndSet(true)) { + this.responseObserver.onError(HgGrpc.toErr(Status.INTERNAL, + msg, t)); + } + } + } + + /*** Worker ***/ + private static class OrderWorker { + private final ScanIterator iterator; + private final OrderDeliverer deliverer; + private final AtomicBoolean pauseFlag = new AtomicBoolean(); + private final AtomicBoolean completeFlag = new AtomicBoolean(); + private final ReentrantLock workingLock = new ReentrantLock(); + private final AtomicBoolean isWorking = new AtomicBoolean(); + private final AtomicBoolean breakdown = new AtomicBoolean(); + private final AtomicInteger receiptTimes = new AtomicInteger(); + private final AtomicInteger curTimes = new AtomicInteger(); + private final ThreadPoolExecutor executor; + private final long limit; + private long packageSize; + private long counter; + + OrderWorker(long limit, long packageSize, ScanIterator iterator, OrderDeliverer deliverer, + ThreadPoolExecutor executor) { + this.limit = limit; + this.packageSize = packageSize; + this.iterator = iterator; + this.deliverer = deliverer; + this.executor = executor; + + if (this.packageSize <= 0) { + this.packageSize = DEFAULT_PACKAGE_SIZE; + log.warn( + "As page-Size is less than or equals 0, default package-size was " + + "effective.[ {} ]", + DEFAULT_PACKAGE_SIZE); + } + + } + + void hereWeGo() { + if (this.completeFlag.get()) { + log.warn("job complete."); + return; + } + if (this.isWorking.get()) { + log.warn("has been working."); + return; + } + + if (this.workingLock.isLocked()) { + log.warn("working now"); + return; + } + + executor.execute(() -> working()); + Thread.yield(); + } + + void setReceipt(int times) { + this.receiptTimes.set(times); + this.continueWorking(); + } + + boolean checkContinue() { + return (this.curTimes.get() - this.receiptTimes.get() < MAX_NOT_RECEIPT); + } + + void continueWorking() { + if (this.checkContinue()) { + synchronized (this.iterator) { + this.iterator.notify(); + } + } + } + + void breakdown() { + this.breakdown.set(true); + synchronized (this.iterator) { + this.iterator.notify(); + } + } + + private void working() { + if (this.isWorking.getAndSet(true)) { + return; + } + + this.workingLock.lock(); + + try { + synchronized (this.iterator) { + KvPageRes.Builder dataBuilder = KvPageRes.newBuilder(); + Kv.Builder kvBuilder = Kv.newBuilder(); + long packageCount = 0; + + while (iterator.hasNext()) { + if (++this.counter > limit) { + this.completeFlag.set(true); + break; + } + + if (++packageCount > packageSize) { + + if (this.breakdown.get()) { + break; + } + + deliverer.deliver(dataBuilder, curTimes.incrementAndGet(), false); + Thread.yield(); + + if (!this.checkContinue()) { + long start = System.currentTimeMillis(); + iterator.wait( + HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * + 1000); + + if (System.currentTimeMillis() - start + >= + HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * 1000) { + throw new TimeoutException("Waiting continue more than " + + + HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS + + " seconds."); + } + + if (this.breakdown.get()) { + break; + } + + } + + packageCount = 1; + dataBuilder = KvPageRes.newBuilder(); + } + + RocksDBSession.BackendColumn col = iterator.next(); + + dataBuilder.addData(kvBuilder + .setKey(ByteString.copyFrom(col.name)) + .setValue(ByteString.copyFrom(col.value)) + .setCode(HgStoreNodeUtil.toInt( + iterator.position())) +//position == partition-id. + ); + + } + + this.completeFlag.set(true); + + deliverer.deliver(dataBuilder, curTimes.incrementAndGet(), true); + + } + + } catch (InterruptedException e) { + log.error("Interrupted waiting of iterator, canceled while.", e); + this.deliverer.error("Failed to finish scanning, cause by InterruptedException."); + } catch (TimeoutException t) { + log.info(t.getMessage()); + this.deliverer.error("Sever waiting exceeded [" + + HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS + + "] seconds."); + } catch (Throwable t) { + log.error("Failed to do while for scanning, cause by:", t); + this.deliverer.error("Failed to finish scanning ", t); + } finally { + this.workingLock.unlock(); + this.iterator.close(); + } + } + + } + } \ No newline at end of file diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanQuery.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanQuery.java index 4cf6c4c9c..e9e54f086 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanQuery.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanQuery.java @@ -1,104 +1,104 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.grpc; - -import java.util.Arrays; - -import org.apache.hugegraph.store.grpc.common.ScanMethod; - -/** - * @author lynn.bond@hotmail.com on 2022/2/28 - */ -class ScanQuery implements QueryCondition { - String graph; - String table; - ScanMethod method; - - byte[] start; - byte[] end; - byte[] prefix; - int keyCode; - int scanType; - byte[] query; - byte[] position; - int serialNo; - - @Override - public byte[] getStart() { - return this.start; - } - - @Override - public byte[] getEnd() { - return this.end; - } - - @Override - public byte[] getPrefix() { - return this.prefix; - } - - @Override - public int getKeyCode() { - return this.keyCode; - } - - @Override - public int getScanType() { - return this.scanType; - } - - @Override - public byte[] getQuery() { - return this.query; - } - - @Override - public byte[] getPosition() { - return this.position; - } - - @Override - public int getSerialNo() { - return this.serialNo; - } - - static ScanQuery of() { - return new ScanQuery(); - } - - private ScanQuery() { - } - - @Override - public String toString() { - return "ScanQuery{" + - "graph='" + graph + '\'' + - ", table='" + table + '\'' + - ", method=" + method + - ", start=" + Arrays.toString(start) + - ", end=" + Arrays.toString(end) + - ", prefix=" + Arrays.toString(prefix) + - ", partition=" + keyCode + - ", scanType=" + scanType + - ", serialNo=" + serialNo + - ", query=" + Arrays.toString(query) + - ", position=" + Arrays.toString(position) + - '}'; - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +import java.util.Arrays; + +import org.apache.hugegraph.store.grpc.common.ScanMethod; + +/** + * @author lynn.bond@hotmail.com on 2022/2/28 + */ +class ScanQuery implements QueryCondition { + String graph; + String table; + ScanMethod method; + + byte[] start; + byte[] end; + byte[] prefix; + int keyCode; + int scanType; + byte[] query; + byte[] position; + int serialNo; + + @Override + public byte[] getStart() { + return this.start; + } + + @Override + public byte[] getEnd() { + return this.end; + } + + @Override + public byte[] getPrefix() { + return this.prefix; + } + + @Override + public int getKeyCode() { + return this.keyCode; + } + + @Override + public int getScanType() { + return this.scanType; + } + + @Override + public byte[] getQuery() { + return this.query; + } + + @Override + public byte[] getPosition() { + return this.position; + } + + @Override + public int getSerialNo() { + return this.serialNo; + } + + static ScanQuery of() { + return new ScanQuery(); + } + + private ScanQuery() { + } + + @Override + public String toString() { + return "ScanQuery{" + + "graph='" + graph + '\'' + + ", table='" + table + '\'' + + ", method=" + method + + ", start=" + Arrays.toString(start) + + ", end=" + Arrays.toString(end) + + ", prefix=" + Arrays.toString(prefix) + + ", partition=" + keyCode + + ", scanType=" + scanType + + ", serialNo=" + serialNo + + ", query=" + Arrays.toString(query) + + ", position=" + Arrays.toString(position) + + '}'; + } +} diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/GRpcExMetrics.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/GRpcExMetrics.java index 3e9122f9a..28073a190 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/GRpcExMetrics.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/GRpcExMetrics.java @@ -1,94 +1,94 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.metrics; - -import java.util.concurrent.ThreadPoolExecutor; - -import org.apache.hugegraph.store.node.grpc.GRpcServerConfig; -import org.apache.hugegraph.store.node.util.HgExecutorUtil; - -import io.micrometer.core.instrument.Gauge; -import io.micrometer.core.instrument.MeterRegistry; - -/** - * @author lynn.bond@hotmail.com on 2022/3/8 - */ -public class GRpcExMetrics { - public final static String PREFIX = "grpc"; - private static MeterRegistry registry; - private final static ExecutorWrapper wrapper = new ExecutorWrapper(); - - private GRpcExMetrics() { - } - - public synchronized static void init(MeterRegistry meterRegistry) { - if (registry == null) { - registry = meterRegistry; - registerMeters(); - } - } - - private static void registerMeters() { - registerExecutor(); - - } - - private static void registerExecutor() { - - Gauge.builder(PREFIX + ".executor.pool.size", wrapper, (e) -> e.getPoolSize()) - .description("The current number of threads in the pool.") - .register(registry); - - Gauge.builder(PREFIX + ".executor.core.pool.size", wrapper, (e) -> e.getCorePoolSize()) - .description( - "The largest number of threads that have ever simultaneously been in the " + - "pool.") - .register(registry); - - Gauge.builder(PREFIX + ".executor.active.count", wrapper, (e) -> e.getActiveCount()) - .description("The approximate number of threads that are actively executing tasks.") - .register(registry); - } - - private static class ExecutorWrapper { - ThreadPoolExecutor pool; - - void init() { - if (this.pool == null) { - pool = HgExecutorUtil.getThreadPoolExecutor(GRpcServerConfig.EXECUTOR_NAME); - } - } - - double getPoolSize() { - init(); - return this.pool == null ? 0d : this.pool.getPoolSize(); - } - - int getCorePoolSize() { - init(); - return this.pool == null ? 0 : this.pool.getCorePoolSize(); - } - - int getActiveCount() { - init(); - return this.pool == null ? 0 : this.pool.getActiveCount(); - } - - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.metrics; + +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.hugegraph.store.node.grpc.GRpcServerConfig; +import org.apache.hugegraph.store.node.util.HgExecutorUtil; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; + +/** + * @author lynn.bond@hotmail.com on 2022/3/8 + */ +public class GRpcExMetrics { + public final static String PREFIX = "grpc"; + private static MeterRegistry registry; + private final static ExecutorWrapper wrapper = new ExecutorWrapper(); + + private GRpcExMetrics() { + } + + public synchronized static void init(MeterRegistry meterRegistry) { + if (registry == null) { + registry = meterRegistry; + registerMeters(); + } + } + + private static void registerMeters() { + registerExecutor(); + + } + + private static void registerExecutor() { + + Gauge.builder(PREFIX + ".executor.pool.size", wrapper, (e) -> e.getPoolSize()) + .description("The current number of threads in the pool.") + .register(registry); + + Gauge.builder(PREFIX + ".executor.core.pool.size", wrapper, (e) -> e.getCorePoolSize()) + .description( + "The largest number of threads that have ever simultaneously been in the " + + "pool.") + .register(registry); + + Gauge.builder(PREFIX + ".executor.active.count", wrapper, (e) -> e.getActiveCount()) + .description("The approximate number of threads that are actively executing tasks.") + .register(registry); + } + + private static class ExecutorWrapper { + ThreadPoolExecutor pool; + + void init() { + if (this.pool == null) { + pool = HgExecutorUtil.getThreadPoolExecutor(GRpcServerConfig.EXECUTOR_NAME); + } + } + + double getPoolSize() { + init(); + return this.pool == null ? 0d : this.pool.getPoolSize(); + } + + int getCorePoolSize() { + init(); + return this.pool == null ? 0 : this.pool.getCorePoolSize(); + } + + int getActiveCount() { + init(); + return this.pool == null ? 0 : this.pool.getActiveCount(); + } + + } + +} diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/JRaftMetrics.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/JRaftMetrics.java index 11adf85a9..324dd2014 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/JRaftMetrics.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/JRaftMetrics.java @@ -1,310 +1,310 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.metrics; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.hugegraph.store.HgStoreEngine; -import org.apache.hugegraph.store.node.util.HgRegexUtil; - -import com.alipay.sofa.jraft.core.NodeMetrics; -import com.codahale.metrics.Counter; -import com.codahale.metrics.Meter; -import com.codahale.metrics.Snapshot; -import com.codahale.metrics.Timer; - -import io.micrometer.core.instrument.Gauge; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Tag; -import lombok.extern.slf4j.Slf4j; - -/** - * @author lynn.bond@hotmail.com on 2022/1/4 - */ -@Slf4j -public class JRaftMetrics { - public final static String PREFIX = "jraft"; - public static final String LABELS = "quantile"; - public static final String LABEL_50 = "0.5"; - public static final String LABEL_75 = "0.75"; - public static final String LABEL_95 = "0.95"; - public static final String LABEL_98 = "0.98"; - public static final String LABEL_99 = "0.99"; - public static final String LABEL_999 = "0.999"; - - private static MeterRegistry registry; - private final static HgStoreEngine storeEngine = HgStoreEngine.getInstance(); - private final static AtomicInteger groups = new AtomicInteger(0); - private final static Tag handleDataTag = Tag.of("handle", "data"); - // private final static Tag handleTxTag = Tag.of("handle", "tx"); //reservation - private final static Set groupSet = new HashSet<>(); - - private final static String REGEX_REFINE_REPLICATOR = "(replicator)(.+?:\\d+)(.*)"; - - private JRaftMetrics() { - } - - public synchronized static void init(MeterRegistry meterRegistry) { - if (registry == null) { - registry = meterRegistry; - registerMeters(); - } - } - - private static void registerMeters() { - Gauge.builder(PREFIX + ".groups", JRaftMetrics::updateGroups) - .description("Number of raft-groups, which handled the data of graph.") - .tags(Collections.singleton(handleDataTag)) - .register(registry); - - } - - private static int updateGroups() { - int buf = getGroups(); - if (buf != groups.get()) { - groups.set(buf); - registerNodeMetrics(); - } - return buf; - } - - private static int getGroups() { - return storeEngine.getRaftGroupCount(); - } - - private static Map getRaftGroupMetrics() { - Map map = storeEngine.getNodeMetrics(); - - if (map == null) { - return Collections.emptyMap(); - } - - return map; - } - - private static void registerNodeMetrics() { - Map map = getRaftGroupMetrics(); - - synchronized (groupSet) { - map.forEach((group, metrics) -> { - if (!groupSet.add(group)) return; - - metrics.getMetricRegistry().getGauges() - .forEach((k, v) -> registerGauge(group, k, v)); - metrics.getMetricRegistry().getMeters() - .forEach((k, v) -> registerMeter(group, k, v)); - metrics.getMetricRegistry().getCounters() - .forEach((k, v) -> registerCounter(group, k, v)); - metrics.getMetricRegistry().getTimers() - .forEach((k, v) -> registerTimer(group, k, v)); - metrics.getMetricRegistry().getHistograms() - .forEach((k, v) -> registerHistogram(group, k, v)); - }); - } - - } - - private static class HistogramWrapper { - private final com.codahale.metrics.Histogram histogram; - - private Snapshot snapshot; - private long ts = System.currentTimeMillis(); - - HistogramWrapper(com.codahale.metrics.Histogram histogram) { - this.histogram = histogram; - this.snapshot = this.histogram.getSnapshot(); - } - - Snapshot getSnapshot() { - if (System.currentTimeMillis() - this.ts > 30_000) { - this.snapshot = this.histogram.getSnapshot(); - this.ts = System.currentTimeMillis(); - } - return this.snapshot; - } - } - - private static HistogramWrapper toWrapper(com.codahale.metrics.Histogram histogram) { - return new HistogramWrapper(histogram); - } - - private static String refineMetrics(String name, List tags) { - if (name == null || name.isEmpty()) return name; - - List buf = HgRegexUtil.toGroupValues(REGEX_REFINE_REPLICATOR, name); - String res = null; - - /*Extracted name of replicator into a tag.*/ - - if (buf != null && buf.size() == 4) { - res = buf.get(1) + buf.get(3); - - String value = buf.get(2); - - if (value != null && value.startsWith("-")) { - value = value.substring(1); - } - - tags.add(Tag.of("replicator", value)); - } else { - res = name; - } - - return res; - } - - private static void registerHistogram(String group, String name, - com.codahale.metrics.Histogram histogram) { - if (histogram == null) return; - - List tags = new LinkedList<>(); - tags.add(handleDataTag); - tags.add(Tag.of("group", group)); - - name = refineMetrics(name, tags); - - String baseName = PREFIX + "." + name.toLowerCase(); - - HistogramWrapper wrapper = toWrapper(histogram); - - Gauge.builder(baseName + ".median", wrapper, (d) -> d.getSnapshot().getMedian()) - .tags(tags).register(registry); - Gauge.builder(baseName + ".min", wrapper, (d) -> d.getSnapshot().getMin()) - .tags(tags).register(registry); - Gauge.builder(baseName + ".max", wrapper, (d) -> d.getSnapshot().getMax()) - .tags(tags).register(registry); - Gauge.builder(baseName + ".mean", wrapper, (d) -> d.getSnapshot().getMean()) - .tags(tags).register(registry); - - baseName = baseName + ".summary"; - Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().getMedian()) - .tags(tags).tag(LABELS, LABEL_50).register(registry); - Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get75thPercentile()) - .tags(tags).tag(LABELS, LABEL_75).register(registry); - Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get95thPercentile()) - .tags(tags).tag(LABELS, LABEL_95).register(registry); - Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get98thPercentile()) - .tags(tags).tag(LABELS, LABEL_98).register(registry); - Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get99thPercentile()) - .tags(tags).tag(LABELS, LABEL_99).register(registry); - Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get999thPercentile()) - .tags(tags).tag(LABELS, LABEL_999).register(registry); - - Gauge.builder(baseName + ".sum", wrapper, - (d) -> Arrays.stream(d.getSnapshot().getValues()).sum()) - .tags(tags).register(registry); - Gauge.builder(baseName + ".count", wrapper, (d) -> d.getSnapshot().size()) - .tags(tags).register(registry); - - } - - private static void registerTimer(String group, String name, com.codahale.metrics.Timer timer) { - List tags = new LinkedList<>(); - tags.add(handleDataTag); - tags.add(Tag.of("group", group)); - - name = refineMetrics(name, tags); - - String baseName = PREFIX + "." + name.toLowerCase(); - - Gauge.builder(baseName + ".count", timer, Timer::getCount) - .tags(tags).register(registry); - - Gauge.builder(baseName + ".timer", timer, Timer::getCount) - .tags(tags).tag("rate", "1m").register(registry); - Gauge.builder(baseName + ".timer", timer, Timer::getCount) - .tags(tags).tag("rate", "5m").register(registry); - Gauge.builder(baseName + ".timer", timer, Timer::getCount) - .tags(tags).tag("rate", "15m").register(registry); - Gauge.builder(baseName + ".timer", timer, Timer::getCount) - .tags(tags).tag("rate", "mean").register(registry); - - } - - private static void registerMeter(String group, String name, com.codahale.metrics.Meter meter) { - List tags = new LinkedList<>(); - tags.add(handleDataTag); - tags.add(Tag.of("group", group)); - - name = refineMetrics(name, tags); - - String baseName = PREFIX + "." + name.toLowerCase(); - - Gauge.builder(baseName + ".count", meter, Meter::getCount) - .tags(tags) - .register(registry); - - Gauge.builder(baseName + ".rate", meter, Meter::getCount) - .tags(tags).tag("rate", "1m") - .register(registry); - Gauge.builder(baseName + ".rate", meter, Meter::getCount) - .tags(tags).tag("rate", "5m") - .register(registry); - Gauge.builder(baseName + ".rate", meter, Meter::getCount) - .tags(tags).tag("rate", "15m") - .register(registry); - Gauge.builder(baseName + ".rate", meter, Meter::getCount) - .tags(tags).tag("rate", "mean") - .register(registry); - - } - - private static void registerCounter(String group, String name, - com.codahale.metrics.Counter counter) { - List tags = new LinkedList<>(); - tags.add(handleDataTag); - tags.add(Tag.of("group", group)); - - name = refineMetrics(name, tags); - - name = name.toLowerCase(); - - //Adapted a counter to be a gauge. - Gauge.builder(PREFIX + "." + name + ".count", counter, Counter::getCount) - .tags(tags).register(registry); - } - - private static void registerGauge(String group, String name, - com.codahale.metrics.Gauge gauge) { - List tags = new LinkedList<>(); - tags.add(handleDataTag); - tags.add(Tag.of("group", group)); - - name = refineMetrics(name, tags); - - name = name.toLowerCase(); - - if (gauge.getValue() instanceof Number) { - Gauge.builder(PREFIX + "." + name, gauge, (g) -> ((Number) g.getValue()).doubleValue()) - .tags(tags).register(registry); - } else { - Gauge.builder(PREFIX + "." + name, () -> 1.0) - .tags(tags) - .tag("str.gauge", String.valueOf(gauge.getValue())).register(registry); - } - - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.metrics; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.node.util.HgRegexUtil; + +import com.alipay.sofa.jraft.core.NodeMetrics; +import com.codahale.metrics.Counter; +import com.codahale.metrics.Meter; +import com.codahale.metrics.Snapshot; +import com.codahale.metrics.Timer; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import lombok.extern.slf4j.Slf4j; + +/** + * @author lynn.bond@hotmail.com on 2022/1/4 + */ +@Slf4j +public class JRaftMetrics { + public final static String PREFIX = "jraft"; + public static final String LABELS = "quantile"; + public static final String LABEL_50 = "0.5"; + public static final String LABEL_75 = "0.75"; + public static final String LABEL_95 = "0.95"; + public static final String LABEL_98 = "0.98"; + public static final String LABEL_99 = "0.99"; + public static final String LABEL_999 = "0.999"; + + private static MeterRegistry registry; + private final static HgStoreEngine storeEngine = HgStoreEngine.getInstance(); + private final static AtomicInteger groups = new AtomicInteger(0); + private final static Tag handleDataTag = Tag.of("handle", "data"); + // private final static Tag handleTxTag = Tag.of("handle", "tx"); //reservation + private final static Set groupSet = new HashSet<>(); + + private final static String REGEX_REFINE_REPLICATOR = "(replicator)(.+?:\\d+)(.*)"; + + private JRaftMetrics() { + } + + public synchronized static void init(MeterRegistry meterRegistry) { + if (registry == null) { + registry = meterRegistry; + registerMeters(); + } + } + + private static void registerMeters() { + Gauge.builder(PREFIX + ".groups", JRaftMetrics::updateGroups) + .description("Number of raft-groups, which handled the data of graph.") + .tags(Collections.singleton(handleDataTag)) + .register(registry); + + } + + private static int updateGroups() { + int buf = getGroups(); + if (buf != groups.get()) { + groups.set(buf); + registerNodeMetrics(); + } + return buf; + } + + private static int getGroups() { + return storeEngine.getRaftGroupCount(); + } + + private static Map getRaftGroupMetrics() { + Map map = storeEngine.getNodeMetrics(); + + if (map == null) { + return Collections.emptyMap(); + } + + return map; + } + + private static void registerNodeMetrics() { + Map map = getRaftGroupMetrics(); + + synchronized (groupSet) { + map.forEach((group, metrics) -> { + if (!groupSet.add(group)) return; + + metrics.getMetricRegistry().getGauges() + .forEach((k, v) -> registerGauge(group, k, v)); + metrics.getMetricRegistry().getMeters() + .forEach((k, v) -> registerMeter(group, k, v)); + metrics.getMetricRegistry().getCounters() + .forEach((k, v) -> registerCounter(group, k, v)); + metrics.getMetricRegistry().getTimers() + .forEach((k, v) -> registerTimer(group, k, v)); + metrics.getMetricRegistry().getHistograms() + .forEach((k, v) -> registerHistogram(group, k, v)); + }); + } + + } + + private static class HistogramWrapper { + private final com.codahale.metrics.Histogram histogram; + + private Snapshot snapshot; + private long ts = System.currentTimeMillis(); + + HistogramWrapper(com.codahale.metrics.Histogram histogram) { + this.histogram = histogram; + this.snapshot = this.histogram.getSnapshot(); + } + + Snapshot getSnapshot() { + if (System.currentTimeMillis() - this.ts > 30_000) { + this.snapshot = this.histogram.getSnapshot(); + this.ts = System.currentTimeMillis(); + } + return this.snapshot; + } + } + + private static HistogramWrapper toWrapper(com.codahale.metrics.Histogram histogram) { + return new HistogramWrapper(histogram); + } + + private static String refineMetrics(String name, List tags) { + if (name == null || name.isEmpty()) return name; + + List buf = HgRegexUtil.toGroupValues(REGEX_REFINE_REPLICATOR, name); + String res = null; + + /*Extracted name of replicator into a tag.*/ + + if (buf != null && buf.size() == 4) { + res = buf.get(1) + buf.get(3); + + String value = buf.get(2); + + if (value != null && value.startsWith("-")) { + value = value.substring(1); + } + + tags.add(Tag.of("replicator", value)); + } else { + res = name; + } + + return res; + } + + private static void registerHistogram(String group, String name, + com.codahale.metrics.Histogram histogram) { + if (histogram == null) return; + + List tags = new LinkedList<>(); + tags.add(handleDataTag); + tags.add(Tag.of("group", group)); + + name = refineMetrics(name, tags); + + String baseName = PREFIX + "." + name.toLowerCase(); + + HistogramWrapper wrapper = toWrapper(histogram); + + Gauge.builder(baseName + ".median", wrapper, (d) -> d.getSnapshot().getMedian()) + .tags(tags).register(registry); + Gauge.builder(baseName + ".min", wrapper, (d) -> d.getSnapshot().getMin()) + .tags(tags).register(registry); + Gauge.builder(baseName + ".max", wrapper, (d) -> d.getSnapshot().getMax()) + .tags(tags).register(registry); + Gauge.builder(baseName + ".mean", wrapper, (d) -> d.getSnapshot().getMean()) + .tags(tags).register(registry); + + baseName = baseName + ".summary"; + Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().getMedian()) + .tags(tags).tag(LABELS, LABEL_50).register(registry); + Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get75thPercentile()) + .tags(tags).tag(LABELS, LABEL_75).register(registry); + Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get95thPercentile()) + .tags(tags).tag(LABELS, LABEL_95).register(registry); + Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get98thPercentile()) + .tags(tags).tag(LABELS, LABEL_98).register(registry); + Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get99thPercentile()) + .tags(tags).tag(LABELS, LABEL_99).register(registry); + Gauge.builder(baseName, wrapper, (d) -> d.getSnapshot().get999thPercentile()) + .tags(tags).tag(LABELS, LABEL_999).register(registry); + + Gauge.builder(baseName + ".sum", wrapper, + (d) -> Arrays.stream(d.getSnapshot().getValues()).sum()) + .tags(tags).register(registry); + Gauge.builder(baseName + ".count", wrapper, (d) -> d.getSnapshot().size()) + .tags(tags).register(registry); + + } + + private static void registerTimer(String group, String name, com.codahale.metrics.Timer timer) { + List tags = new LinkedList<>(); + tags.add(handleDataTag); + tags.add(Tag.of("group", group)); + + name = refineMetrics(name, tags); + + String baseName = PREFIX + "." + name.toLowerCase(); + + Gauge.builder(baseName + ".count", timer, Timer::getCount) + .tags(tags).register(registry); + + Gauge.builder(baseName + ".timer", timer, Timer::getCount) + .tags(tags).tag("rate", "1m").register(registry); + Gauge.builder(baseName + ".timer", timer, Timer::getCount) + .tags(tags).tag("rate", "5m").register(registry); + Gauge.builder(baseName + ".timer", timer, Timer::getCount) + .tags(tags).tag("rate", "15m").register(registry); + Gauge.builder(baseName + ".timer", timer, Timer::getCount) + .tags(tags).tag("rate", "mean").register(registry); + + } + + private static void registerMeter(String group, String name, com.codahale.metrics.Meter meter) { + List tags = new LinkedList<>(); + tags.add(handleDataTag); + tags.add(Tag.of("group", group)); + + name = refineMetrics(name, tags); + + String baseName = PREFIX + "." + name.toLowerCase(); + + Gauge.builder(baseName + ".count", meter, Meter::getCount) + .tags(tags) + .register(registry); + + Gauge.builder(baseName + ".rate", meter, Meter::getCount) + .tags(tags).tag("rate", "1m") + .register(registry); + Gauge.builder(baseName + ".rate", meter, Meter::getCount) + .tags(tags).tag("rate", "5m") + .register(registry); + Gauge.builder(baseName + ".rate", meter, Meter::getCount) + .tags(tags).tag("rate", "15m") + .register(registry); + Gauge.builder(baseName + ".rate", meter, Meter::getCount) + .tags(tags).tag("rate", "mean") + .register(registry); + + } + + private static void registerCounter(String group, String name, + com.codahale.metrics.Counter counter) { + List tags = new LinkedList<>(); + tags.add(handleDataTag); + tags.add(Tag.of("group", group)); + + name = refineMetrics(name, tags); + + name = name.toLowerCase(); + + //Adapted a counter to be a gauge. + Gauge.builder(PREFIX + "." + name + ".count", counter, Counter::getCount) + .tags(tags).register(registry); + } + + private static void registerGauge(String group, String name, + com.codahale.metrics.Gauge gauge) { + List tags = new LinkedList<>(); + tags.add(handleDataTag); + tags.add(Tag.of("group", group)); + + name = refineMetrics(name, tags); + + name = name.toLowerCase(); + + if (gauge.getValue() instanceof Number) { + Gauge.builder(PREFIX + "." + name, gauge, (g) -> ((Number) g.getValue()).doubleValue()) + .tags(tags).register(registry); + } else { + Gauge.builder(PREFIX + "." + name, () -> 1.0) + .tags(tags) + .tag("str.gauge", String.valueOf(gauge.getValue())).register(registry); + } + + } + +} diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsMetrics.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsMetrics.java index d434c90c7..773c9b66a 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsMetrics.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/ProcfsMetrics.java @@ -1,67 +1,67 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.metrics; - - -import io.micrometer.core.instrument.Gauge; -import io.micrometer.core.instrument.MeterRegistry; - -/** - * @author lynn.bond@hotmail.com on 2022/3/1 - * @version 0.1.0 - */ -public class ProcfsMetrics { - - private static MeterRegistry registry; - public final static String PREFIX = "process_memory"; - - private final static ProcfsSmaps smaps = new ProcfsSmaps(); - - private ProcfsMetrics() { - - } - - public synchronized static void init(MeterRegistry meterRegistry) { - if (registry == null) { - registry = meterRegistry; - registerMeters(); - } - } - - private static void registerMeters() { - registerProcessGauge(); - } - - private static void registerProcessGauge() { - Gauge.builder(PREFIX + ".rss.bytes", () -> smaps.get(ProcfsSmaps.KEY.RSS)) - .register(registry); - - Gauge.builder(PREFIX + ".pss.bytes", () -> smaps.get(ProcfsSmaps.KEY.PSS)) - .register(registry); - - Gauge.builder(PREFIX + ".vss.bytes", () -> smaps.get(ProcfsSmaps.KEY.VSS)) - .register(registry); - - Gauge.builder(PREFIX + ".swap.bytes", () -> smaps.get(ProcfsSmaps.KEY.SWAP)) - .register(registry); - - Gauge.builder(PREFIX + ".swappss.bytes", () -> smaps.get(ProcfsSmaps.KEY.SWAPPSS)) - .register(registry); - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.metrics; + + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; + +/** + * @author lynn.bond@hotmail.com on 2022/3/1 + * @version 0.1.0 + */ +public class ProcfsMetrics { + + private static MeterRegistry registry; + public final static String PREFIX = "process_memory"; + + private final static ProcfsSmaps smaps = new ProcfsSmaps(); + + private ProcfsMetrics() { + + } + + public synchronized static void init(MeterRegistry meterRegistry) { + if (registry == null) { + registry = meterRegistry; + registerMeters(); + } + } + + private static void registerMeters() { + registerProcessGauge(); + } + + private static void registerProcessGauge() { + Gauge.builder(PREFIX + ".rss.bytes", () -> smaps.get(ProcfsSmaps.KEY.RSS)) + .register(registry); + + Gauge.builder(PREFIX + ".pss.bytes", () -> smaps.get(ProcfsSmaps.KEY.PSS)) + .register(registry); + + Gauge.builder(PREFIX + ".vss.bytes", () -> smaps.get(ProcfsSmaps.KEY.VSS)) + .register(registry); + + Gauge.builder(PREFIX + ".swap.bytes", () -> smaps.get(ProcfsSmaps.KEY.SWAP)) + .register(registry); + + Gauge.builder(PREFIX + ".swappss.bytes", () -> smaps.get(ProcfsSmaps.KEY.SWAPPSS)) + .register(registry); + } + +} diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetrics.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetrics.java index 586f73b03..4bf3469d1 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetrics.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetrics.java @@ -1,414 +1,414 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.metrics; - -import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.HISTOGRAMS; -import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABELS; -import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_50; -import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_95; -import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_99; -import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.PREFIX; -import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.TICKERS; - -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import org.apache.hugegraph.rocksdb.access.RocksDBFactory; -import org.apache.hugegraph.rocksdb.access.RocksDBSession; -import org.apache.hugegraph.store.HgStoreEngine; -import org.apache.hugegraph.store.node.util.HgAssert; -import org.rocksdb.HistogramData; -import org.rocksdb.HistogramType; -import org.rocksdb.MemoryUsageType; -import org.rocksdb.Statistics; -import org.rocksdb.TickerType; - -import io.micrometer.core.instrument.Gauge; -import io.micrometer.core.instrument.Meter; -import io.micrometer.core.instrument.MeterRegistry; -import lombok.extern.slf4j.Slf4j; - -/** - * @author lynn.bond@hotmail.com on 2021/12/30 - * @version 1.2.0 on 2022/03/22 added auto meter removing when graph has been closed. - */ -@Slf4j -public class RocksDBMetrics { - private static MeterRegistry registry; - private final static RocksDBFactory rocksDBFactory = RocksDBFactory.getInstance(); - private final static AtomicInteger rocks = new AtomicInteger(0); - private final static Set graphSet = new HashSet<>(); - private final static HgStoreEngine storeEngine = HgStoreEngine.getInstance(); - private final static MemoryUseWrapper memoryUseWrapper = new MemoryUseWrapper(); - private final static Map statisticsHolder = new HashMap<>(); - private final static Map histogramHolder = new HashMap<>(); - private final static Map> graphMeterMap = new ConcurrentHashMap<>(); - - private RocksDBMetrics() { - } - - public static void init(final MeterRegistry meterRegistry) { - HgAssert.isArgumentNotNull(meterRegistry, "meterRegistry"); - - if (registry != null) { - return; - - } - - registry = meterRegistry; - - Gauge.builder("rocks.num", RocksDBMetrics::updateRocks) - .description("Number of instance of RocksDB running in this node") - .register(registry); - - registerMemoryUse(); - } - - private static int updateRocks() { - int buf = getRocks(); - - if (buf != rocks.get()) { - rocks.set(buf); - registerMeter(); - } - - return buf; - } - - private static int getRocks() { - return rocksDBFactory.getSessionSize(); - } - - private static Set getGraphs() { - return rocksDBFactory.getGraphNames(); - } - - private static RocksDBSession getRocksDBSession(String graph) { - return rocksDBFactory.queryGraphDB(graph); - } - - - private static synchronized void registerMeter() { - Set graphs = getGraphs(); - - if (graphs == null) { - log.error( - "Failed to fetch the collection of names of graph, when invoking to register " + - "RocksDB gauge."); - return; - } - - graphs.forEach(g -> { - if (!graphSet.add(g)) { - return; - } - - StatisticsWrapper stats = new StatisticsWrapper(g); - statisticsHolder.put(g, stats); - - for (final TickerType ticker : TICKERS) { - String gaugeName = PREFIX + "." + ticker.name().toLowerCase(); - - saveGraphMeter(g, - Gauge.builder(gaugeName, () -> stats.getTickerCount(ticker)) - .description("RocksDB reported statistics for " + ticker.name()) - .tag("graph", g) - .register(registry) - ); - } - - for (final HistogramType histogram : HISTOGRAMS) { - registerHistogram(g, registry, histogram, stats); - } - - registrySessionRefNum(g); - - }); - - graphSet.removeAll(graphSet.stream().filter(g -> !graphs.contains(g)) - .peek(g -> removeMeters(g)) - .collect(Collectors.toList()) - ); - - } - - private static void saveGraphMeter(String g, Meter meter) { - graphMeterMap.computeIfAbsent(g, k -> new HashSet<>()).add(meter); - } - - private static void removeMeters(String g) { - graphMeterMap.getOrDefault(g, Collections.emptySet()).forEach(e -> registry.remove(e)); - } - - private static void registerHistogram(String graph, MeterRegistry registry, HistogramType - histogramType, StatisticsWrapper stats) { - - HistogramDataWrapper histogram = new HistogramDataWrapper(histogramType, - () -> stats.getHistogramData( - histogramType)); - histogramHolder.put(histogram, histogramType); - - String baseName = PREFIX + "." + histogramType.name().toLowerCase(); - saveGraphMeter(graph, - Gauge.builder(baseName + ".max", histogram, HistogramDataWrapper::getMax) - .tag("graph", graph).register(registry)); - saveGraphMeter(graph, Gauge.builder(baseName + ".mean", histogram, - HistogramDataWrapper::getAverage).tag("graph", graph) - .register(registry)); - saveGraphMeter(graph, - Gauge.builder(baseName + ".min", histogram, HistogramDataWrapper::getMin) - .tag("graph", graph).register(registry)); - - baseName = baseName + ".summary"; - saveGraphMeter(graph, Gauge.builder(baseName, histogram, HistogramDataWrapper::getMedian) - .tags("graph", graph, LABELS, LABEL_50).register(registry)); - saveGraphMeter(graph, - Gauge.builder(baseName, histogram, HistogramDataWrapper::getPercentile95) - .tags("graph", graph, LABELS, LABEL_95).register(registry)); - saveGraphMeter(graph, - Gauge.builder(baseName, histogram, HistogramDataWrapper::getPercentile99) - .tags("graph", graph, LABELS, LABEL_99).register(registry)); - saveGraphMeter(graph, - Gauge.builder(baseName + ".sum", histogram, HistogramDataWrapper::getSum) - .tags("graph", graph).register(registry)); - saveGraphMeter(graph, - Gauge.builder(baseName + ".count", histogram, HistogramDataWrapper::getCount) - .tags("graph", graph).register(registry)); - - } - - private static void registerMemoryUse() { - Gauge.builder(PREFIX + ".table.reader.total", memoryUseWrapper, - (e) -> e.getTableReaderTotal()) - .description("The current number of threads in the pool.") - .register(registry); - Gauge.builder(PREFIX + ".mem.table.total", memoryUseWrapper, (e) -> e.getMemTableTotal()) - .description("The current number of threads in the pool.") - .register(registry); - Gauge.builder(PREFIX + ".mem.table.unFlushed", memoryUseWrapper, - (e) -> e.getMemTableUnFlushed()) - .description("The current number of threads in the pool.") - .register(registry); - Gauge.builder(PREFIX + ".cache.total", memoryUseWrapper, (e) -> e.getCacheTotal()) - .description("The current number of threads in the pool.") - .register(registry); - Gauge.builder(PREFIX + ".block.cache.pinned-usage", memoryUseWrapper, - (e) -> e.getProperty("rocksdb.block-cache-pinned-usage")) - .description("The current number of threads in the pool.") - .register(registry); - - } - - private static void registrySessionRefNum(String graph) { - - SessionWrapper sessionWrapper = new SessionWrapper(graph); - saveGraphMeter(graph, - Gauge.builder(PREFIX + ".session.ref.count", sessionWrapper, - (e) -> e.getRefCount() - 1) - .description("The current amount of reference of session") - .tag("ref", "self").tag("graph", graph) - .strongReference(true) - .register(registry) - ); - - } - - private static class SessionWrapper { - - private final String graphName; - - SessionWrapper(String graph) { - - this.graphName = graph; - } - - public int getRefCount() { - try (RocksDBSession session = getRocksDBSession(graphName)) { - if (session != null) { - return getValue(session, e -> e.getRefCount(), -1); - } - return 0; - } - } - } - - private static class MemoryUseWrapper { - Map mems = null; - long lastTime = 0; - - private void loadData() { - if (mems == null || System.currentTimeMillis() - lastTime > 30000) { - mems = storeEngine.getBusinessHandler().getApproximateMemoryUsageByType(null); - lastTime = System.currentTimeMillis(); - } - } - - public Long getTableReaderTotal() { - loadData(); - return mems.get(MemoryUsageType.kTableReadersTotal); - } - - public Long getMemTableTotal() { - loadData(); - return mems.get(MemoryUsageType.kMemTableTotal); - } - - public Long getCacheTotal() { - loadData(); - return mems.get(MemoryUsageType.kCacheTotal); - } - - public Long getMemTableUnFlushed() { - loadData(); - return mems.get(MemoryUsageType.kMemTableUnFlushed); - } - - public Long getProperty(String property) { - Set graphs = rocksDBFactory.getGraphNames(); - if (graphs.size() > 0) { - try (RocksDBSession session = getRocksDBSession((String) graphs.toArray()[0])) { - if (session != null) { - return Long.parseLong(session.getProperty(property)); - } - } - } - return null; - } - } - - private static class StatisticsWrapper { - - private final String graphName; - long lastTime = 0; - private final Map tickerCounteMap = new ConcurrentHashMap<>(); - private final Map histogramDataMap = - new ConcurrentHashMap<>(); - - StatisticsWrapper(String graph) { - - this.graphName = graph; - loadData(); - - } - - private void loadData() { - if (System.currentTimeMillis() - lastTime < 30000) return; - lastTime = System.currentTimeMillis(); - try (RocksDBSession session = getRocksDBSession(graphName)) { - if (session == null) { - // log.error("Failed to fetch the RocksDBSession with graph's name: [ " + - // graph + " ]"); - return; - } - - Statistics statistics = session.getRocksDbStats(); - for (final TickerType ticker : TICKERS) { - tickerCounteMap.put(ticker, statistics.getTickerCount(ticker)); - } - - for (final HistogramType histogram : HISTOGRAMS) { - histogramDataMap.put(histogram, statistics.getHistogramData(histogram)); - } - } - } - - public long getTickerCount(TickerType tickerType) { - this.loadData(); - return tickerCounteMap.containsKey(tickerType) ? tickerCounteMap.get(tickerType) : 0; - } - - public HistogramData getHistogramData(HistogramType histogramType) { - this.loadData(); - return histogramDataMap.get(histogramType); - } - - } - - private static T getValue(S stat, Function fun, T defaultValue) { - if (stat == null) return defaultValue; - return fun.apply(stat); - } - - private static class HistogramDataWrapper { - private HistogramData data = new HistogramData(0d, 0d, 0d, 0d, 0d); - private long ts = System.currentTimeMillis() - 30_000; - private final Supplier supplier; - private final HistogramType histogramType; - - HistogramDataWrapper(HistogramType histogramType, Supplier supplier) { - this.supplier = supplier; - this.histogramType = histogramType; - } - - private HistogramData getData() { - if (System.currentTimeMillis() - this.ts > 30_000) { - HistogramData buf = this.supplier.get(); - if (buf != null) { - this.data = buf; - this.ts = System.currentTimeMillis(); - } - } - return this.data; - } - - public double getMedian() { - return getData().getMedian(); - } - - public double getPercentile95() { - return getData().getPercentile95(); - } - - public double getPercentile99() { - return getData().getPercentile99(); - } - - public double getAverage() { - return getData().getAverage(); - } - - public double getStandardDeviation() { - return getData().getStandardDeviation(); - } - - public double getMax() { - return getData().getMax(); - } - - public long getCount() { - return getData().getCount(); - } - - public long getSum() { - return getData().getSum(); - } - - public double getMin() { - return getData().getMin(); - } - - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.metrics; + +import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.HISTOGRAMS; +import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABELS; +import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_50; +import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_95; +import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.LABEL_99; +import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.PREFIX; +import static org.apache.hugegraph.store.node.metrics.RocksDBMetricsConst.TICKERS; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.node.util.HgAssert; +import org.rocksdb.HistogramData; +import org.rocksdb.HistogramType; +import org.rocksdb.MemoryUsageType; +import org.rocksdb.Statistics; +import org.rocksdb.TickerType; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; + +/** + * @author lynn.bond@hotmail.com on 2021/12/30 + * @version 1.2.0 on 2022/03/22 added auto meter removing when graph has been closed. + */ +@Slf4j +public class RocksDBMetrics { + private static MeterRegistry registry; + private final static RocksDBFactory rocksDBFactory = RocksDBFactory.getInstance(); + private final static AtomicInteger rocks = new AtomicInteger(0); + private final static Set graphSet = new HashSet<>(); + private final static HgStoreEngine storeEngine = HgStoreEngine.getInstance(); + private final static MemoryUseWrapper memoryUseWrapper = new MemoryUseWrapper(); + private final static Map statisticsHolder = new HashMap<>(); + private final static Map histogramHolder = new HashMap<>(); + private final static Map> graphMeterMap = new ConcurrentHashMap<>(); + + private RocksDBMetrics() { + } + + public static void init(final MeterRegistry meterRegistry) { + HgAssert.isArgumentNotNull(meterRegistry, "meterRegistry"); + + if (registry != null) { + return; + + } + + registry = meterRegistry; + + Gauge.builder("rocks.num", RocksDBMetrics::updateRocks) + .description("Number of instance of RocksDB running in this node") + .register(registry); + + registerMemoryUse(); + } + + private static int updateRocks() { + int buf = getRocks(); + + if (buf != rocks.get()) { + rocks.set(buf); + registerMeter(); + } + + return buf; + } + + private static int getRocks() { + return rocksDBFactory.getSessionSize(); + } + + private static Set getGraphs() { + return rocksDBFactory.getGraphNames(); + } + + private static RocksDBSession getRocksDBSession(String graph) { + return rocksDBFactory.queryGraphDB(graph); + } + + + private static synchronized void registerMeter() { + Set graphs = getGraphs(); + + if (graphs == null) { + log.error( + "Failed to fetch the collection of names of graph, when invoking to register " + + "RocksDB gauge."); + return; + } + + graphs.forEach(g -> { + if (!graphSet.add(g)) { + return; + } + + StatisticsWrapper stats = new StatisticsWrapper(g); + statisticsHolder.put(g, stats); + + for (final TickerType ticker : TICKERS) { + String gaugeName = PREFIX + "." + ticker.name().toLowerCase(); + + saveGraphMeter(g, + Gauge.builder(gaugeName, () -> stats.getTickerCount(ticker)) + .description("RocksDB reported statistics for " + ticker.name()) + .tag("graph", g) + .register(registry) + ); + } + + for (final HistogramType histogram : HISTOGRAMS) { + registerHistogram(g, registry, histogram, stats); + } + + registrySessionRefNum(g); + + }); + + graphSet.removeAll(graphSet.stream().filter(g -> !graphs.contains(g)) + .peek(g -> removeMeters(g)) + .collect(Collectors.toList()) + ); + + } + + private static void saveGraphMeter(String g, Meter meter) { + graphMeterMap.computeIfAbsent(g, k -> new HashSet<>()).add(meter); + } + + private static void removeMeters(String g) { + graphMeterMap.getOrDefault(g, Collections.emptySet()).forEach(e -> registry.remove(e)); + } + + private static void registerHistogram(String graph, MeterRegistry registry, HistogramType + histogramType, StatisticsWrapper stats) { + + HistogramDataWrapper histogram = new HistogramDataWrapper(histogramType, + () -> stats.getHistogramData( + histogramType)); + histogramHolder.put(histogram, histogramType); + + String baseName = PREFIX + "." + histogramType.name().toLowerCase(); + saveGraphMeter(graph, + Gauge.builder(baseName + ".max", histogram, HistogramDataWrapper::getMax) + .tag("graph", graph).register(registry)); + saveGraphMeter(graph, Gauge.builder(baseName + ".mean", histogram, + HistogramDataWrapper::getAverage).tag("graph", graph) + .register(registry)); + saveGraphMeter(graph, + Gauge.builder(baseName + ".min", histogram, HistogramDataWrapper::getMin) + .tag("graph", graph).register(registry)); + + baseName = baseName + ".summary"; + saveGraphMeter(graph, Gauge.builder(baseName, histogram, HistogramDataWrapper::getMedian) + .tags("graph", graph, LABELS, LABEL_50).register(registry)); + saveGraphMeter(graph, + Gauge.builder(baseName, histogram, HistogramDataWrapper::getPercentile95) + .tags("graph", graph, LABELS, LABEL_95).register(registry)); + saveGraphMeter(graph, + Gauge.builder(baseName, histogram, HistogramDataWrapper::getPercentile99) + .tags("graph", graph, LABELS, LABEL_99).register(registry)); + saveGraphMeter(graph, + Gauge.builder(baseName + ".sum", histogram, HistogramDataWrapper::getSum) + .tags("graph", graph).register(registry)); + saveGraphMeter(graph, + Gauge.builder(baseName + ".count", histogram, HistogramDataWrapper::getCount) + .tags("graph", graph).register(registry)); + + } + + private static void registerMemoryUse() { + Gauge.builder(PREFIX + ".table.reader.total", memoryUseWrapper, + (e) -> e.getTableReaderTotal()) + .description("The current number of threads in the pool.") + .register(registry); + Gauge.builder(PREFIX + ".mem.table.total", memoryUseWrapper, (e) -> e.getMemTableTotal()) + .description("The current number of threads in the pool.") + .register(registry); + Gauge.builder(PREFIX + ".mem.table.unFlushed", memoryUseWrapper, + (e) -> e.getMemTableUnFlushed()) + .description("The current number of threads in the pool.") + .register(registry); + Gauge.builder(PREFIX + ".cache.total", memoryUseWrapper, (e) -> e.getCacheTotal()) + .description("The current number of threads in the pool.") + .register(registry); + Gauge.builder(PREFIX + ".block.cache.pinned-usage", memoryUseWrapper, + (e) -> e.getProperty("rocksdb.block-cache-pinned-usage")) + .description("The current number of threads in the pool.") + .register(registry); + + } + + private static void registrySessionRefNum(String graph) { + + SessionWrapper sessionWrapper = new SessionWrapper(graph); + saveGraphMeter(graph, + Gauge.builder(PREFIX + ".session.ref.count", sessionWrapper, + (e) -> e.getRefCount() - 1) + .description("The current amount of reference of session") + .tag("ref", "self").tag("graph", graph) + .strongReference(true) + .register(registry) + ); + + } + + private static class SessionWrapper { + + private final String graphName; + + SessionWrapper(String graph) { + + this.graphName = graph; + } + + public int getRefCount() { + try (RocksDBSession session = getRocksDBSession(graphName)) { + if (session != null) { + return getValue(session, e -> e.getRefCount(), -1); + } + return 0; + } + } + } + + private static class MemoryUseWrapper { + Map mems = null; + long lastTime = 0; + + private void loadData() { + if (mems == null || System.currentTimeMillis() - lastTime > 30000) { + mems = storeEngine.getBusinessHandler().getApproximateMemoryUsageByType(null); + lastTime = System.currentTimeMillis(); + } + } + + public Long getTableReaderTotal() { + loadData(); + return mems.get(MemoryUsageType.kTableReadersTotal); + } + + public Long getMemTableTotal() { + loadData(); + return mems.get(MemoryUsageType.kMemTableTotal); + } + + public Long getCacheTotal() { + loadData(); + return mems.get(MemoryUsageType.kCacheTotal); + } + + public Long getMemTableUnFlushed() { + loadData(); + return mems.get(MemoryUsageType.kMemTableUnFlushed); + } + + public Long getProperty(String property) { + Set graphs = rocksDBFactory.getGraphNames(); + if (graphs.size() > 0) { + try (RocksDBSession session = getRocksDBSession((String) graphs.toArray()[0])) { + if (session != null) { + return Long.parseLong(session.getProperty(property)); + } + } + } + return null; + } + } + + private static class StatisticsWrapper { + + private final String graphName; + long lastTime = 0; + private final Map tickerCounteMap = new ConcurrentHashMap<>(); + private final Map histogramDataMap = + new ConcurrentHashMap<>(); + + StatisticsWrapper(String graph) { + + this.graphName = graph; + loadData(); + + } + + private void loadData() { + if (System.currentTimeMillis() - lastTime < 30000) return; + lastTime = System.currentTimeMillis(); + try (RocksDBSession session = getRocksDBSession(graphName)) { + if (session == null) { + // log.error("Failed to fetch the RocksDBSession with graph's name: [ " + + // graph + " ]"); + return; + } + + Statistics statistics = session.getRocksDbStats(); + for (final TickerType ticker : TICKERS) { + tickerCounteMap.put(ticker, statistics.getTickerCount(ticker)); + } + + for (final HistogramType histogram : HISTOGRAMS) { + histogramDataMap.put(histogram, statistics.getHistogramData(histogram)); + } + } + } + + public long getTickerCount(TickerType tickerType) { + this.loadData(); + return tickerCounteMap.containsKey(tickerType) ? tickerCounteMap.get(tickerType) : 0; + } + + public HistogramData getHistogramData(HistogramType histogramType) { + this.loadData(); + return histogramDataMap.get(histogramType); + } + + } + + private static T getValue(S stat, Function fun, T defaultValue) { + if (stat == null) return defaultValue; + return fun.apply(stat); + } + + private static class HistogramDataWrapper { + private HistogramData data = new HistogramData(0d, 0d, 0d, 0d, 0d); + private long ts = System.currentTimeMillis() - 30_000; + private final Supplier supplier; + private final HistogramType histogramType; + + HistogramDataWrapper(HistogramType histogramType, Supplier supplier) { + this.supplier = supplier; + this.histogramType = histogramType; + } + + private HistogramData getData() { + if (System.currentTimeMillis() - this.ts > 30_000) { + HistogramData buf = this.supplier.get(); + if (buf != null) { + this.data = buf; + this.ts = System.currentTimeMillis(); + } + } + return this.data; + } + + public double getMedian() { + return getData().getMedian(); + } + + public double getPercentile95() { + return getData().getPercentile95(); + } + + public double getPercentile99() { + return getData().getPercentile99(); + } + + public double getAverage() { + return getData().getAverage(); + } + + public double getStandardDeviation() { + return getData().getStandardDeviation(); + } + + public double getMax() { + return getData().getMax(); + } + + public long getCount() { + return getData().getCount(); + } + + public long getSum() { + return getData().getSum(); + } + + public double getMin() { + return getData().getMin(); + } + + } +} diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java index 553d42f93..0b8d9a9af 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java @@ -1,165 +1,165 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.metrics; - -import org.rocksdb.HistogramType; -import org.rocksdb.TickerType; - -/** - * @author lynn.bond@hotmail.com on 2021/12/30 - */ -public final class RocksDBMetricsConst { - public static final String PREFIX = "rocks.stats"; - public static final String LABELS = "quantile"; - public static final String LABEL_50 = "0.5"; - public static final String LABEL_95 = "0.95"; - public static final String LABEL_99 = "0.99"; - - // Tickers - RocksDB equivalent of counters - public static final TickerType[] TICKERS = { - TickerType.BLOCK_CACHE_ADD, - TickerType.BLOCK_CACHE_HIT, - TickerType.BLOCK_CACHE_ADD_FAILURES, - TickerType.BLOCK_CACHE_INDEX_MISS, - TickerType.BLOCK_CACHE_INDEX_HIT, - TickerType.BLOCK_CACHE_INDEX_ADD, - TickerType.BLOCK_CACHE_INDEX_BYTES_INSERT, - TickerType.BLOCK_CACHE_INDEX_BYTES_EVICT, - TickerType.BLOCK_CACHE_FILTER_MISS, - TickerType.BLOCK_CACHE_FILTER_HIT, - TickerType.BLOCK_CACHE_FILTER_ADD, - TickerType.BLOCK_CACHE_FILTER_BYTES_INSERT, - TickerType.BLOCK_CACHE_FILTER_BYTES_EVICT, - TickerType.BLOCK_CACHE_DATA_MISS, - TickerType.BLOCK_CACHE_DATA_HIT, - TickerType.BLOCK_CACHE_DATA_ADD, - TickerType.BLOCK_CACHE_DATA_BYTES_INSERT, - TickerType.BLOCK_CACHE_BYTES_READ, - TickerType.BLOCK_CACHE_BYTES_WRITE, - TickerType.BLOOM_FILTER_USEFUL, - TickerType.PERSISTENT_CACHE_HIT, - TickerType.PERSISTENT_CACHE_MISS, - TickerType.SIM_BLOCK_CACHE_HIT, - TickerType.SIM_BLOCK_CACHE_MISS, - TickerType.MEMTABLE_HIT, - TickerType.MEMTABLE_MISS, - TickerType.GET_HIT_L0, - TickerType.GET_HIT_L1, - TickerType.GET_HIT_L2_AND_UP, - TickerType.COMPACTION_KEY_DROP_NEWER_ENTRY, - TickerType.COMPACTION_KEY_DROP_OBSOLETE, - TickerType.COMPACTION_KEY_DROP_RANGE_DEL, - TickerType.COMPACTION_KEY_DROP_USER, - TickerType.COMPACTION_RANGE_DEL_DROP_OBSOLETE, - TickerType.NUMBER_KEYS_WRITTEN, - TickerType.NUMBER_KEYS_READ, - TickerType.NUMBER_KEYS_UPDATED, - TickerType.BYTES_WRITTEN, - TickerType.BYTES_READ, - TickerType.NUMBER_DB_SEEK, - TickerType.NUMBER_DB_NEXT, - TickerType.NUMBER_DB_PREV, - TickerType.NUMBER_DB_SEEK_FOUND, - TickerType.NUMBER_DB_NEXT_FOUND, - TickerType.NUMBER_DB_PREV_FOUND, - TickerType.ITER_BYTES_READ, - TickerType.NO_FILE_CLOSES, - TickerType.NO_FILE_OPENS, - TickerType.NO_FILE_ERRORS, - // TickerType.STALL_L0_SLOWDOWN_MICROS, - // TickerType.STALL_MEMTABLE_COMPACTION_MICROS, - // TickerType.STALL_L0_NUM_FILES_MICROS, - TickerType.STALL_MICROS, - TickerType.DB_MUTEX_WAIT_MICROS, - TickerType.RATE_LIMIT_DELAY_MILLIS, - TickerType.NO_ITERATORS, - TickerType.NUMBER_MULTIGET_BYTES_READ, - TickerType.NUMBER_MULTIGET_KEYS_READ, - TickerType.NUMBER_MULTIGET_CALLS, - TickerType.NUMBER_FILTERED_DELETES, - TickerType.NUMBER_MERGE_FAILURES, - TickerType.BLOOM_FILTER_PREFIX_CHECKED, - TickerType.BLOOM_FILTER_PREFIX_USEFUL, - TickerType.NUMBER_OF_RESEEKS_IN_ITERATION, - TickerType.GET_UPDATES_SINCE_CALLS, - TickerType.BLOCK_CACHE_COMPRESSED_MISS, - TickerType.BLOCK_CACHE_COMPRESSED_HIT, - TickerType.BLOCK_CACHE_COMPRESSED_ADD, - TickerType.BLOCK_CACHE_COMPRESSED_ADD_FAILURES, - TickerType.WAL_FILE_SYNCED, - TickerType.WAL_FILE_BYTES, - TickerType.WRITE_DONE_BY_SELF, - TickerType.WRITE_DONE_BY_OTHER, - TickerType.WRITE_TIMEDOUT, - TickerType.WRITE_WITH_WAL, - TickerType.COMPACT_READ_BYTES, - TickerType.COMPACT_WRITE_BYTES, - TickerType.FLUSH_WRITE_BYTES, - TickerType.NUMBER_DIRECT_LOAD_TABLE_PROPERTIES, - TickerType.NUMBER_SUPERVERSION_ACQUIRES, - TickerType.NUMBER_SUPERVERSION_RELEASES, - TickerType.NUMBER_SUPERVERSION_CLEANUPS, - TickerType.NUMBER_BLOCK_COMPRESSED, - TickerType.NUMBER_BLOCK_DECOMPRESSED, - TickerType.NUMBER_BLOCK_NOT_COMPRESSED, - TickerType.MERGE_OPERATION_TOTAL_TIME, - TickerType.FILTER_OPERATION_TOTAL_TIME, - TickerType.ROW_CACHE_HIT, - TickerType.ROW_CACHE_MISS, - TickerType.READ_AMP_ESTIMATE_USEFUL_BYTES, - TickerType.READ_AMP_TOTAL_READ_BYTES, - TickerType.NUMBER_RATE_LIMITER_DRAINS, - TickerType.NUMBER_ITER_SKIP, - TickerType.NUMBER_MULTIGET_KEYS_FOUND, - }; - - // Histograms - treated as prometheus summaries - public static final HistogramType[] HISTOGRAMS = { - HistogramType.DB_GET, - HistogramType.DB_WRITE, - HistogramType.COMPACTION_TIME, - HistogramType.SUBCOMPACTION_SETUP_TIME, - HistogramType.TABLE_SYNC_MICROS, - HistogramType.COMPACTION_OUTFILE_SYNC_MICROS, - HistogramType.WAL_FILE_SYNC_MICROS, - HistogramType.MANIFEST_FILE_SYNC_MICROS, - HistogramType.TABLE_OPEN_IO_MICROS, - HistogramType.DB_MULTIGET, - HistogramType.READ_BLOCK_COMPACTION_MICROS, - HistogramType.READ_BLOCK_GET_MICROS, - HistogramType.WRITE_RAW_BLOCK_MICROS, - HistogramType.STALL_L0_SLOWDOWN_COUNT, - HistogramType.STALL_MEMTABLE_COMPACTION_COUNT, - HistogramType.STALL_L0_NUM_FILES_COUNT, - HistogramType.HARD_RATE_LIMIT_DELAY_COUNT, - HistogramType.SOFT_RATE_LIMIT_DELAY_COUNT, - HistogramType.NUM_FILES_IN_SINGLE_COMPACTION, - HistogramType.DB_SEEK, - HistogramType.WRITE_STALL, - HistogramType.SST_READ_MICROS, - HistogramType.NUM_SUBCOMPACTIONS_SCHEDULED, - HistogramType.BYTES_PER_READ, - HistogramType.BYTES_PER_WRITE, - HistogramType.BYTES_PER_MULTIGET, - HistogramType.BYTES_COMPRESSED, - HistogramType.BYTES_DECOMPRESSED, - HistogramType.COMPRESSION_TIMES_NANOS, - HistogramType.DECOMPRESSION_TIMES_NANOS, - HistogramType.READ_NUM_MERGE_OPERANDS, - }; -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.metrics; + +import org.rocksdb.HistogramType; +import org.rocksdb.TickerType; + +/** + * @author lynn.bond@hotmail.com on 2021/12/30 + */ +public final class RocksDBMetricsConst { + public static final String PREFIX = "rocks.stats"; + public static final String LABELS = "quantile"; + public static final String LABEL_50 = "0.5"; + public static final String LABEL_95 = "0.95"; + public static final String LABEL_99 = "0.99"; + + // Tickers - RocksDB equivalent of counters + public static final TickerType[] TICKERS = { + TickerType.BLOCK_CACHE_ADD, + TickerType.BLOCK_CACHE_HIT, + TickerType.BLOCK_CACHE_ADD_FAILURES, + TickerType.BLOCK_CACHE_INDEX_MISS, + TickerType.BLOCK_CACHE_INDEX_HIT, + TickerType.BLOCK_CACHE_INDEX_ADD, + TickerType.BLOCK_CACHE_INDEX_BYTES_INSERT, + TickerType.BLOCK_CACHE_INDEX_BYTES_EVICT, + TickerType.BLOCK_CACHE_FILTER_MISS, + TickerType.BLOCK_CACHE_FILTER_HIT, + TickerType.BLOCK_CACHE_FILTER_ADD, + TickerType.BLOCK_CACHE_FILTER_BYTES_INSERT, + TickerType.BLOCK_CACHE_FILTER_BYTES_EVICT, + TickerType.BLOCK_CACHE_DATA_MISS, + TickerType.BLOCK_CACHE_DATA_HIT, + TickerType.BLOCK_CACHE_DATA_ADD, + TickerType.BLOCK_CACHE_DATA_BYTES_INSERT, + TickerType.BLOCK_CACHE_BYTES_READ, + TickerType.BLOCK_CACHE_BYTES_WRITE, + TickerType.BLOOM_FILTER_USEFUL, + TickerType.PERSISTENT_CACHE_HIT, + TickerType.PERSISTENT_CACHE_MISS, + TickerType.SIM_BLOCK_CACHE_HIT, + TickerType.SIM_BLOCK_CACHE_MISS, + TickerType.MEMTABLE_HIT, + TickerType.MEMTABLE_MISS, + TickerType.GET_HIT_L0, + TickerType.GET_HIT_L1, + TickerType.GET_HIT_L2_AND_UP, + TickerType.COMPACTION_KEY_DROP_NEWER_ENTRY, + TickerType.COMPACTION_KEY_DROP_OBSOLETE, + TickerType.COMPACTION_KEY_DROP_RANGE_DEL, + TickerType.COMPACTION_KEY_DROP_USER, + TickerType.COMPACTION_RANGE_DEL_DROP_OBSOLETE, + TickerType.NUMBER_KEYS_WRITTEN, + TickerType.NUMBER_KEYS_READ, + TickerType.NUMBER_KEYS_UPDATED, + TickerType.BYTES_WRITTEN, + TickerType.BYTES_READ, + TickerType.NUMBER_DB_SEEK, + TickerType.NUMBER_DB_NEXT, + TickerType.NUMBER_DB_PREV, + TickerType.NUMBER_DB_SEEK_FOUND, + TickerType.NUMBER_DB_NEXT_FOUND, + TickerType.NUMBER_DB_PREV_FOUND, + TickerType.ITER_BYTES_READ, + TickerType.NO_FILE_CLOSES, + TickerType.NO_FILE_OPENS, + TickerType.NO_FILE_ERRORS, + // TickerType.STALL_L0_SLOWDOWN_MICROS, + // TickerType.STALL_MEMTABLE_COMPACTION_MICROS, + // TickerType.STALL_L0_NUM_FILES_MICROS, + TickerType.STALL_MICROS, + TickerType.DB_MUTEX_WAIT_MICROS, + TickerType.RATE_LIMIT_DELAY_MILLIS, + TickerType.NO_ITERATORS, + TickerType.NUMBER_MULTIGET_BYTES_READ, + TickerType.NUMBER_MULTIGET_KEYS_READ, + TickerType.NUMBER_MULTIGET_CALLS, + TickerType.NUMBER_FILTERED_DELETES, + TickerType.NUMBER_MERGE_FAILURES, + TickerType.BLOOM_FILTER_PREFIX_CHECKED, + TickerType.BLOOM_FILTER_PREFIX_USEFUL, + TickerType.NUMBER_OF_RESEEKS_IN_ITERATION, + TickerType.GET_UPDATES_SINCE_CALLS, + TickerType.BLOCK_CACHE_COMPRESSED_MISS, + TickerType.BLOCK_CACHE_COMPRESSED_HIT, + TickerType.BLOCK_CACHE_COMPRESSED_ADD, + TickerType.BLOCK_CACHE_COMPRESSED_ADD_FAILURES, + TickerType.WAL_FILE_SYNCED, + TickerType.WAL_FILE_BYTES, + TickerType.WRITE_DONE_BY_SELF, + TickerType.WRITE_DONE_BY_OTHER, + TickerType.WRITE_TIMEDOUT, + TickerType.WRITE_WITH_WAL, + TickerType.COMPACT_READ_BYTES, + TickerType.COMPACT_WRITE_BYTES, + TickerType.FLUSH_WRITE_BYTES, + TickerType.NUMBER_DIRECT_LOAD_TABLE_PROPERTIES, + TickerType.NUMBER_SUPERVERSION_ACQUIRES, + TickerType.NUMBER_SUPERVERSION_RELEASES, + TickerType.NUMBER_SUPERVERSION_CLEANUPS, + TickerType.NUMBER_BLOCK_COMPRESSED, + TickerType.NUMBER_BLOCK_DECOMPRESSED, + TickerType.NUMBER_BLOCK_NOT_COMPRESSED, + TickerType.MERGE_OPERATION_TOTAL_TIME, + TickerType.FILTER_OPERATION_TOTAL_TIME, + TickerType.ROW_CACHE_HIT, + TickerType.ROW_CACHE_MISS, + TickerType.READ_AMP_ESTIMATE_USEFUL_BYTES, + TickerType.READ_AMP_TOTAL_READ_BYTES, + TickerType.NUMBER_RATE_LIMITER_DRAINS, + TickerType.NUMBER_ITER_SKIP, + TickerType.NUMBER_MULTIGET_KEYS_FOUND, + }; + + // Histograms - treated as prometheus summaries + public static final HistogramType[] HISTOGRAMS = { + HistogramType.DB_GET, + HistogramType.DB_WRITE, + HistogramType.COMPACTION_TIME, + HistogramType.SUBCOMPACTION_SETUP_TIME, + HistogramType.TABLE_SYNC_MICROS, + HistogramType.COMPACTION_OUTFILE_SYNC_MICROS, + HistogramType.WAL_FILE_SYNC_MICROS, + HistogramType.MANIFEST_FILE_SYNC_MICROS, + HistogramType.TABLE_OPEN_IO_MICROS, + HistogramType.DB_MULTIGET, + HistogramType.READ_BLOCK_COMPACTION_MICROS, + HistogramType.READ_BLOCK_GET_MICROS, + HistogramType.WRITE_RAW_BLOCK_MICROS, + HistogramType.STALL_L0_SLOWDOWN_COUNT, + HistogramType.STALL_MEMTABLE_COMPACTION_COUNT, + HistogramType.STALL_L0_NUM_FILES_COUNT, + HistogramType.HARD_RATE_LIMIT_DELAY_COUNT, + HistogramType.SOFT_RATE_LIMIT_DELAY_COUNT, + HistogramType.NUM_FILES_IN_SINGLE_COMPACTION, + HistogramType.DB_SEEK, + HistogramType.WRITE_STALL, + HistogramType.SST_READ_MICROS, + HistogramType.NUM_SUBCOMPACTIONS_SCHEDULED, + HistogramType.BYTES_PER_READ, + HistogramType.BYTES_PER_WRITE, + HistogramType.BYTES_PER_MULTIGET, + HistogramType.BYTES_COMPRESSED, + HistogramType.BYTES_DECOMPRESSED, + HistogramType.COMPRESSION_TIMES_NANOS, + HistogramType.DECOMPRESSION_TIMES_NANOS, + HistogramType.READ_NUM_MERGE_OPERANDS, + }; +} diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/StoreMetrics.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/StoreMetrics.java index fdb9a0b78..af94dac7e 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/StoreMetrics.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/StoreMetrics.java @@ -1,112 +1,112 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.metrics; - - -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; - -import org.apache.hugegraph.store.HgStoreEngine; -import org.apache.hugegraph.store.meta.Partition; - -import io.micrometer.core.instrument.Gauge; -import io.micrometer.core.instrument.MeterRegistry; - -/** - * @author lynn.bond@hotmail.com on 2021/12/28 - */ -public final class StoreMetrics { - public final static String PREFIX = "hg"; - private final static HgStoreEngine storeEngine = HgStoreEngine.getInstance(); - private final static AtomicInteger graphs = new AtomicInteger(0); - private static MeterRegistry registry; - - private StoreMetrics() { - } - - public synchronized static void init(MeterRegistry meterRegistry) { - if (registry == null) { - registry = meterRegistry; - registerMeters(); - } - } - - private static void registerMeters() { - Gauge.builder(PREFIX + ".up", () -> 1).register(registry); - Gauge.builder(PREFIX + ".graphs", StoreMetrics::updateGraphs) - .description("Number of graphs stored in this node") - .register(registry); - - } - - private static int getGraphs() { - return getGraphPartitions().size(); - } - - private static int updateGraphs() { - int buf = getGraphs(); - if (buf != graphs.get()) { - graphs.set(buf); - registerPartitionGauge(); - } - return buf; - } - - private static void registerPartitionGauge() { - Map> map = getGraphPartitions(); - - map.forEach((k, v) -> Gauge.builder(PREFIX + ".partitions", new PartitionsGetter(k)) - .description("Number of partitions stored in the node") - .tag("graph", k) - .register(registry)); - - } - - private static int getPartitions(String graph) { - Map map = getGraphPartitions().get(graph); - if (map == null) { - return 0; - } else { - return map.size(); - } - } - - private static Map> getGraphPartitions() { - Map> map = - storeEngine.getPartitionManager().getPartitions(); - if (map == null) { - return Collections.emptyMap(); - } - return map; - } - - private static class PartitionsGetter implements Supplier { - private final String graph; - - PartitionsGetter(String graph) { - this.graph = graph; - } - - @Override - public Number get() { - return getPartitions(this.graph); - } - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.metrics; + + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.meta.Partition; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; + +/** + * @author lynn.bond@hotmail.com on 2021/12/28 + */ +public final class StoreMetrics { + public final static String PREFIX = "hg"; + private final static HgStoreEngine storeEngine = HgStoreEngine.getInstance(); + private final static AtomicInteger graphs = new AtomicInteger(0); + private static MeterRegistry registry; + + private StoreMetrics() { + } + + public synchronized static void init(MeterRegistry meterRegistry) { + if (registry == null) { + registry = meterRegistry; + registerMeters(); + } + } + + private static void registerMeters() { + Gauge.builder(PREFIX + ".up", () -> 1).register(registry); + Gauge.builder(PREFIX + ".graphs", StoreMetrics::updateGraphs) + .description("Number of graphs stored in this node") + .register(registry); + + } + + private static int getGraphs() { + return getGraphPartitions().size(); + } + + private static int updateGraphs() { + int buf = getGraphs(); + if (buf != graphs.get()) { + graphs.set(buf); + registerPartitionGauge(); + } + return buf; + } + + private static void registerPartitionGauge() { + Map> map = getGraphPartitions(); + + map.forEach((k, v) -> Gauge.builder(PREFIX + ".partitions", new PartitionsGetter(k)) + .description("Number of partitions stored in the node") + .tag("graph", k) + .register(registry)); + + } + + private static int getPartitions(String graph) { + Map map = getGraphPartitions().get(graph); + if (map == null) { + return 0; + } else { + return map.size(); + } + } + + private static Map> getGraphPartitions() { + Map> map = + storeEngine.getPartitionManager().getPartitions(); + if (map == null) { + return Collections.emptyMap(); + } + return map; + } + + private static class PartitionsGetter implements Supplier { + private final String graph; + + PartitionsGetter(String graph) { + this.graph = graph; + } + + @Override + public Number get() { + return getPartitions(this.graph); + } + } +} diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Base58.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Base58.java index 5fdd6231c..2b1ddb8bd 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Base58.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/Base58.java @@ -1,177 +1,177 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.util; - -import java.io.UnsupportedEncodingException; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; - -/** - * Created with IntelliJ IDEA. - * User: noah - * Date: 8/2/13 - * Time: 10:36 AM - * To change this template use File | Settings | File Templates. - */ -public class Base58 { - - public static final char[] ALPHABET = - "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray(); - private static final int[] INDEXES = new int[128]; - - static { - for (int i = 0; i < INDEXES.length; i++) { - INDEXES[i] = -1; - } - for (int i = 0; i < ALPHABET.length; i++) { - INDEXES[ALPHABET[i]] = i; - } - } - - /** - * Encodes the given bytes in base58. No checksum is appended. - */ - public static String encode(byte[] input) { - if (input.length == 0) { - return ""; - } - input = copyOfRange(input, 0, input.length); - // Count leading zeroes. - int zeroCount = 0; - while (zeroCount < input.length && input[zeroCount] == 0) { - ++zeroCount; - } - // The actual encoding. - byte[] temp = new byte[input.length * 2]; - int j = temp.length; - - int startAt = zeroCount; - while (startAt < input.length) { - byte mod = divmod58(input, startAt); - if (input[startAt] == 0) { - ++startAt; - } - temp[--j] = (byte) ALPHABET[mod]; - } - - // Strip extra '1' if there are some after decoding. - while (j < temp.length && temp[j] == ALPHABET[0]) { - ++j; - } - // Add as many leading '1' as there were leading zeros. - while (--zeroCount >= 0) { - temp[--j] = (byte) ALPHABET[0]; - } - - byte[] output = copyOfRange(temp, j, temp.length); - return new String(output, StandardCharsets.US_ASCII); - } - - public static byte[] decode(String input) throws IllegalArgumentException { - if (input.length() == 0) { - return new byte[0]; - } - byte[] input58 = new byte[input.length()]; - // Transform the String to a base58 byte sequence - for (int i = 0; i < input.length(); ++i) { - char c = input.charAt(i); - - int digit58 = -1; - if (c >= 0 && c < 128) { - digit58 = INDEXES[c]; - } - if (digit58 < 0) { - throw new IllegalArgumentException("Illegal character " + c + " at " + i); - } - - input58[i] = (byte) digit58; - } - // Count leading zeroes - int zeroCount = 0; - while (zeroCount < input58.length && input58[zeroCount] == 0) { - ++zeroCount; - } - // The encoding - byte[] temp = new byte[input.length()]; - int j = temp.length; - - int startAt = zeroCount; - while (startAt < input58.length) { - byte mod = divmod256(input58, startAt); - if (input58[startAt] == 0) { - ++startAt; - } - - temp[--j] = mod; - } - // Do no add extra leading zeroes, move j to first non null byte. - while (j < temp.length && temp[j] == 0) { - ++j; - } - - return copyOfRange(temp, j - zeroCount, temp.length); - } - - public static BigInteger decodeToBigInteger(String input) throws IllegalArgumentException { - return new BigInteger(1, decode(input)); - } - - // - // number -> number / 58, returns number % 58 - // - private static byte divmod58(byte[] number, int startAt) { - int remainder = 0; - for (int i = startAt; i < number.length; i++) { - int digit256 = (int) number[i] & 0xFF; - int temp = remainder * 256 + digit256; - - number[i] = (byte) (temp / 58); - - remainder = temp % 58; - } - - return (byte) remainder; - } - - // - // number -> number / 256, returns number % 256 - // - private static byte divmod256(byte[] number58, int startAt) { - int remainder = 0; - for (int i = startAt; i < number58.length; i++) { - int digit58 = (int) number58[i] & 0xFF; - int temp = remainder * 58 + digit58; - - number58[i] = (byte) (temp / 256); - - remainder = temp % 256; - } - - return (byte) remainder; - } - - private static byte[] copyOfRange(byte[] source, int from, int to) { - byte[] range = new byte[to - from]; - System.arraycopy(source, from, range, 0, range.length); - - return range; - } - - -} - +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.util; + +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; + +/** + * Created with IntelliJ IDEA. + * User: noah + * Date: 8/2/13 + * Time: 10:36 AM + * To change this template use File | Settings | File Templates. + */ +public class Base58 { + + public static final char[] ALPHABET = + "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray(); + private static final int[] INDEXES = new int[128]; + + static { + for (int i = 0; i < INDEXES.length; i++) { + INDEXES[i] = -1; + } + for (int i = 0; i < ALPHABET.length; i++) { + INDEXES[ALPHABET[i]] = i; + } + } + + /** + * Encodes the given bytes in base58. No checksum is appended. + */ + public static String encode(byte[] input) { + if (input.length == 0) { + return ""; + } + input = copyOfRange(input, 0, input.length); + // Count leading zeroes. + int zeroCount = 0; + while (zeroCount < input.length && input[zeroCount] == 0) { + ++zeroCount; + } + // The actual encoding. + byte[] temp = new byte[input.length * 2]; + int j = temp.length; + + int startAt = zeroCount; + while (startAt < input.length) { + byte mod = divmod58(input, startAt); + if (input[startAt] == 0) { + ++startAt; + } + temp[--j] = (byte) ALPHABET[mod]; + } + + // Strip extra '1' if there are some after decoding. + while (j < temp.length && temp[j] == ALPHABET[0]) { + ++j; + } + // Add as many leading '1' as there were leading zeros. + while (--zeroCount >= 0) { + temp[--j] = (byte) ALPHABET[0]; + } + + byte[] output = copyOfRange(temp, j, temp.length); + return new String(output, StandardCharsets.US_ASCII); + } + + public static byte[] decode(String input) throws IllegalArgumentException { + if (input.length() == 0) { + return new byte[0]; + } + byte[] input58 = new byte[input.length()]; + // Transform the String to a base58 byte sequence + for (int i = 0; i < input.length(); ++i) { + char c = input.charAt(i); + + int digit58 = -1; + if (c >= 0 && c < 128) { + digit58 = INDEXES[c]; + } + if (digit58 < 0) { + throw new IllegalArgumentException("Illegal character " + c + " at " + i); + } + + input58[i] = (byte) digit58; + } + // Count leading zeroes + int zeroCount = 0; + while (zeroCount < input58.length && input58[zeroCount] == 0) { + ++zeroCount; + } + // The encoding + byte[] temp = new byte[input.length()]; + int j = temp.length; + + int startAt = zeroCount; + while (startAt < input58.length) { + byte mod = divmod256(input58, startAt); + if (input58[startAt] == 0) { + ++startAt; + } + + temp[--j] = mod; + } + // Do no add extra leading zeroes, move j to first non null byte. + while (j < temp.length && temp[j] == 0) { + ++j; + } + + return copyOfRange(temp, j - zeroCount, temp.length); + } + + public static BigInteger decodeToBigInteger(String input) throws IllegalArgumentException { + return new BigInteger(1, decode(input)); + } + + // + // number -> number / 58, returns number % 58 + // + private static byte divmod58(byte[] number, int startAt) { + int remainder = 0; + for (int i = startAt; i < number.length; i++) { + int digit256 = (int) number[i] & 0xFF; + int temp = remainder * 256 + digit256; + + number[i] = (byte) (temp / 58); + + remainder = temp % 58; + } + + return (byte) remainder; + } + + // + // number -> number / 256, returns number % 256 + // + private static byte divmod256(byte[] number58, int startAt) { + int remainder = 0; + for (int i = startAt; i < number58.length; i++) { + int digit58 = (int) number58[i] & 0xFF; + int temp = remainder * 58 + digit58; + + number58[i] = (byte) (temp / 256); + + remainder = temp % 256; + } + + return (byte) remainder; + } + + private static byte[] copyOfRange(byte[] source, int from, int to) { + byte[] range = new byte[to - from]; + System.arraycopy(source, from, range, 0, range.length); + + return range; + } + + +} + diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgBufferProxy.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgBufferProxy.java index 03cb5f721..5534a6f66 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgBufferProxy.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgBufferProxy.java @@ -1,174 +1,174 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.util; - -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; -import java.util.function.Supplier; - -import javax.annotation.CheckForNull; - -import lombok.extern.slf4j.Slf4j; - -/** - * @author lynn.bond@hotmail.com on 2022/3/15 - * @version 0.2.0 - */ -@Slf4j -public final class HgBufferProxy { - private final BlockingQueue> queue; - private final AtomicBoolean closed = new AtomicBoolean(false); - private final ReentrantLock lock = new ReentrantLock(); - private final Runnable applier; - private final Executor executor; - - public static HgBufferProxy of(Executor executor, Runnable applier) { - HgAssert.isArgumentNotNull(applier, "applier"); - HgAssert.isArgumentNotNull(executor, "executor"); - return new HgBufferProxy(executor, applier); - } - - private HgBufferProxy(Executor executor, Runnable applier) { - this.executor = executor; - this.applier = applier; - this.queue = new LinkedBlockingQueue<>(); - } - - public void send(T t) { - if (t == null) { - throw new IllegalArgumentException("the argument t is null"); - } - - if (this.closed.get()) { - log.warn("the proxy has been closed"); - return; - } - - this.lock.lock(); - try { - this.queue.offer(() -> t); - } finally { - lock.unlock(); - } - } - - /** - * return an item from the chan - * - * @throws RuntimeException when fail to receive an item. - */ - @CheckForNull - public T receive(int timeoutSeconds) { - return receive(timeoutSeconds, (timeout) -> { - throw new RuntimeException("Timeout, max time: " + timeout + " seconds;"); - }); - } - - private void apply() { - this.lock.lock(); - try { - if (!this.closed.get()) { - this.executor.execute(this.applier); - Thread.yield(); - } - } finally { - this.lock.unlock(); - } - - } - - /** - * return an item from the chan - * - * @return null when the chan has been closed - * @throws RuntimeException - */ - @CheckForNull - public T receive(int timeoutSeconds, Consumer timeoutCallBack) { - Supplier s = null; - - if (this.closed.get()) { - s = this.queue.poll(); - return s != null ? s.get() : null; - } - - if (this.queue.size() <= 1) { - this.apply(); - } - - lock.lock(); - try { - if (this.isClosed()) { - s = this.queue.poll(); - return s != null ? s.get() : null; - } - } finally { - lock.unlock(); - } - - try { - s = this.queue.poll(timeoutSeconds, TimeUnit.SECONDS); - } catch (Throwable t) { - log.error("Failed to receive a item from chan. cause by: ", t); - throw new RuntimeException(t); - } - - if (s == null) { - if (this.closed.get()) { - s = this.queue.poll(); - } else { - if (timeoutCallBack == null) { - throw new RuntimeException( - "Timeout, max time: " + timeoutSeconds + " seconds;"); - } else { - timeoutCallBack.accept(timeoutSeconds); - } - } - } - - return s != null ? s.get() : null; - - } - - public boolean isClosed() { - return this.closed.get(); - } - - /** - * @throws RuntimeException when fail to close the chan - */ - public void close() { - if (this.closed.get()) { - return; - } - lock.lock(); - this.closed.set(true); - try { - this.queue.offer(() -> null); - } finally { - lock.unlock(); - } - - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.util; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import javax.annotation.CheckForNull; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author lynn.bond@hotmail.com on 2022/3/15 + * @version 0.2.0 + */ +@Slf4j +public final class HgBufferProxy { + private final BlockingQueue> queue; + private final AtomicBoolean closed = new AtomicBoolean(false); + private final ReentrantLock lock = new ReentrantLock(); + private final Runnable applier; + private final Executor executor; + + public static HgBufferProxy of(Executor executor, Runnable applier) { + HgAssert.isArgumentNotNull(applier, "applier"); + HgAssert.isArgumentNotNull(executor, "executor"); + return new HgBufferProxy(executor, applier); + } + + private HgBufferProxy(Executor executor, Runnable applier) { + this.executor = executor; + this.applier = applier; + this.queue = new LinkedBlockingQueue<>(); + } + + public void send(T t) { + if (t == null) { + throw new IllegalArgumentException("the argument t is null"); + } + + if (this.closed.get()) { + log.warn("the proxy has been closed"); + return; + } + + this.lock.lock(); + try { + this.queue.offer(() -> t); + } finally { + lock.unlock(); + } + } + + /** + * return an item from the chan + * + * @throws RuntimeException when fail to receive an item. + */ + @CheckForNull + public T receive(int timeoutSeconds) { + return receive(timeoutSeconds, (timeout) -> { + throw new RuntimeException("Timeout, max time: " + timeout + " seconds;"); + }); + } + + private void apply() { + this.lock.lock(); + try { + if (!this.closed.get()) { + this.executor.execute(this.applier); + Thread.yield(); + } + } finally { + this.lock.unlock(); + } + + } + + /** + * return an item from the chan + * + * @return null when the chan has been closed + * @throws RuntimeException + */ + @CheckForNull + public T receive(int timeoutSeconds, Consumer timeoutCallBack) { + Supplier s = null; + + if (this.closed.get()) { + s = this.queue.poll(); + return s != null ? s.get() : null; + } + + if (this.queue.size() <= 1) { + this.apply(); + } + + lock.lock(); + try { + if (this.isClosed()) { + s = this.queue.poll(); + return s != null ? s.get() : null; + } + } finally { + lock.unlock(); + } + + try { + s = this.queue.poll(timeoutSeconds, TimeUnit.SECONDS); + } catch (Throwable t) { + log.error("Failed to receive a item from chan. cause by: ", t); + throw new RuntimeException(t); + } + + if (s == null) { + if (this.closed.get()) { + s = this.queue.poll(); + } else { + if (timeoutCallBack == null) { + throw new RuntimeException( + "Timeout, max time: " + timeoutSeconds + " seconds;"); + } else { + timeoutCallBack.accept(timeoutSeconds); + } + } + } + + return s != null ? s.get() : null; + + } + + public boolean isClosed() { + return this.closed.get(); + } + + /** + * @throws RuntimeException when fail to close the chan + */ + public void close() { + if (this.closed.get()) { + return; + } + lock.lock(); + this.closed.set(true); + try { + this.queue.offer(() -> null); + } finally { + lock.unlock(); + } + + } + +} diff --git a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgRegexUtil.java b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgRegexUtil.java index c83f93d72..9b77786ad 100644 --- a/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgRegexUtil.java +++ b/hg-store-node/src/main/java/org/apache/hugegraph/store/node/util/HgRegexUtil.java @@ -1,86 +1,86 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.util; - -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @author lynn.bond@hotmail.com created on 2022/03/07 - */ -public class HgRegexUtil { - - public static String getGroupValue(String regex, String source, int groupId) { - if (regex == null || "".equals(regex) || source == null || "".equals(source)) return null; - - String value = ""; - - Pattern p = Pattern.compile(regex); - Matcher m = p.matcher(source); - - while (m.find()) { - for (int i = 0; i <= m.groupCount(); i++) { - if (i == groupId) { - value = m.group(i); - } - } - } - return value; - } - - public static List toGroupValues(String regex, String source) { - if (regex == null || "".equals(regex) || source == null || "".equals(source)) return null; - - Pattern p = Pattern.compile(regex); - Matcher m = p.matcher(source); - - List list = null; - - while (m.find()) { - list = new ArrayList<>(m.groupCount()); - for (int i = 0; i <= m.groupCount(); i++) { - list.add(m.group(i)); - } - } - - return list; - } - - public static List getMatchList(String regex, String source) { - if (regex == null || "".equals(regex) || source == null || "".equals(source)) return null; - Pattern p = Pattern.compile(regex, Pattern.MULTILINE); - Matcher m = p.matcher(source); - List list = new ArrayList(); - while (m.find()) { - list.add(m.group(0)); - } - return list.isEmpty() ? null : list; - } - - - public static void main(String[] args) { - List res = toGroupValues("(replicator)(.+?:\\d+)(.*)", - "replicator_10.14.139.10:8081_append_entries_times"); - if (res != null) { - res.stream().forEach(e -> System.out.println(e)); - } - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.util; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author lynn.bond@hotmail.com created on 2022/03/07 + */ +public class HgRegexUtil { + + public static String getGroupValue(String regex, String source, int groupId) { + if (regex == null || "".equals(regex) || source == null || "".equals(source)) return null; + + String value = ""; + + Pattern p = Pattern.compile(regex); + Matcher m = p.matcher(source); + + while (m.find()) { + for (int i = 0; i <= m.groupCount(); i++) { + if (i == groupId) { + value = m.group(i); + } + } + } + return value; + } + + public static List toGroupValues(String regex, String source) { + if (regex == null || "".equals(regex) || source == null || "".equals(source)) return null; + + Pattern p = Pattern.compile(regex); + Matcher m = p.matcher(source); + + List list = null; + + while (m.find()) { + list = new ArrayList<>(m.groupCount()); + for (int i = 0; i <= m.groupCount(); i++) { + list.add(m.group(i)); + } + } + + return list; + } + + public static List getMatchList(String regex, String source) { + if (regex == null || "".equals(regex) || source == null || "".equals(source)) return null; + Pattern p = Pattern.compile(regex, Pattern.MULTILINE); + Matcher m = p.matcher(source); + List list = new ArrayList(); + while (m.find()) { + list.add(m.group(0)); + } + return list.isEmpty() ? null : list; + } + + + public static void main(String[] args) { + List res = toGroupValues("(replicator)(.+?:\\d+)(.*)", + "replicator_10.14.139.10:8081_append_entries_times"); + if (res != null) { + res.stream().forEach(e -> System.out.println(e)); + } + } + +} diff --git a/hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/JraftMetricsTest.java b/hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/JraftMetricsTest.java index bd7a8b1a0..81644a729 100644 --- a/hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/JraftMetricsTest.java +++ b/hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/JraftMetricsTest.java @@ -1,59 +1,59 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.store.node.metrics; - -import java.util.Arrays; -import java.util.List; - -import org.apache.hugegraph.store.node.util.HgRegexUtil; -import org.junit.Test; - - -/** - * @author lynn.bond@hotmail.com on 2022/3/7 - */ -public class JraftMetricsTest { - - @Test - public void testRefineLabel() { - String regex = "(replicator)(.+?:\\d+)(.*)"; - - String[] sources = { - "replicator_10.14.139.10:8081_append_entries_times", - "replicator10.14.139.10:8081appendentriestimes", - "replicator.10.14.139.10:8081.append.entries.times", - "replicator-10-14-139-10:8081-append-entries-times", - "replicator_hg_0_10_14_139_10:8081::100_replicate_inflights_count_min", - "replicasdf14-13dasfies-times", - }; - - - Arrays.stream(sources).forEach(e -> { - System.out.println("--- " + e + " ---"); - List list = HgRegexUtil.toGroupValues(regex, e); - if (list != null) { - list.forEach(System.out::println); - } else { - System.out.println("NONE"); - } - System.out.println(); - }); - - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.store.node.metrics; + +import java.util.Arrays; +import java.util.List; + +import org.apache.hugegraph.store.node.util.HgRegexUtil; +import org.junit.Test; + + +/** + * @author lynn.bond@hotmail.com on 2022/3/7 + */ +public class JraftMetricsTest { + + @Test + public void testRefineLabel() { + String regex = "(replicator)(.+?:\\d+)(.*)"; + + String[] sources = { + "replicator_10.14.139.10:8081_append_entries_times", + "replicator10.14.139.10:8081appendentriestimes", + "replicator.10.14.139.10:8081.append.entries.times", + "replicator-10-14-139-10:8081-append-entries-times", + "replicator_hg_0_10_14_139_10:8081::100_replicate_inflights_count_min", + "replicasdf14-13dasfies-times", + }; + + + Arrays.stream(sources).forEach(e -> { + System.out.println("--- " + e + " ---"); + List list = HgRegexUtil.toGroupValues(regex, e); + if (list != null) { + list.forEach(System.out::println); + } else { + System.out.println("NONE"); + } + System.out.println(); + }); + + } + +} diff --git a/hg-store-rocksdb/pom.xml b/hg-store-rocksdb/pom.xml index 772ada3fd..7600f55f1 100644 --- a/hg-store-rocksdb/pom.xml +++ b/hg-store-rocksdb/pom.xml @@ -1,89 +1,89 @@ - - - - - 4.0.0 - - - org.apache.hugegraph - hugegraph-store-root - ${revision} - - - hg-store-rocksdb - - - - org.apache.hugegraph - hugegraph-common - 1.8.12 - - - org.glassfish.jersey.inject - * - - - org.glassfish.jersey.media - * - - - org.glassfish.jersey.connectors - * - - - - - org.apache.hugegraph - hg-store-common - ${revision} - - - org.rocksdb - rocksdbjni - 7.7.3 - - - org.projectlombok - lombok - 1.18.22 - provided - - - org.apache.logging.log4j - log4j-slf4j-impl - 2.15.0 - - - com.alibaba - fastjson - 1.2.83 - - - - - Baidu_Local - http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local - - - Baidu_Local_Snapshots - http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots - - + + + + + 4.0.0 + + + org.apache.hugegraph + hugegraph-store-root + ${revision} + + + hg-store-rocksdb + + + + org.apache.hugegraph + hugegraph-common + 1.8.12 + + + org.glassfish.jersey.inject + * + + + org.glassfish.jersey.media + * + + + org.glassfish.jersey.connectors + * + + + + + org.apache.hugegraph + hg-store-common + ${revision} + + + org.rocksdb + rocksdbjni + 7.7.3 + + + org.projectlombok + lombok + 1.18.22 + provided + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.15.0 + + + com.alibaba + fastjson + 1.2.83 + + + + + Baidu_Local + http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local + + + Baidu_Local_Snapshots + http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 1d180d7f9..310cd0f82 100644 --- a/pom.xml +++ b/pom.xml @@ -1,256 +1,256 @@ - - - - - - 4.0.0 - - org.apache.hugegraph - hugegraph-store-root - ${revision} - pom - - - org.springframework.boot - spring-boot-starter-parent - 2.5.14 - - - - hg-store-common - hg-store-grpc - hg-store-client - hg-store-rocksdb - hg-store-core - hg-store-node - hg-store-dist - hg-store-cli - hg-store-test - - - - 1.5.0 - 11 - 11 - 3.6.4-SNAPSHOT - 2.15.0 - - - - - - org.apache.hugegraph - hg-store-common - ${project.version} - - - org.apache.hugegraph - hg-store-grpc - ${project.version} - - - org.apache.hugegraph - hg-store-rocksdb - ${project.version} - - - org.apache.hugegraph - hg-store-client - ${project.version} - - - org.apache.hugegraph - hg-store-core - ${project.version} - - - org.apache.hugegraph - hg-store-transfer - ${project.version} - - - org.apache.logging.log4j - log4j-slf4j-impl - 2.15.0 - - - - - - - junit - junit - test - - - - - - Baidu_Local - http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local - - - Baidu_Local_Snapshots - http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots - - - - - - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - - - prepare-agent - - - - - - - - - - client-test - - true - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.20 - - - client-test - - test - - test - - - - - - - - core-test - - true - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.20 - - - core-test - - test - - test - - - - - - - - common-test - - true - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.20 - - - common-test - - test - - test - - - - - - - - rocksdb-test - - true - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.20 - - - rocksdb-test - - test - - test - - - - - - - - - - - + + + + + + 4.0.0 + + org.apache.hugegraph + hugegraph-store-root + ${revision} + pom + + + org.springframework.boot + spring-boot-starter-parent + 2.5.14 + + + + hg-store-common + hg-store-grpc + hg-store-client + hg-store-rocksdb + hg-store-core + hg-store-node + hg-store-dist + hg-store-cli + hg-store-test + + + + 1.5.0 + 11 + 11 + 3.6.4-SNAPSHOT + 2.15.0 + + + + + + org.apache.hugegraph + hg-store-common + ${project.version} + + + org.apache.hugegraph + hg-store-grpc + ${project.version} + + + org.apache.hugegraph + hg-store-rocksdb + ${project.version} + + + org.apache.hugegraph + hg-store-client + ${project.version} + + + org.apache.hugegraph + hg-store-core + ${project.version} + + + org.apache.hugegraph + hg-store-transfer + ${project.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.15.0 + + + + + + + junit + junit + test + + + + + + Baidu_Local + http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local + + + Baidu_Local_Snapshots + http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots + + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.4 + + + + prepare-agent + + + + + + + + + + client-test + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + client-test + + test + + test + + + + + + + + core-test + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + core-test + + test + + test + + + + + + + + common-test + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + common-test + + test + + test + + + + + + + + rocksdb-test + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + rocksdb-test + + test + + test + + + + + + + + + + + diff --git a/settings.xml b/settings.xml index 226ca701f..5d92d8c18 100644 --- a/settings.xml +++ b/settings.xml @@ -1,132 +1,132 @@ - - - - - - - - - - - - - - - - - - - - baidu - - - baidu-nexus - http://maven.baidu-int.com/nexus/content/groups/public - - true - - - false - - - - baidu-nexus-snapshot - http://maven.baidu-int.com/nexus/content/groups/public-snapshots - - false - - - false - - - - - - - - - - - - - - - - - - - - Baidu_Local - http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local - - true - - - false - - - - Baidu_Local_Snapshots - - http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots - - - false - - - true - always - - - - - - baidu-nexus - http://maven.baidu-int.com/nexus/content/groups/public - - true - - - false - - - - baidu-nexus-snapshot - http://maven.baidu-int.com/nexus/content/groups/public-snapshots - - false - - - true - - - - - - - - - baidu - + + + + + + + + + + + + + + + + + + + + baidu + + + baidu-nexus + http://maven.baidu-int.com/nexus/content/groups/public + + true + + + false + + + + baidu-nexus-snapshot + http://maven.baidu-int.com/nexus/content/groups/public-snapshots + + false + + + false + + + + + + + + + + + + + + + + + + + + Baidu_Local + http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local + + true + + + false + + + + Baidu_Local_Snapshots + + http://maven.baidu-int.com/nexus/content/repositories/Baidu_Local_Snapshots + + + false + + + true + always + + + + + + baidu-nexus + http://maven.baidu-int.com/nexus/content/groups/public + + true + + + false + + + + baidu-nexus-snapshot + http://maven.baidu-int.com/nexus/content/groups/public-snapshots + + false + + + true + + + + + + + + + baidu + \ No newline at end of file